Release v3.7.1

This release fixes the bug introduced in v3.7.0 that caused valid
renames from old versions to be dropped. Instead of pruning orphaned
renames, they are now marked as deleted as well.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.7.0, matthew.johnson@intel.com)
+AC_INIT(swupd-server, 3.7.1, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -198,6 +198,7 @@ extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
 extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
+extern void clean_renames(struct manifest *manifest);
 extern int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *, struct file *));
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
@@ -257,6 +258,7 @@ extern void type_change_detection(struct manifest *manifest);
 
 extern void rename_detection(struct manifest *manifest);
 extern void link_renames(GList *newfiles, int to_version);
+extern void final_link(GList *files);
 extern void __create_delta(struct file *file, int from_version, char *from_hash);
 
 extern void account_delta_hit(void);

src/create_update.c

@@ -405,6 +405,8 @@ int main(int argc, char **argv)
 		old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
 		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
 		sort_manifest_by_version(new_core); /* sorts by filename */
+		/* clean up orphaned renames by marking them as deleted */
+		clean_renames(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
 			LOG(NULL, "", "Core component has not changed (after pruning), exiting");
@@ -528,6 +530,8 @@ int main(int argc, char **argv)
 			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
+			/* clean up orphaned renames by marking them as deleted */
+			clean_renames(newm);
 			newfiles = prune_manifest(newm);
 			if (newfiles > 0 || old_deleted > 0 || old_ghosted > 0 || changed_includes(oldm, newm)) {
 				LOG(NULL, "", "%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", group, newfiles, old_deleted, old_ghosted);
@@ -568,6 +572,8 @@ int main(int argc, char **argv)
 	maximize_to_full(new_MoM, new_full);
 
 	sort_manifest_by_version(new_full);
+	/* clean up orphaned renames by marking them as deleted */
+	clean_renames(new_full);
 	prune_manifest(new_full);
 	if (write_manifest(new_full) != 0) {
 		goto exit;

src/manifest.c

@@ -1050,6 +1050,32 @@ int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*com
 	return count;
 }
 
+/* Performs a final link of renames in the manifest to catch any renames from
+ * old versions and then removes any orphaned renames */
+void clean_renames(struct manifest *manifest)
+{
+	GList *list;
+	struct file *file;
+
+	/* make sure all renames are linked, this is necessary for renames
+	 * from old manifests that carry over to the current one */
+	final_link(manifest->files);
+
+	list = g_list_first(manifest->files);
+	while (list) {
+		file = list->data;
+		/* if the file is marked as deleted and renamed it is a
+		 * renamed-from file. If the rename_peer field is still NULL
+		 * then it has been orphaned. Mark these as deleted */
+		if (file->is_deleted && file->is_rename && !file->rename_peer) {
+			hash_set_zeros(file->hash);
+			file->is_rename = 0;
+		}
+
+		list = g_list_next(list);
+	}
+}
+
 /* Conditionally remove some things from a manifest.
  * Returns > 0 when the pruned manifest has new files.
  * Returns 0 when the pruned manifest no longer has new files.
@@ -1068,14 +1094,6 @@ int prune_manifest(struct manifest *manifest)
 		next = g_list_next(list);
 		file = list->data;
 
-		/* if the file is marked as deleted and renamed it is a renamed_from
-		 * file. If the rename_peer field is still NULL then it has been
-		 * orphaned. Prune these files */
-		if (file->is_deleted && file->is_rename && !file->rename_peer) {
-			manifest->files = g_list_delete_link(manifest->files, list);
-			manifest->count--;
-		}
-
 		if (OS_IS_STATELESS && (!file->is_deleted) && (file->is_config)) {
 			// toward being a stateless OS
 			LOG(file, "Skipping config file in manifest write", "component %s", manifest->component);

src/rename.c

@@ -446,6 +446,48 @@ redo:
 	g_list_free(deleted_files);
 }
 
+void final_link(GList *files)
+{
+	GList *list1, *list2;
+	struct file *file1, *file2;
+
+	list1 = new_list_renamed_files(files);
+	list1 = list2 = g_list_sort(list1, file_sort_version);
+
+	for (; list1; list1 = g_list_next(list1)) {
+		file1 = list1->data;
+		/* do not re-link already-linked renames */
+		if (file1->rename_peer != NULL) {
+			continue;
+		}
+
+		/* check the rest of the list */
+		list2 = g_list_next(list1);
+		for (; list2; list2 = g_list_next(list2)) {
+			file2 = list2->data;
+			/* must be the same version to link */
+			if (file1->last_change != file2->last_change) {
+				continue;
+			}
+
+			/* need one and only one to be deleted to link as this
+			 * indicates a renamed-from/renamed-to relationship
+			 * (.d.r -> F..r) */
+			if (file1->is_deleted == file2->is_deleted) {
+				continue;
+			}
+
+			/* must have same hash to link */
+			if (!hash_compare(file2->hash, file1->hash)) {
+				continue;
+			}
+
+			file1->rename_peer = file2;
+			file2->rename_peer = file1;
+		}
+	}
+}
+
 /* What do we need this for?
  *
  * rename_detection has already set up the links in the manifest it

test/functional/orphaned-renames/test.bats

@@ -26,11 +26,25 @@ setup() {
   track_bundle 30 os-core
   track_bundle 30 test-bundle
 
+  set_os_release 40 os-core
+  set_os_release 40 test-bundle
+  track_bundle 40 os-core
+  track_bundle 40 test-bundle
+
+  # /usr/lib/bar and /one will be renamed to /usr/lib/baz and /two
   gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
+  gen_file_plain_with_content 10 test-bundle /one "$(printf 'a%.0s' {1..200})"
+
   gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
+  gen_file_plain_with_content 20 test-bundle /two "$(printf 'a%.0s' {1..200})"
+
   # different content just to make sure this works with delta renames as well as
   # direct renames
   gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
+
+  gen_file_plain_with_content 40 test-bundle /usr/lib/foo "$(seq 100) new"
+  # new file to force manifest generation
+  gen_file_plain_with_content 40 test-bundle /a "testfile"
 }
 
 @test "create updates with renamed-to file getting deleted" {
@@ -46,16 +60,36 @@ setup() {
 
   sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
   sudo $MAKE_FULLFILES --statedir $DIR 30
-  # version 10: add file to 10
-  [ 1 -eq $(grep 'F\.\.\.	.*	10	/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
-  # version 20: rename bar to baz
-  [ 1 -eq $(grep '\.d\.r	.*	20	/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep 'F\.\.r	.*	20	/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  # version 30: prune original renamed-from file (bar), baz is now a renamed-from file
-  # Check for the new renamed-to file (foo)
-  [ 0 -eq $(grep '/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep '\.d\.r	.*	30	/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep 'F\.\.r	.*	30	/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+
+  set_latest_ver 30
+
+  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 40
+  # version 10: add files to 10
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/one' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  # version 20: rename bar to baz and one to two
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/one' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/two' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  # version 30: original renamed-from file (bar) is now orphaned and therefore
+  # deleted, baz is now a renamed-from file and foo is a renamed-to file.
+  # /two was deleted in this version, so both /one and /two should be marked as
+  # deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+
+  # version 40: the existing rename from baz -> foo must persist while all
+  # others remain deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/40/Manifest.test-bundle | wc -l) ]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80