Release v3.7.5

This release specifies that /usr/src/kernel/* files are not state files.
These files are needed to ship kernel source files for kernel
development.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.7.1, matthew.johnson@intel.com)
+AC_INIT(swupd-server, 3.7.5, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

src/create_update.c

@@ -336,7 +336,12 @@ int main(int argc, char **argv)
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
 
-	old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
+	if (old_full->format < new_full->format) {
+		old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
+	} else {
+		old_deleted = 0;
+	}
+
 	if (old_deleted > 0) {
 		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
 		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
@@ -402,7 +407,12 @@ int main(int argc, char **argv)
 		/* Detect renamed files specifically for os-core */
 		rename_detection(new_core);
 #endif
-		old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
+		if (old_core->format < new_core->format) {
+			old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
+		} else {
+			old_deleted = 0;
+		}
+
 		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
 		sort_manifest_by_version(new_core); /* sorts by filename */
 		/* clean up orphaned renames by marking them as deleted */
@@ -526,7 +536,12 @@ int main(int argc, char **argv)
 			rename_detection(newm);
 #endif
 			/* Step 6b: otherwise, write out the manifest */
-			old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
+			if (oldm->format < newm->format) {
+				old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
+			} else {
+				old_deleted = 0;
+			}
+
 			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);

src/heuristics.c

@@ -45,6 +45,7 @@ static void runtime_state_heuristics(struct file *file)
 	/* these are shipped directories that are not themselves state,
 	 * rather only their contents are state */
 	if ((strcmp(file->filename, "/usr/src/debug") == 0) ||
+	    (strcmp(file->filename, "/usr/src/kernel") == 0) ||
 	    (strcmp(file->filename, "/dev") == 0) ||
 	    (strcmp(file->filename, "/home") == 0) ||
 	    (strcmp(file->filename, "/proc") == 0) ||
@@ -56,6 +57,12 @@ static void runtime_state_heuristics(struct file *file)
 		return;
 	}
 
+	/* the contents of these directory are not state,
+	 * but it belongs to a state directory  */
+	if ((strncmp(file->filename, "/usr/src/kernel/", 16) == 0)) {
+		return;
+	}
+
 	/* the contents of these directory are state, ideally this never
 	 * triggers if our package builds are clean */
 	if ((strncmp(file->filename, "/dev/", 5) == 0) ||
@@ -72,7 +79,7 @@ static void runtime_state_heuristics(struct file *file)
 		return;
 	}
 
-	/* these are commonly added directories for user customizatio,
+	/* these are commonly added directories for user customization,
 	 * ideally this never triggers if our package builds are clean */
 	if ((strncmp(file->filename, "/acct", 5) == 0) ||
 	    (strncmp(file->filename, "/cache", 6) == 0) ||

src/manifest.c

@@ -323,6 +323,21 @@ struct manifest *manifest_from_file(int version, char *component)
 	return manifest;
 }
 
+/* do not set peer for the case where the old file was deleted and the new file
+ * was added back. This peer is normally set when the names match up and this
+ * triggers an attempt at creating a delta during pack creation. For this case
+ * where the old file is deleted or ghosted and is brought back we do not want
+ * this to be attempted, as the old file will not be present when trying to
+ * create the delta.
+ *
+ * We can go even further and not set peers for files when the old one did not
+ * exist (deleted or ghosted). It is not necessary to check the status of the
+ * new file */
+bool should_have_peer(struct file *file1, struct file *file2)
+{
+	return (!(file1->is_deleted || file1->is_ghosted));
+}
+
 void free_manifest(struct manifest *manifest)
 {
 	struct file *file;
@@ -340,6 +355,62 @@ void free_manifest(struct manifest *manifest)
 	free(manifest);
 }
 
+static bool same_file_contents(struct file *file1, struct file *file2)
+{
+	return (file1->is_dir == file2->is_dir &&
+		file1->is_link == file2->is_link &&
+		file1->is_deleted == file2->is_deleted &&
+		file1->is_file == file2->is_file &&
+		file1->is_config == file2->is_config &&
+		file1->is_state == file2->is_state &&
+		file1->is_boot == file2->is_boot &&
+		hash_compare(file1->hash, file2->hash));
+}
+
+/*
+ * Add a deleted file entry for it in the target list. However, since we're
+ * currently walking the list we HAVE to prepend the entry. Calling function
+ * should track to sort at the end.
+ */
+static void add_deleted_file(struct file *source, struct manifest *manifest)
+{
+	struct file *deleted;
+	deleted = calloc(1, sizeof(struct file));
+	if (deleted == NULL) {
+		assert(0);
+	}
+
+	deleted->filename = strdup(source->filename);
+	hash_set_zeros(deleted->hash);
+	deleted->is_deleted = 1;
+	deleted->is_config = source->is_config;
+	deleted->is_state = source->is_state;
+	deleted->is_boot = source->is_boot;
+	/* ghost deleted boot files */
+	deleted->is_ghosted = source->is_ghosted | source->is_boot & source->is_deleted;
+
+	if (deleted->is_ghosted || source->is_deleted) {
+		/* if the new file is ghosted or the file was deleted, preserve
+		 * hash (all zeros if source->is_deleted) and rename status */
+		deleted->is_rename = source->is_rename;
+		hash_assign(source->hash, deleted->hash);
+	}
+
+	/* for deleted files last_change remains the same.
+	 * otherwise last change is now */
+	deleted->last_change = source->is_deleted ? source->last_change : manifest->version;
+
+	deleted->peer = source;
+	source->peer = deleted;
+
+	/* if we are adding a deleted file we are walking the old and new
+	 * manifest files in-sync. we need to prepend this file in order to
+	 * not process it twice. This is why it is important for the calling
+	 * function to sort the list again at the end */
+	manifest->files = g_list_prepend(manifest->files, deleted);
+	manifest->count++;
+}
+
 /*
 backfill the "last changed" of each file in a manifest
 by comparing the hash against the same file in the previous manifest
@@ -354,7 +425,6 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 	struct file *file1, *file2;
 	int must_sort = 0;
 	int count = 0;
-	int first = 1;
 
 	if (!m1) {
 		printf("Matching manifests up failed: No old manifest!\n");
@@ -382,163 +452,72 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->is_deleted && file2->is_deleted && file1->is_rename) {
-				file2->is_rename = file1->is_rename;
-				hash_assign(file1->hash, file2->hash);
-			}
-
-			if (hash_compare(file1->hash, file2->hash) &&
-			    file1->is_dir == file2->is_dir &&
-			    file1->is_link == file2->is_link &&
-			    file1->is_deleted == file2->is_deleted &&
-			    file1->is_file == file2->is_file &&
-			    file1->is_config == file2->is_config &&
-			    file1->is_state == file2->is_state &&
-			    file1->is_boot == file2->is_boot &&
-			    file1->last_change >= minversion) {
+			/* file is present in both manifests */
+			if (same_file_contents(file1, file2) && file1->last_change >= minversion) {
+				/* file did not change */
 				file2->last_change = file1->last_change;
 				file2->is_rename = file1->is_rename;
 			} else {
+				/* file changed */
 				account_changed_file();
-				if (first) {
-					LOG(file1, "file changed", "");
-					first = 0;
-				}
 				count++;
 			}
 
-			if (!file1->is_deleted || file2->is_deleted) {
+			/* check if these files should be peers */
+			if (should_have_peer(file1, file2)) {
 				file1->peer = file2;
 				file2->peer = file1;
 			}
 
+			/* there was a match, advance both lists */
 			list1 = g_list_next(list1);
 			list2 = g_list_next(list2);
-			continue;
-		}
-		if (first) {
-			LOG(file1, "file added? ", "(file2 is %s)", file2->filename);
-			first = 0;
-		}
-		if (ret < 0) {
-			struct file *file3;
-			/*
-			 * if we get here, file1 got deleted... what we must do
-			 * is add a file entry for it in the target list.
-			 * However, since we're currently walking the list we
-			 * HAVE to prepend the entry.. and mark for sort at the
-			 * end.
-			 */
-			file3 = calloc(1, sizeof(struct file));
-			if (file3 == NULL) {
-				assert(0);
-			}
-
-			file3->filename = strdup(file1->filename);
-			hash_set_zeros(file3->hash);
-			file3->is_deleted = 1;
-			file3->is_config = file1->is_config;
-			file3->is_state = file1->is_state;
-			file3->is_boot = file1->is_boot;
-			/* ghost deleted boot files */
-			file3->is_ghosted = file1->is_boot && file1->is_deleted;
-
-			if (file3->is_ghosted || file1->is_deleted) {
-				/* if the new file is ghosted or the file was deleted, preserve
-				 * hash (all zeros if file1->is_deleted) and rename status */
-				file3->is_rename = file1->is_rename;
-				hash_assign(file1->hash, file3->hash);
-			}
-
-			/* for deleted files last_change remains the same.
-			 * otherwise last change is now */
-			file3->last_change = file1->is_deleted ? file1->last_change : m2->version;
-
-			file3->peer = file1;
-			file1->peer = file3;
-
-			list1 = g_list_next(list1);
-			m2->files = g_list_prepend(m2->files, file3);
-			m2->count++;
+		} else if (ret < 0) {
+			/* file1 was deleted, create entry for deleted file */
+			add_deleted_file(file1, m2);
 			if (!file1->is_deleted) {
 				account_deleted_file();
 				count++;
-				if (first) {
-					LOG(file1, "file got deleted", "");
-					first = 0;
-				}
 			}
+
 			must_sort = 1;
-			continue;
+			/* advance list1 for next file */
+			list1 = g_list_next(list1);
+		} else {
+			/* if we get here, ret is > 0, which means this is a new file added */
+			/* all we do is advance the pointer */
+			account_new_file();
+			count++;
+			/* advance list2 to check against same file in list1 */
+			list2 = g_list_next(list2);
 		}
-		/* if we get here, ret is > 0, which means this is a new file added */
-		/* all we do is advance the pointer */
-		account_new_file();
-		list2 = g_list_next(list2);
-		count++;
 	}
 
 	/* now deal with the tail ends */
-	while (list1) {
+	/* deleted files from list1 */
+	for (; list1; list1 = g_list_next(list1)) {
 		file1 = list1->data;
-
-		struct file *file3;
-
-		if (first) {
-			LOG(file1, "file changed tail", "");
-			first = 0;
-		}
-		count++;
-		/*
-		 * if we get here, file1 got deleted... what we must do is add
-		 * a file entry for it in the target list.  However, since
-		 * we're currently walking the list we HAVE to prepend the
-		 * entry.. and mark for sort at the end.
-		 */
-		file3 = calloc(1, sizeof(struct file));
-		if (file3 == NULL) {
-			assert(0);
-		}
-
-		file3->filename = strdup(file1->filename);
-		hash_set_zeros(file3->hash);
-		file3->is_deleted = 1;
-		file3->is_config = file1->is_config;
-		file3->is_state = file1->is_state;
-		file3->is_boot = file1->is_boot;
-
-		if (!file1->is_deleted) {
-			file3->last_change = m2->version;
-		} else {
-			file3->last_change = file1->last_change;
-			file3->is_rename = file1->is_rename;
-			hash_assign(file1->hash, file3->hash);
-		}
-
-		file3->peer = file1;
-		file1->peer = file3;
-
-		list1 = g_list_next(list1);
-		m2->files = g_list_prepend(m2->files, file3);
-		m2->count++;
+		add_deleted_file(file1, m2);
 		if (!file1->is_deleted) {
 			account_deleted_file();
+			count++;
 		}
+
 		must_sort = 1;
 	}
 
-	while (list2) {
+	/* added files from list2 */
+	for (; list2; list2 = g_list_next(list2)) {
 		account_new_file();
-		list2 = g_list_next(list2);
-		if (first) {
-			first = 0;
-		}
 		count++;
 	}
+
+	/* finally, sort the list if necessary */
 	if (must_sort) {
 		m2->files = g_list_sort(m2->files, file_sort_filename);
 	}
 
+	/* returned count of changed files */
 	return count;
 }
 
@@ -983,10 +962,9 @@ bool changed_includes(struct manifest *old, struct manifest *new)
 	return false;
 }
 
-/* For a format bump, it's convenient to remove deleted files from manifests
- * that last changed prior to the format bump, since they are no longer
- * considered for deletion as part of an update.
- *
+/* Remove files deprecated in this version as identified by the compfunc
+ * function. This function can be used to remove old deleted files over a format
+ * bump and remove deprecated ghosted files.
  * Note: this function should be called after match_manifests().
  */
 int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *file1, struct file *file2))
@@ -1005,11 +983,6 @@ int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*com
 		return -1;
 	}
 
-	/* This is the common case, so bail early, reporting no deletions */
-	if (m1->format == m2->format) {
-		return 0;
-	}
-
 	/* At this point, the manifest formats mismatch, and it's assumed that
 	 * m1 is the old manifest, and m2 is the new.
 	 */
@@ -1064,12 +1037,16 @@ void clean_renames(struct manifest *manifest)
 	list = g_list_first(manifest->files);
 	while (list) {
 		file = list->data;
-		/* if the file is marked as deleted and renamed it is a
-		 * renamed-from file. If the rename_peer field is still NULL
-		 * then it has been orphaned. Mark these as deleted */
-		if (file->is_deleted && file->is_rename && !file->rename_peer) {
-			hash_set_zeros(file->hash);
+		/* if a file is marked as a rename but has lost its rename_peer
+		 * it needs to be cleaned up */
+		if (file->is_rename && !file->rename_peer) {
+			/* no longer a rename */
 			file->is_rename = 0;
+			/* if the file is marked as deleted and renamed it is a
+			 * renamed-from file. Mark these as deleted now */
+			if (file->is_deleted) {
+				hash_set_zeros(file->hash);
+			}
 		}
 
 		list = g_list_next(list);

src/pack.c

@@ -138,12 +138,13 @@ static void prepare_pack(struct packdata *pack)
 		return;
 	}
 
+	/* read in manifest from file */
 	pack->end_manifest = manifest_from_file(pack->to, pack->module);
-
+	/* wipe any old packs (failed) and re-create pack directory structure */
 	empty_pack_stage(0, pack->from, pack->to, pack->module);
-
+	/* match up old and new manifests */
 	match_manifests(manifest, pack->end_manifest);
-
+	/* link renames together */
 	link_renames(pack->end_manifest->files, pack->to);
 }
 
@@ -161,7 +162,11 @@ static void make_pack_full_files(struct packdata *pack)
 	while (item) {
 		file = item->data;
 		item = g_list_next(item);
-		if ((!file->peer || file->peer->is_deleted) && !file->is_deleted && !file->rename_peer) {
+		/* only create full files if renames or deltas are not appropriate */
+		if ((!file->peer || file->peer->is_deleted || file->peer->is_ghosted) &&
+		    !file->is_deleted &&  /* no full-files for deletes */
+		    !file->is_ghosted &&  /* no full-files for ghosts */
+		    !file->rename_peer) { /* no full-files for renames */
 			char *from, *to;
 			char *fullfrom, *fullto;
 
@@ -248,6 +253,8 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 		file = item->data;
 		item = g_list_next(item);
 
+		/* skip old files, files without a peer, and files that are not
+		 * files, directories, or links */
 		if ((file->last_change <= pack->from) ||
 		    (!file->peer) ||
 		    (!file->is_file && !file->is_dir && !file->is_link)) {
@@ -257,7 +264,10 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
 			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
 
+		/* check for existence */
 		ret = stat(from, &stat_delta);
+		/* only add if delta does not already exist and the file is not
+		 * in files */
 		if (ret && !find_file_in_list(files, file)) {
 			files = g_list_prepend(files, file);
 		}

test/functional/ghosting/test.bats

@@ -52,7 +52,7 @@ setup() {
   # version 20: add a new boot file
   [ 1 -eq $(grep 'F\.b\.	.*	20	/usr/lib/kernel/baz' $DIR/www/20/Manifest.full | wc -l) ]
   # version 30: old ghosted file /usr/lib/kernel/bar cleaned up
-  [ 0 -eq $(grep '10	/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
+  [ 0 -eq $(grep '/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
   # version 30: boot file added in version 20 ghosted
   [ 1 -eq $(grep '\.gb\.	.*	30	/usr/lib/kernel/baz' $DIR/www/30/Manifest.full | wc -l) ]
 }

test/functional/orphaned-renames/test.bats

@@ -34,15 +34,19 @@ setup() {
   # /usr/lib/bar and /one will be renamed to /usr/lib/baz and /two
   gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
   gen_file_plain_with_content 10 test-bundle /one "$(printf 'a%.0s' {1..200})"
+  gen_file_plain_with_content 10 test-bundle /usr/lib/kernel/a "$(printf 'a%.0s' {1..210})"
 
   gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
   gen_file_plain_with_content 20 test-bundle /two "$(printf 'a%.0s' {1..200})"
+  gen_file_plain_with_content 20 test-bundle /usr/lib/kernel/ab "$(printf 'a%.0s' {1..210})"
 
   # different content just to make sure this works with delta renames as well as
   # direct renames
   gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
+  gen_file_plain_with_content 30 test-bundle /usr/lib/kernel/abc "$(printf 'a%.0s' {1..210})"
 
   gen_file_plain_with_content 40 test-bundle /usr/lib/foo "$(seq 100) new"
+  gen_file_plain_with_content 40 test-bundle /usr/lib/kernel/abc "$(printf 'a%.0s' {1..210})"
   # new file to force manifest generation
   gen_file_plain_with_content 40 test-bundle /a "testfile"
 }
@@ -68,11 +72,14 @@ setup() {
   # version 10: add files to 10
   [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/one' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.b\.\t.*\t10\t/usr/lib/kernel/a' $DIR/www/10/Manifest.test-bundle | wc -l) ]
   # version 20: rename bar to baz and one to two
   [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/one' $DIR/www/20/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/two' $DIR/www/20/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.gbr\t.*\t20\t/usr/lib/kernel/a' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.br\t.*\t20\t/usr/lib/kernel/ab' $DIR/www/20/Manifest.test-bundle | wc -l) ]
   # version 30: original renamed-from file (bar) is now orphaned and therefore
   # deleted, baz is now a renamed-from file and foo is a renamed-to file.
   # /two was deleted in this version, so both /one and /two should be marked as
@@ -82,6 +89,8 @@ setup() {
   [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.gbr\t.*\t30\t/usr/lib/kernel/ab' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.br\t.*\t30\t/usr/lib/kernel/abc' $DIR/www/30/Manifest.test-bundle | wc -l) ]
 
   # version 40: the existing rename from baz -> foo must persist while all
   # others remain deleted
@@ -90,6 +99,7 @@ setup() {
   [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/40/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/40/Manifest.test-bundle | wc -l) ]
   [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.b\.\t.*\t30\t/usr/lib/kernel/abc' $DIR/www/40/Manifest.test-bundle | wc -l) ]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80