Release v3.7.0

- Track ghosted files to enable renames on files that will be deleted by
  third-party programs (boot files via clr-boot-manager).
- Prune orphaned renames from manifest (renamed-to deleted or renamed
  again)

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.travis.yml

@@ -21,5 +21,5 @@ install:
 # Ubuntu's default umask is 0002, but this break's swupd hash calculations.
 script:
         - sudo find test/functional -exec chmod g-w {} \;
-        - autoreconf --verbose --warnings=none --install --force && ./configure && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
+        - autoreconf --verbose --warnings=none --install --force && ./configure --enable-rename-detection && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
 after_failure: cat test-suite.log

Makefile.am

@@ -116,6 +116,7 @@ dist_check_SCRIPTS = \
 	test/functional/full-run-delta/test.bats \
 	test/functional/full-run/test.bats \
 	test/functional/fullfiles/test.bats \
+	test/functional/ghosting/test.bats \
 	test/functional/include-version-bump/test.bats \
 	test/functional/includes-deduplicate/test.bats \
 	test/functional/no-delta/test.bats \
@@ -127,6 +128,7 @@ dist_check_SCRIPTS = \
 
 if RENAMES
 dist_check_SCRIPTS += \
+	test/functional/orphaned-renames/test.bats \
 	test/functional/renames/test.bats
 endif
 endif

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.6.3, matthew.johnson@intel.com)
+AC_INIT(swupd-server, 3.7.0, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -117,6 +117,10 @@ struct file {
 	unsigned int is_file : 1;
 	unsigned int is_link : 1;
 	unsigned int is_deleted : 1;
+	/* a ghosted file is treated as deleted so that it can be used as a rename,
+	 * but is not actually deleted in the client due to a 3rd-party program
+	 * cleaning it up. This happens with boot files managed by a boot manager */
+	unsigned int is_ghosted : 1;
 	unsigned int is_manifest : 1;
 
 	/* and these are modifiers */
@@ -194,7 +198,7 @@ extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
 extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
-extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
+extern int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *, struct file *));
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
 extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);

src/analyze_fs.c

@@ -151,7 +151,13 @@ void populate_file_struct(struct file *file, char *filename)
 	ret = lstat(filename, &stat);
 	if (ret < 0) {
 		LOG(NULL, "stat error ", "%s: %s", filename, strerror(errno));
-		file->is_deleted = 1;
+		/* delete the file if it isn't found and isn't a boot file,
+		 * mark as ghosted if this is a boot file */
+		if (file->is_boot) {
+			file->is_ghosted = 1;
+		} else {
+			file->is_deleted = 1;
+		}
 		return;
 	}
 	file->stat.st_mode = stat.st_mode;

src/create_update.c

@@ -236,6 +236,16 @@ static int check_group_file(void)
 	return ret;
 }
 
+static bool both_ghosted(struct file *file1, struct file *file2)
+{
+	return (file1->is_ghosted && file2->is_ghosted);
+}
+
+static bool both_deleted(struct file *file1, struct file *file2)
+{
+	return (file1->is_deleted && file2->is_deleted);
+}
+
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -252,6 +262,7 @@ int main(int argc, char **argv)
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
 	int old_deleted = 0;
+	int old_ghosted = 0;
 
 	struct timeval current_time;
 	struct timeval previous_time;
@@ -325,12 +336,18 @@ int main(int argc, char **argv)
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
 
-	old_deleted = remove_old_deleted_files(old_full, new_full);
+	old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
 	if (old_deleted > 0) {
 		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
 		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
 	}
 
+	old_ghosted = remove_deprecated_files(old_full, new_full, both_ghosted);
+	if (old_ghosted > 0) {
+		LOG(NULL, "", "Old ghosted files (%d) removed from full manifest", old_ghosted);
+		printf("Old ghosted files (%d) removed from full manifest", old_ghosted);
+	}
+
 	apply_heuristics(new_full);
 
 	print_elapsed_time("full manifest creation", &previous_time, &current_time);
@@ -385,7 +402,8 @@ int main(int argc, char **argv)
 		/* Detect renamed files specifically for os-core */
 		rename_detection(new_core);
 #endif
-		old_deleted = remove_old_deleted_files(old_core, new_core);
+		old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
+		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
 		sort_manifest_by_version(new_core); /* sorts by filename */
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
@@ -393,8 +411,8 @@ int main(int argc, char **argv)
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
-		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", newfiles, old_deleted, old_ghosted);
+		printf("Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", newfiles, old_deleted, old_ghosted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
@@ -506,13 +524,14 @@ int main(int argc, char **argv)
 			rename_detection(newm);
 #endif
 			/* Step 6b: otherwise, write out the manifest */
-			old_deleted = remove_old_deleted_files(oldm, newm);
+			old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
+			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
-				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
-				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
+			if (newfiles > 0 || old_deleted > 0 || old_ghosted > 0 || changed_includes(oldm, newm)) {
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", group, newfiles, old_deleted, old_ghosted);
+				printf("%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", group, newfiles, old_deleted, old_ghosted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);

src/fullfiles.c

@@ -298,6 +298,11 @@ static void submit_fullfile_tasks(GList *files)
 		file = item->data;
 		item = g_list_next(item);
 
+		/* do not push ghosted files */
+		if (file->is_ghosted) {
+			continue;
+		}
+
 		ret = g_thread_pool_push(threadpool, file, &err);
 		if (ret == FALSE) {
 			printf("GThread create_fullfile_task push error\n");

src/manifest.c

@@ -240,10 +240,22 @@ struct manifest *manifest_from_file(int version, char *component)
 			assert(0); /* unknown file type */
 		}
 
-		if (c[1] == 'd') {
+		switch (c[1]) {
+		case 'd':
+			/* file is deleted */
 			file->is_deleted = 1;
-		} else if (c[1] != '.') {
-			assert(0); /* unknown deleted status */
+			break;
+		case 'g':
+			/* file is ghosted */
+			file->is_ghosted = 1;
+			break;
+		case '.':
+			/* no flag at this index */
+			break;
+		default:
+			/* unknown deleted status */
+			LOG(NULL, "Invalid flag at index 1", "%s", c[1]);
+			assert(0);
 		}
 
 		if (c[2] == 'C') {
@@ -428,15 +440,20 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			file3->is_config = file1->is_config;
 			file3->is_state = file1->is_state;
 			file3->is_boot = file1->is_boot;
+			/* ghost deleted boot files */
+			file3->is_ghosted = file1->is_boot && file1->is_deleted;
 
-			if (!file1->is_deleted) {
-				file3->last_change = m2->version;
-			} else {
-				file3->last_change = file1->last_change;
+			if (file3->is_ghosted || file1->is_deleted) {
+				/* if the new file is ghosted or the file was deleted, preserve
+				 * hash (all zeros if file1->is_deleted) and rename status */
 				file3->is_rename = file1->is_rename;
 				hash_assign(file1->hash, file3->hash);
 			}
 
+			/* for deleted files last_change remains the same.
+			 * otherwise last change is now */
+			file3->last_change = file1->is_deleted ? file1->last_change : m2->version;
+
 			file3->peer = file1;
 			file1->peer = file3;
 
@@ -658,6 +675,10 @@ char *file_type_to_string(struct file *file)
 		type[1] = 'd';
 	}
 
+	if (file->is_ghosted) {
+		type[1] = 'g';
+	}
+
 	if (file->is_config) {
 		type[2] = 'C';
 	}
@@ -968,7 +989,7 @@ bool changed_includes(struct manifest *old, struct manifest *new)
  *
  * Note: this function should be called after match_manifests().
  */
-int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
+int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *file1, struct file *file2))
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
@@ -1006,7 +1027,9 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->is_deleted && file2->is_deleted) {
+			/* use the comparison function passed in to determine if this file
+			 * should be removed */
+			if (compfunc(file1, file2)) {
 				GList *to_delete = list2;
 				list1 = g_list_next(list1);
 				list2 = g_list_next(list2);
@@ -1045,16 +1068,24 @@ int prune_manifest(struct manifest *manifest)
 		next = g_list_next(list);
 		file = list->data;
 
+		/* if the file is marked as deleted and renamed it is a renamed_from
+		 * file. If the rename_peer field is still NULL then it has been
+		 * orphaned. Prune these files */
+		if (file->is_deleted && file->is_rename && !file->rename_peer) {
+			manifest->files = g_list_delete_link(manifest->files, list);
+			manifest->count--;
+		}
+
 		if (OS_IS_STATELESS && (!file->is_deleted) && (file->is_config)) {
 			// toward being a stateless OS
 			LOG(file, "Skipping config file in manifest write", "component %s", manifest->component);
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		} else if (file->is_boot && file->is_deleted) {
-			// only expose the current best boot files, a client side entity can manage /boot's actual contents
-			// LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component);
-			manifest->files = g_list_delete_link(manifest->files, list);
-			manifest->count--;
+			/* mark boot files that are going away as ghosted, these will be
+			 * cleaned up with the next update */
+			file->is_deleted = 0;
+			file->is_ghosted = 1;
 		} else if (config_ban_debuginfo() && file_is_debuginfo(file->filename)) {
 			/* The configuration option to ban debuginfo from the manifests was
 			 * set in server.ini via the [Debuginfo][banned] option. Although

src/rename.c

@@ -184,7 +184,7 @@ double rename_score(struct file *old, struct file *new)
 	return score;
 }
 
-static void precompute_file_data(int version, const char *component, struct file *file, bool fast)
+static void precompute_file_data(int version, const char *component, struct file *file)
 {
 	char *c1, *c2;
 	char *filename = NULL;
@@ -301,7 +301,7 @@ static GList *new_filtered_list(GList *list, int version, int (*f)(struct file *
 	return newlist;
 }
 
-static int renamed_file_p(struct file *file, int unused)
+static int renamed_file_p(struct file *file, int unused __attribute__((unused)))
 {
 	return file->is_rename;
 }
@@ -331,7 +331,7 @@ static GList *list_new_files(struct manifest *manifest)
 	GList *ret = list;
 	for (list = g_list_first(list); list; list = g_list_next(list)) {
 		struct file *file = list->data;
-		precompute_file_data(manifest->version, manifest->component, file, true);
+		precompute_file_data(manifest->version, manifest->component, file);
 	}
 	return ret;
 }
@@ -357,7 +357,7 @@ static GList *list_deleted_files(struct manifest *manifest)
 		struct file *peer = file->peer;
 		/* Need to get things from the /full/ as we do not know
 		 * which  component may be coming from? */
-		precompute_file_data(peer->last_change, "full", peer, false);
+		precompute_file_data(peer->last_change, "full", peer);
 	}
 	return ret;
 }

src/versions.c

@@ -235,8 +235,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 	DIR *dir;
 	GList *list = NULL;
 	GList *cur_item, *next_item;
-	struct dirent entry;
-	struct dirent *result;
+	struct dirent *entry;
 	struct stat stat;
 	char *filename = NULL;
 	int idx, build_num, build_type, jump_point;
@@ -249,13 +248,13 @@ GList *get_last_versions_list(int next_version, int max_versions)
 		return NULL;
 	}
 
-	while (readdir_r(dir, &entry, &result) == 0 && result != NULL) {
-		if (strspn(entry.d_name, "0123456789") != strlen(entry.d_name)) {
+	while ((entry = readdir(dir))) {
+		if (strspn(entry->d_name, "0123456789") != strlen(entry->d_name)) {
 			continue;
 		}
 
 		free(filename);
-		string_or_die(&filename, "%s/%s", staging_dir, entry.d_name);
+		string_or_die(&filename, "%s/%s", staging_dir, entry->d_name);
 
 		if (lstat(filename, &stat)) {
 			LOG(NULL, "lstat failed", "path= %s, strerror= %s",
@@ -267,11 +266,11 @@ GList *get_last_versions_list(int next_version, int max_versions)
 			continue;
 		}
 
-		if (atoi(entry.d_name) >= next_version) {
+		if (atoi(entry->d_name) >= next_version) {
 			continue;
 		}
 
-		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry.d_name)));
+		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry->d_name)));
 	}
 	free(filename);
 	closedir(dir);

test/functional/ghosting/test.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  gen_file_plain_with_content 10 test-bundle usr/lib/kernel/bar testfile_contents
+  gen_file_plain_with_content 20 test-bundle usr/lib/kernel/baz new_testfile_contents
+  # make sure /usr/lib/kernel stays around
+  mkdir $DIR/image/30/test-bundle/usr/lib/kernel
+}
+
+@test "create updates while ghosting boot files before removing them" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  # version 10: add boot file to 10
+  [ 1 -eq $(grep 'F\.b\.	.*	10	/usr/lib/kernel/bar' $DIR/www/10/Manifest.full | wc -l) ]
+  # version 20: ghost boot file instead of deleting it when it doesn't exist
+  [ 1 -eq $(grep '\.gb\.	.*	20	/usr/lib/kernel/bar' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 20: add a new boot file
+  [ 1 -eq $(grep 'F\.b\.	.*	20	/usr/lib/kernel/baz' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 30: old ghosted file /usr/lib/kernel/bar cleaned up
+  [ 0 -eq $(grep '10	/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
+  # version 30: boot file added in version 20 ghosted
+  [ 1 -eq $(grep '\.gb\.	.*	30	/usr/lib/kernel/baz' $DIR/www/30/Manifest.full | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/orphaned-renames/test.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
+  gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
+  # different content just to make sure this works with delta renames as well as
+  # direct renames
+  gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
+}
+
+@test "create updates with renamed-to file getting deleted" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  # version 10: add file to 10
+  [ 1 -eq $(grep 'F\.\.\.	.*	10	/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  # version 20: rename bar to baz
+  [ 1 -eq $(grep '\.d\.r	.*	20	/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep 'F\.\.r	.*	20	/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  # version 30: prune original renamed-from file (bar), baz is now a renamed-from file
+  # Check for the new renamed-to file (foo)
+  [ 0 -eq $(grep '/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep '\.d\.r	.*	30	/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep 'F\.\.r	.*	30	/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80