Release v3.6.1

This release fixes a bug where swupd-server was creating the staged and
delta directories within the packs with 0750 permissions instead of the
expected 0700 permissions. It also adds a configuration option to
server.ini to ban debuginfo from the manifests and configure where
debuginfo libs and src are stored.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.travis.yml

@@ -11,9 +11,9 @@ before_install:
         - sudo apt-get install -y libmagic-dev
 
 install:
-        - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
-        - tar -xvf check-0.11.0.tar.gz
-        - pushd check-0.11.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - wget http://downloads.sourceforge.net/project/check/check/0.10.0/check-0.10.0.tar.gz
+        - tar -xvf check-0.10.0.tar.gz
+        - pushd check-0.10.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
         - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
         - tar -xvf bsdiff-1.0.2.tar.xz
         - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd
@@ -21,5 +21,5 @@ install:
 # Ubuntu's default umask is 0002, but this break's swupd hash calculations.
 script:
         - sudo find test/functional -exec chmod g-w {} \;
-        - autoreconf --verbose --warnings=none --install --force && ./configure --enable-rename-detection && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
+        - autoreconf --verbose --warnings=none --install --force && ./configure && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
 after_failure: cat test-suite.log

Makefile.am

@@ -111,12 +111,10 @@ dist_check_SCRIPTS = \
 	test/functional/contentsize-across-versions-includes/test.bats \
 	test/functional/delete-no-version-bump/test.bats \
 	test/functional/file-name-blacklisted/test.bats \
-	test/functional/file-name-debuginfo/test.bats \
 	test/functional/format-no-decrement/test.bats \
 	test/functional/full-run-delta/test.bats \
 	test/functional/full-run/test.bats \
 	test/functional/fullfiles/test.bats \
-	test/functional/ghosting/test.bats \
 	test/functional/include-version-bump/test.bats \
 	test/functional/includes-deduplicate/test.bats \
 	test/functional/no-delta/test.bats \
@@ -128,7 +126,6 @@ dist_check_SCRIPTS = \
 
 if RENAMES
 dist_check_SCRIPTS += \
-	test/functional/orphaned-renames/test.bats \
 	test/functional/renames/test.bats
 endif
 endif

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.7.2, matthew.johnson@intel.com)
+AC_INIT(swupd-server, 3.6.1, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -117,10 +117,6 @@ struct file {
 	unsigned int is_file : 1;
 	unsigned int is_link : 1;
 	unsigned int is_deleted : 1;
-	/* a ghosted file is treated as deleted so that it can be used as a rename,
-	 * but is not actually deleted in the client due to a 3rd-party program
-	 * cleaning it up. This happens with boot files managed by a boot manager */
-	unsigned int is_ghosted : 1;
 	unsigned int is_manifest : 1;
 
 	/* and these are modifiers */
@@ -179,9 +175,7 @@ extern void release_configuration_data(void);
 extern char *config_image_base(void);
 extern char *config_output_dir(void);
 extern char *config_empty_dir(void);
-extern char *config_debuginfo_path(const char *path);
 extern int config_initial_version(void);
-extern bool config_ban_debuginfo(void);
 
 extern void read_current_version(char *filename);
 extern void write_new_version(char *filename, int version);
@@ -198,8 +192,7 @@ extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
 extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
-extern void clean_renames(struct manifest *manifest);
-extern int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *, struct file *));
+extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
 extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);
@@ -258,7 +251,6 @@ extern void type_change_detection(struct manifest *manifest);
 
 extern void rename_detection(struct manifest *manifest);
 extern void link_renames(GList *newfiles, int to_version);
-extern void final_link(GList *files);
 extern void __create_delta(struct file *file, int from_version, char *from_hash);
 
 extern void account_delta_hit(void);
@@ -268,18 +260,10 @@ extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
 extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
-extern int system_argv_pipe(char *const lhscmd[], char *const rhscmd[]);
-extern int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
-			       int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
-extern void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
-			 int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
 extern int system_argv(char *const argv[]);
-extern int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[]);
-extern pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
-extern void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
-extern void move_fd(int oldfd, int newfd);
-extern int wait_process_terminate(pid_t pid);
+extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
+extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+			    char *const argvp2[], int stdoutp2, int stderrp2);
 extern int num_threads(float scaling);
-extern bool file_is_debuginfo(const char *path);
 
 #endif

server.ini

@@ -2,8 +2,3 @@
 emptydir=/var/lib/update/empty/
 imagebase=/var/lib/update/image/
 outputdir=/var/lib/update/www/
-
-[Debuginfo]
-banned=true
-lib=/usr/lib/debug/
-src=/usr/src/debug/

src/analyze_fs.c

@@ -151,13 +151,7 @@ void populate_file_struct(struct file *file, char *filename)
 	ret = lstat(filename, &stat);
 	if (ret < 0) {
 		LOG(NULL, "stat error ", "%s: %s", filename, strerror(errno));
-		/* delete the file if it isn't found and isn't a boot file,
-		 * mark as ghosted if this is a boot file */
-		if (file->is_boot) {
-			file->is_ghosted = 1;
-		} else {
-			file->is_deleted = 1;
-		}
+		file->is_deleted = 1;
 		return;
 	}
 	file->stat.st_mode = stat.st_mode;
@@ -320,13 +314,6 @@ static struct file *add_file(struct manifest *manifest,
 	GError *err = NULL;
 	struct file *file;
 
-	if (config_ban_debuginfo() && file_is_debuginfo(sub_filename)) {
-		printf("WARNING: File %s is banned ...skipping.\n", sub_filename);
-		free(sub_filename);
-		free(fullname);
-		return NULL;
-	}
-
 	if (illegal_characters(entry_name)) {
 		printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
 		free(sub_filename);

src/config.c

@@ -67,20 +67,6 @@ int config_initial_version(void)
 	return version;
 }
 
-bool config_ban_debuginfo(void)
-{
-	assert(keyfile != NULL);
-
-	return g_key_file_get_boolean(keyfile, "Debuginfo", "banned", NULL);
-}
-
-char *config_debuginfo_path(const char *comp)
-{
-	assert(keyfile != NULL);
-
-	return g_key_file_get_value(keyfile, "Debuginfo", comp, NULL);
-}
-
 bool read_configuration_file(char *filename)
 {
 	GError *error = NULL;

src/create_update.c

@@ -236,16 +236,6 @@ static int check_group_file(void)
 	return ret;
 }
 
-static bool both_ghosted(struct file *file1, struct file *file2)
-{
-	return (file1->is_ghosted && file2->is_ghosted);
-}
-
-static bool both_deleted(struct file *file1, struct file *file2)
-{
-	return (file1->is_deleted && file2->is_deleted);
-}
-
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -262,7 +252,6 @@ int main(int argc, char **argv)
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
 	int old_deleted = 0;
-	int old_ghosted = 0;
 
 	struct timeval current_time;
 	struct timeval previous_time;
@@ -336,23 +325,12 @@ int main(int argc, char **argv)
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
 
-	if (old_full->format < new_full->format) {
-		old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
-	} else {
-		old_deleted = 0;
-	}
-
+	old_deleted = remove_old_deleted_files(old_full, new_full);
 	if (old_deleted > 0) {
 		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
 		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
 	}
 
-	old_ghosted = remove_deprecated_files(old_full, new_full, both_ghosted);
-	if (old_ghosted > 0) {
-		LOG(NULL, "", "Old ghosted files (%d) removed from full manifest", old_ghosted);
-		printf("Old ghosted files (%d) removed from full manifest", old_ghosted);
-	}
-
 	apply_heuristics(new_full);
 
 	print_elapsed_time("full manifest creation", &previous_time, &current_time);
@@ -407,24 +385,16 @@ int main(int argc, char **argv)
 		/* Detect renamed files specifically for os-core */
 		rename_detection(new_core);
 #endif
-		if (old_core->format < new_core->format) {
-			old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
-		} else {
-			old_deleted = 0;
-		}
-
-		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
+		old_deleted = remove_old_deleted_files(old_core, new_core);
 		sort_manifest_by_version(new_core); /* sorts by filename */
-		/* clean up orphaned renames by marking them as deleted */
-		clean_renames(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
 			LOG(NULL, "", "Core component has not changed (after pruning), exiting");
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", newfiles, old_deleted, old_ghosted);
-		printf("Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", newfiles, old_deleted, old_ghosted);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
+		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
@@ -536,21 +506,13 @@ int main(int argc, char **argv)
 			rename_detection(newm);
 #endif
 			/* Step 6b: otherwise, write out the manifest */
-			if (oldm->format < newm->format) {
-				old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
-			} else {
-				old_deleted = 0;
-			}
-
-			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
+			old_deleted = remove_old_deleted_files(oldm, newm);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
-			/* clean up orphaned renames by marking them as deleted */
-			clean_renames(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0 || old_deleted > 0 || old_ghosted > 0 || changed_includes(oldm, newm)) {
-				LOG(NULL, "", "%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", group, newfiles, old_deleted, old_ghosted);
-				printf("%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", group, newfiles, old_deleted, old_ghosted);
+			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
+				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);
@@ -587,8 +549,6 @@ int main(int argc, char **argv)
 	maximize_to_full(new_MoM, new_full);
 
 	sort_manifest_by_version(new_full);
-	/* clean up orphaned renames by marking them as deleted */
-	clean_renames(new_full);
 	prune_manifest(new_full);
 	if (write_manifest(new_full) != 0) {
 		goto exit;

src/fullfiles.c

@@ -50,6 +50,7 @@ static void create_fullfile(struct file *file)
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
 	char *param1, *param2;
+	int stderrfd;
 
 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -97,13 +98,17 @@ static void create_fullfile(struct file *file)
 		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
 		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
 
-		int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
-		if (tarcmdresult != 0) {
-			LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
+		stderrfd = open("/dev/null", O_WRONLY);
+		if (stderrfd == -1) {
+			LOG(NULL, "Failed to open /dev/null", "");
+			assert(0);
+		}
+		if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
 			assert(0);
 		}
 		free(param1);
 		free(param2);
+		close(stderrfd);
 
 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
@@ -298,11 +303,6 @@ static void submit_fullfile_tasks(GList *files)
 		file = item->data;
 		item = g_list_next(item);
 
-		/* do not push ghosted files */
-		if (file->is_ghosted) {
-			continue;
-		}
-
 		ret = g_thread_pool_push(threadpool, file, &err);
 		if (ret == FALSE) {
 			printf("GThread create_fullfile_task push error\n");

src/helpers.c

@@ -149,117 +149,130 @@ void concat_str_array(char **output, char *const argv[])
 	}
 }
 
-int system_argv_pipe(char *const lhscmd[], char *const rhscmd[])
-{
-	return system_argv_pipe_fd(-1, -1, lhscmd, -1, -1, rhscmd);
-}
-
-int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
-			int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
-{
-	pid_t monitorpid = fork();
-	if (monitorpid == -1) {
-		LOG(NULL, "Failed to create child process to monitor pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
-		return -1;
-	} else if (monitorpid == 0) {
-		pipe_monitor(lnewstdinfd, lnewstderrfd, lhscmd, rnewstdoutfd, rnewstderrfd, rhscmd);
-	}
-	return wait_process_terminate(monitorpid);
-}
-
-void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
-		  int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
-{
-	int pipefd[2];
-	if (pipe(pipefd) == -1) {
-		LOG(NULL, "Failed to create a pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
-		assert(0);
-	}
-
-	pid_t lhspid = system_argv_fd_nowait(lnewstdinfd, pipefd[1], lnewstderrfd, pipefd[0], lhscmd);
-	pid_t rhspid = system_argv_fd_nowait(pipefd[0], rnewstdoutfd, rnewstderrfd, pipefd[1], rhscmd);
-
-	if (close(pipefd[1]) == -1) {
-		LOG(NULL, "Could not close write end of pipe file descriptor", "%d", pipefd[1]);
-		assert(0);
-	}
-	if (close(pipefd[0]) == -1) {
-		LOG(NULL, "Could not close read end of pipe file descriptor", "%d", pipefd[0]);
-		assert(0);
-	}
-
-	int lhsresult = wait_process_terminate(lhspid);
-	int rhsresult = wait_process_terminate(rhspid);
-	exit(rhsresult != EXIT_SUCCESS ? rhsresult : lhsresult);
-}
-
 int system_argv(char *const argv[])
 {
-	return system_argv_fd(-1, -1, -1, argv);
-}
+	int child_exit_status;
+	pid_t pid;
+	int status = -1;
 
-int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[])
-{
-	pid_t cmdpid = system_argv_fd_nowait(newstdinfd, newstdoutfd, newstderrfd, -1, cmd);
-	return wait_process_terminate(cmdpid);
-}
+	pid = fork();
 
-pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
-{
-	pid_t cmdpid = fork();
-	if (cmdpid == -1) {
-		LOG(NULL, "Failed to fork to execute command", "%s", cmd[0]);
+	if (pid == 0) { /* child */
+		execvp(*argv, argv);
+		LOG(NULL, "This line must not be reached", "");
 		assert(0);
-	} else if (cmdpid == 0) {
-		exec_cmd_fd(newstdinfd, newstdoutfd, newstderrfd, closefd, cmd);
-	}
-	return cmdpid;
-}
-
-void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
-{
-	move_fd(newstdinfd, STDIN_FILENO);
-	move_fd(newstdoutfd, STDOUT_FILENO);
-	move_fd(newstderrfd, STDERR_FILENO);
-	if (closefd >= 0 && close(closefd) == -1) {
-		LOG(NULL, "Could not close file descriptor", "%d", closefd);
+	} else if (pid < 0) {
+		LOG(NULL, "Failed to fork a child process", "");
 		assert(0);
-	}
-	execvp(*cmd, cmd);
-	LOG(NULL, "Command", "%s failed", cmd[0]);
-	assert(0);
-}
-
-void move_fd(int oldfd, int newfd)
-{
-	if (oldfd < 0 || newfd < 0 || oldfd == newfd) {
-		return;
-	}
-	if (dup2(oldfd, newfd) == -1) {
-		LOG(NULL, "Could not create duplicate file descriptor", "%d from %d", newfd, oldfd);
-		assert(0);
-	}
-	if (close(oldfd) == -1) {
-		LOG(NULL, "Could not close file descriptor", "%d", oldfd);
-		assert(0);
-	}
-}
-
-int wait_process_terminate(pid_t pid)
-{
-	int status;
-	do {
-		if (waitpid(pid, &status, 0) == -1) {
-			LOG(NULL, "Failed to wait for PID", "%d", pid);
-			return -1;
-		}
-	} while (!WIFEXITED(status) && !WIFSIGNALED(status));
-	// Exit statuses fall in the range of [0, 255].  Make signal statuses fall in a non-overlapping range starting with 256.
-	if (WIFEXITED(status)) {
-		return WEXITSTATUS(status);
 	} else {
-		return 256 + WTERMSIG(status);
+		pid_t ws = waitpid(pid, &child_exit_status, 0);
+
+		if (ws == -1) {
+			LOG(NULL, "Failed to wait for child process", "");
+			assert(0);
+		}
+
+		if (WIFEXITED(child_exit_status)) {
+			status = WEXITSTATUS(child_exit_status);
+		} else {
+			LOG(NULL, "Child process didn't exit", "");
+			assert(0);
+		}
+
+		if (status != 0) {
+			char *cmdline = NULL;
+
+			concat_str_array(&cmdline, argv);
+			LOG(NULL, "Failed to run command:", "%s", cmdline);
+			free(cmdline);
+		}
 	}
+
+	return status;
+}
+
+int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr)
+{
+	int child_exit_status;
+	pid_t pid;
+	int status = -1;
+
+	pid = fork();
+
+	if (pid == 0) { /* child */
+		if (newstdin >= 0) {
+			if (dup2(newstdin, STDIN_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stdin", "");
+				assert(0);
+			}
+			close(newstdin);
+		}
+		if (newstdout >= 0) {
+			if (dup2(newstdout, STDOUT_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stdout", "");
+				assert(0);
+			}
+			close(newstdout);
+		}
+		if (newstderr >= 0) {
+			if (dup2(newstderr, STDERR_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stderr", "");
+				assert(0);
+			}
+			close(newstderr);
+		}
+
+		execvp(*argv, argv);
+		LOG(NULL, "This line must not be reached", "");
+		assert(0);
+	} else if (pid < 0) {
+		LOG(NULL, "Failed to fork a child process", "");
+		assert(0);
+	} else {
+		pid_t ws = waitpid(pid, &child_exit_status, 0);
+
+		if (ws == -1) {
+			LOG(NULL, "Failed to wait for child process", "");
+			assert(0);
+		}
+
+		if (WIFEXITED(child_exit_status)) {
+			status = WEXITSTATUS(child_exit_status);
+		} else {
+			LOG(NULL, "Child process didn't exit", "");
+			assert(0);
+		}
+
+		if (status != 0) {
+			char *cmdline = NULL;
+
+			concat_str_array(&cmdline, argv);
+			LOG(NULL, "Failed to run command:", "%s", cmdline);
+			free(cmdline);
+		}
+	}
+
+	return status;
+}
+
+int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+		     char *const argvp2[], int stdoutp2, int stderrp2)
+{
+	int statusp2;
+	int pipefd[2];
+
+	if (pipe(pipefd)) {
+		LOG(NULL, "Failed to create a pipe", "");
+		return -1;
+	}
+	system_argv_fd(argvp1, stdinp1, pipefd[1], stderrp1);
+	close(pipefd[1]);
+	statusp2 = system_argv_fd(argvp2, pipefd[0], stdoutp2, stderrp2);
+	close(pipefd[0]);
+
+	/* Returns the status of the failed process if any
+       If both processes failed returns the status of first one */
+	return statusp2;
 }
 
 void check_root(void)
@@ -296,31 +309,3 @@ int num_threads(float scaling)
 
 	return result;
 }
-
-/* This function is called when configuration specifies a ban on debuginfo files
- * from manifests. Returns true if the passed file path matches the src or lib
- * debuginfo configuration. */
-bool file_is_debuginfo(const char *path)
-{
-	bool ret = false;
-	char *lib;
-	char *src;
-
-	lib = config_debuginfo_path("lib");
-	src = config_debuginfo_path("src");
-
-	if (lib && (strncmp(path, lib, strlen(lib)) == 0)) {
-		ret = true;
-		goto out;
-	}
-
-	if (src && (strncmp(path, src, strlen(src)) == 0)) {
-		ret = true;
-		goto out;
-	}
-
-out:
-	free(lib);
-	free(src);
-	return ret;
-}

src/manifest.c

@@ -240,22 +240,10 @@ struct manifest *manifest_from_file(int version, char *component)
 			assert(0); /* unknown file type */
 		}
 
-		switch (c[1]) {
-		case 'd':
-			/* file is deleted */
+		if (c[1] == 'd') {
 			file->is_deleted = 1;
-			break;
-		case 'g':
-			/* file is ghosted */
-			file->is_ghosted = 1;
-			break;
-		case '.':
-			/* no flag at this index */
-			break;
-		default:
-			/* unknown deleted status */
-			LOG(NULL, "Invalid flag at index 1", "%s", c[1]);
-			assert(0);
+		} else if (c[1] != '.') {
+			assert(0); /* unknown deleted status */
 		}
 
 		if (c[2] == 'C') {
@@ -440,23 +428,15 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			file3->is_config = file1->is_config;
 			file3->is_state = file1->is_state;
 			file3->is_boot = file1->is_boot;
-			file3->is_ghosted = file1->is_ghosted;
-			/* ghost deleted boot files */
-			if (!file3->is_ghosted) {
-				file3->is_ghosted = file1->is_boot && file1->is_deleted;
-			}
 
-			if (file3->is_ghosted || file1->is_deleted) {
-				/* if the new file is ghosted or the file was deleted, preserve
-				 * hash (all zeros if file1->is_deleted) and rename status */
+			if (!file1->is_deleted) {
+				file3->last_change = m2->version;
+			} else {
+				file3->last_change = file1->last_change;
 				file3->is_rename = file1->is_rename;
 				hash_assign(file1->hash, file3->hash);
 			}
 
-			/* for deleted files last_change remains the same.
-			 * otherwise last change is now */
-			file3->last_change = file1->is_deleted ? file1->last_change : m2->version;
-
 			file3->peer = file1;
 			file1->peer = file3;
 
@@ -678,10 +658,6 @@ char *file_type_to_string(struct file *file)
 		type[1] = 'd';
 	}
 
-	if (file->is_ghosted) {
-		type[1] = 'g';
-	}
-
 	if (file->is_config) {
 		type[2] = 'C';
 	}
@@ -986,12 +962,13 @@ bool changed_includes(struct manifest *old, struct manifest *new)
 	return false;
 }
 
-/* Remove files deprecated in this version as identified by the compfunc
- * function. This function can be used to remove old deleted files over a format
- * bump and remove deprecated ghosted files.
+/* For a format bump, it's convenient to remove deleted files from manifests
+ * that last changed prior to the format bump, since they are no longer
+ * considered for deletion as part of an update.
+ *
  * Note: this function should be called after match_manifests().
  */
-int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *file1, struct file *file2))
+int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
@@ -1007,6 +984,11 @@ int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*com
 		return -1;
 	}
 
+	/* This is the common case, so bail early, reporting no deletions */
+	if (m1->format == m2->format) {
+		return 0;
+	}
+
 	/* At this point, the manifest formats mismatch, and it's assumed that
 	 * m1 is the old manifest, and m2 is the new.
 	 */
@@ -1024,9 +1006,7 @@ int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*com
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			/* use the comparison function passed in to determine if this file
-			 * should be removed */
-			if (compfunc(file1, file2)) {
+			if (file1->is_deleted && file2->is_deleted) {
 				GList *to_delete = list2;
 				list1 = g_list_next(list1);
 				list2 = g_list_next(list2);
@@ -1047,32 +1027,6 @@ int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*com
 	return count;
 }
 
-/* Performs a final link of renames in the manifest to catch any renames from
- * old versions and then removes any orphaned renames */
-void clean_renames(struct manifest *manifest)
-{
-	GList *list;
-	struct file *file;
-
-	/* make sure all renames are linked, this is necessary for renames
-	 * from old manifests that carry over to the current one */
-	final_link(manifest->files);
-
-	list = g_list_first(manifest->files);
-	while (list) {
-		file = list->data;
-		/* if the file is marked as deleted and renamed it is a
-		 * renamed-from file. If the rename_peer field is still NULL
-		 * then it has been orphaned. Mark these as deleted */
-		if (file->is_deleted && file->is_rename && !file->rename_peer) {
-			hash_set_zeros(file->hash);
-			file->is_rename = 0;
-		}
-
-		list = g_list_next(list);
-	}
-}
-
 /* Conditionally remove some things from a manifest.
  * Returns > 0 when the pruned manifest has new files.
  * Returns 0 when the pruned manifest no longer has new files.
@@ -1097,16 +1051,8 @@ int prune_manifest(struct manifest *manifest)
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		} else if (file->is_boot && file->is_deleted) {
-			/* mark boot files that are going away as ghosted, these will be
-			 * cleaned up with the next update */
-			file->is_deleted = 0;
-			file->is_ghosted = 1;
-		} else if (config_ban_debuginfo() && file_is_debuginfo(file->filename)) {
-			/* The configuration option to ban debuginfo from the manifests was
-			 * set in server.ini via the [Debuginfo][banned] option. Although
-			 * debuginfo additions are banned via analyze_fs, prune it here
-			 * to insure mistakenly included debuginfo from old versions is
-			 * removed from the manifests. */
+			// only expose the current best boot files, a client side entity can manage /boot's actual contents
+			// LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component);
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		}

src/pack.c

@@ -58,11 +58,11 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m
 		// (re)create module/version/{delta,staged}
 		string_or_die(&path, "%s/%s/%i_to_%i/delta", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU);
+		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 		free(path);
 		string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU);
+		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 		free(path);
 	}
 }
@@ -79,7 +79,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 
 	string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 		      from_version, to_version);
-	g_mkdir_with_parents(path, S_IRWXU);
+	g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 	dir = opendir(path);
 	if (!dir) {
 		fprintf(stderr, "There are problems accessing %s, exiting\n", path);

src/rename.c

@@ -184,7 +184,7 @@ double rename_score(struct file *old, struct file *new)
 	return score;
 }
 
-static void precompute_file_data(int version, const char *component, struct file *file)
+static void precompute_file_data(int version, const char *component, struct file *file, bool fast)
 {
 	char *c1, *c2;
 	char *filename = NULL;
@@ -301,7 +301,7 @@ static GList *new_filtered_list(GList *list, int version, int (*f)(struct file *
 	return newlist;
 }
 
-static int renamed_file_p(struct file *file, int unused __attribute__((unused)))
+static int renamed_file_p(struct file *file, int unused)
 {
 	return file->is_rename;
 }
@@ -331,7 +331,7 @@ static GList *list_new_files(struct manifest *manifest)
 	GList *ret = list;
 	for (list = g_list_first(list); list; list = g_list_next(list)) {
 		struct file *file = list->data;
-		precompute_file_data(manifest->version, manifest->component, file);
+		precompute_file_data(manifest->version, manifest->component, file, true);
 	}
 	return ret;
 }
@@ -357,7 +357,7 @@ static GList *list_deleted_files(struct manifest *manifest)
 		struct file *peer = file->peer;
 		/* Need to get things from the /full/ as we do not know
 		 * which  component may be coming from? */
-		precompute_file_data(peer->last_change, "full", peer);
+		precompute_file_data(peer->last_change, "full", peer, false);
 	}
 	return ret;
 }
@@ -446,48 +446,6 @@ redo:
 	g_list_free(deleted_files);
 }
 
-void final_link(GList *files)
-{
-	GList *list1, *list2;
-	struct file *file1, *file2;
-
-	list1 = new_list_renamed_files(files);
-	list1 = list2 = g_list_sort(list1, file_sort_version);
-
-	for (; list1; list1 = g_list_next(list1)) {
-		file1 = list1->data;
-		/* do not re-link already-linked renames */
-		if (file1->rename_peer != NULL) {
-			continue;
-		}
-
-		/* check the rest of the list */
-		list2 = g_list_next(list1);
-		for (; list2; list2 = g_list_next(list2)) {
-			file2 = list2->data;
-			/* must be the same version to link */
-			if (file1->last_change != file2->last_change) {
-				continue;
-			}
-
-			/* need one and only one to be deleted to link as this
-			 * indicates a renamed-from/renamed-to relationship
-			 * (.d.r -> F..r) */
-			if (file1->is_deleted == file2->is_deleted) {
-				continue;
-			}
-
-			/* must have same hash to link */
-			if (!hash_compare(file2->hash, file1->hash)) {
-				continue;
-			}
-
-			file1->rename_peer = file2;
-			file2->rename_peer = file1;
-		}
-	}
-}
-
 /* What do we need this for?
  *
  * rename_detection has already set up the links in the manifest it

src/versions.c

@@ -235,7 +235,8 @@ GList *get_last_versions_list(int next_version, int max_versions)
 	DIR *dir;
 	GList *list = NULL;
 	GList *cur_item, *next_item;
-	struct dirent *entry;
+	struct dirent entry;
+	struct dirent *result;
 	struct stat stat;
 	char *filename = NULL;
 	int idx, build_num, build_type, jump_point;
@@ -248,13 +249,13 @@ GList *get_last_versions_list(int next_version, int max_versions)
 		return NULL;
 	}
 
-	while ((entry = readdir(dir))) {
-		if (strspn(entry->d_name, "0123456789") != strlen(entry->d_name)) {
+	while (readdir_r(dir, &entry, &result) == 0 && result != NULL) {
+		if (strspn(entry.d_name, "0123456789") != strlen(entry.d_name)) {
 			continue;
 		}
 
 		free(filename);
-		string_or_die(&filename, "%s/%s", staging_dir, entry->d_name);
+		string_or_die(&filename, "%s/%s", staging_dir, entry.d_name);
 
 		if (lstat(filename, &stat)) {
 			LOG(NULL, "lstat failed", "path= %s, strerror= %s",
@@ -266,11 +267,11 @@ GList *get_last_versions_list(int next_version, int max_versions)
 			continue;
 		}
 
-		if (atoi(entry->d_name) >= next_version) {
+		if (atoi(entry.d_name) >= next_version) {
 			continue;
 		}
 
-		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry->d_name)));
+		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry.d_name)));
 	}
 	free(filename);
 	closedir(dir);

test/functional/file-name-debuginfo/test.bats

@@ -1,56 +0,0 @@
-#!/usr/bin/env bats
-
-# common functions
-load "../swupdlib"
-
-setup() {
-  clean_test_dir
-  init_test_dir
-
-  init_server_ini
-  # unban debuginfo
-  sed -i "s|banned=true|banned=false|" $DIR/server.ini
-  set_latest_ver 0
-  init_groups_ini os-core
-  init_groups_ini test-bundle
-
-  set_os_release 10 os-core
-  track_bundle 10 os-core
-  set_os_release 10 test-bundle
-  track_bundle 10 test-bundle
-
-  gen_file_plain 10 test-bundle "/usr/lib/debug/foo"
-  gen_file_plain 10 test-bundle "/usr/src/debug/bar"
-  gen_file_plain 10 test-bundle "/usr/bin/foobar"
-
-  track_bundle 20 os-core
-  set_os_release 20 test-bundle
-  track_bundle 20 test-bundle
-
-  gen_file_plain 20 test-bundle "/usr/foo"
-}
-
-@test "debuginfo files pruned" {
-  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
-  echo "$output"
-  # This should not be pruned
-  [[ 1 -eq $(grep '/usr/bin/foobar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-  [[ 1 -eq $(grep '/usr/lib/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-  [[ 1 -eq $(grep '/usr/src/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-  # These should not be pruned right now, but should be pruned with the next
-  # update after being banned in server.ini again
-  [[ 1 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-  [[ 1 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-
-  # grab the server.ini that bans debuginfo again
-  init_server_ini
-  set_os_release 20 os-core
-  run sudo sh -c "$CREATE_UPDATE --osversion 20 --statedir $DIR --format 3"
-  # This should not be pruned
-  [[ 1 -eq $(grep '/usr/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
-  # These should be pruned
-  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
-  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
-}
-
-# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run/test.bats

@@ -40,12 +40,6 @@ setup() {
   [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
   [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
-
-  # extract test-bundle pack to make sure staged and delta are created with the
-  # correct permissions
-  sudo tar -xf $DIR/www/10/pack-test-bundle-from-0.tar --directory $DIR/www/10/
-  [[ $(stat -c %a $DIR/www/10/staged) -eq 700 ]]
-  [[ $(stat -c %a $DIR/www/10/delta) -eq 700 ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/ghosting/test.bats

@@ -1,60 +0,0 @@
-#!/usr/bin/env bats
-
-# common functions
-load "../swupdlib"
-
-setup() {
-  clean_test_dir
-  init_test_dir
-
-  init_server_ini
-  set_latest_ver 0
-  init_groups_ini os-core test-bundle
-
-  set_os_release 10 os-core
-  set_os_release 10 test-bundle
-  track_bundle 10 os-core
-  track_bundle 10 test-bundle
-
-  set_os_release 20 os-core
-  set_os_release 20 test-bundle
-  track_bundle 20 os-core
-  track_bundle 20 test-bundle
-
-  set_os_release 30 os-core
-  set_os_release 30 test-bundle
-  track_bundle 30 os-core
-  track_bundle 30 test-bundle
-
-  gen_file_plain_with_content 10 test-bundle usr/lib/kernel/bar testfile_contents
-  gen_file_plain_with_content 20 test-bundle usr/lib/kernel/baz new_testfile_contents
-  # make sure /usr/lib/kernel stays around
-  mkdir $DIR/image/30/test-bundle/usr/lib/kernel
-}
-
-@test "create updates while ghosting boot files before removing them" {
-  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 10
-
-  set_latest_ver 10
-
-  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 20
-
-  set_latest_ver 20
-
-  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 30
-  # version 10: add boot file to 10
-  [ 1 -eq $(grep 'F\.b\.	.*	10	/usr/lib/kernel/bar' $DIR/www/10/Manifest.full | wc -l) ]
-  # version 20: ghost boot file instead of deleting it when it doesn't exist
-  [ 1 -eq $(grep '\.gb\.	.*	20	/usr/lib/kernel/bar' $DIR/www/20/Manifest.full | wc -l) ]
-  # version 20: add a new boot file
-  [ 1 -eq $(grep 'F\.b\.	.*	20	/usr/lib/kernel/baz' $DIR/www/20/Manifest.full | wc -l) ]
-  # version 30: old ghosted file /usr/lib/kernel/bar cleaned up
-  [ 0 -eq $(grep '/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
-  # version 30: boot file added in version 20 ghosted
-  [ 1 -eq $(grep '\.gb\.	.*	30	/usr/lib/kernel/baz' $DIR/www/30/Manifest.full | wc -l) ]
-}
-
-# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/orphaned-renames/test.bats

@@ -1,95 +0,0 @@
-#!/usr/bin/env bats
-
-# common functions
-load "../swupdlib"
-
-setup() {
-  clean_test_dir
-  init_test_dir
-
-  init_server_ini
-  set_latest_ver 0
-  init_groups_ini os-core test-bundle
-
-  set_os_release 10 os-core
-  set_os_release 10 test-bundle
-  track_bundle 10 os-core
-  track_bundle 10 test-bundle
-
-  set_os_release 20 os-core
-  set_os_release 20 test-bundle
-  track_bundle 20 os-core
-  track_bundle 20 test-bundle
-
-  set_os_release 30 os-core
-  set_os_release 30 test-bundle
-  track_bundle 30 os-core
-  track_bundle 30 test-bundle
-
-  set_os_release 40 os-core
-  set_os_release 40 test-bundle
-  track_bundle 40 os-core
-  track_bundle 40 test-bundle
-
-  # /usr/lib/bar and /one will be renamed to /usr/lib/baz and /two
-  gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
-  gen_file_plain_with_content 10 test-bundle /one "$(printf 'a%.0s' {1..200})"
-
-  gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
-  gen_file_plain_with_content 20 test-bundle /two "$(printf 'a%.0s' {1..200})"
-
-  # different content just to make sure this works with delta renames as well as
-  # direct renames
-  gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
-
-  gen_file_plain_with_content 40 test-bundle /usr/lib/foo "$(seq 100) new"
-  # new file to force manifest generation
-  gen_file_plain_with_content 40 test-bundle /a "testfile"
-}
-
-@test "create updates with renamed-to file getting deleted" {
-  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 10
-
-  set_latest_ver 10
-
-  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 20
-
-  set_latest_ver 20
-
-  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 30
-
-  set_latest_ver 30
-
-  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
-  sudo $MAKE_FULLFILES --statedir $DIR 40
-  # version 10: add files to 10
-  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/one' $DIR/www/10/Manifest.test-bundle | wc -l) ]
-  # version 20: rename bar to baz and one to two
-  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/one' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/two' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
-  # version 30: original renamed-from file (bar) is now orphaned and therefore
-  # deleted, baz is now a renamed-from file and foo is a renamed-to file.
-  # /two was deleted in this version, so both /one and /two should be marked as
-  # deleted
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
-
-  # version 40: the existing rename from baz -> foo must persist while all
-  # others remain deleted
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/40/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/40/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/40/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/40/Manifest.test-bundle | wc -l) ]
-  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/40/Manifest.test-bundle | wc -l) ]
-}
-
-# vi: ft=sh ts=8 sw=2 sts=2 et tw=80