Release v3.6.3

This release improves full-file creation to make it thread-safe,
allowing parallelized runs of many instances of swupd_make_fullfiles.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.travis.yml

@@ -11,9 +11,9 @@ before_install:
         - sudo apt-get install -y libmagic-dev
 
 install:
-        - wget http://downloads.sourceforge.net/project/check/check/0.10.0/check-0.10.0.tar.gz
+        - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
-        - tar -xvf check-0.10.0.tar.gz
+        - tar -xvf check-0.11.0.tar.gz
-        - pushd check-0.10.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - pushd check-0.11.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
         - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
         - tar -xvf bsdiff-1.0.2.tar.xz
         - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.6.2, matthew.johnson@intel.com)
+AC_INIT(swupd-server, 3.6.3, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -262,10 +262,17 @@ extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
 extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
+extern int system_argv_pipe(char *const lhscmd[], char *const rhscmd[]);
+extern int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			       int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
+extern void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			 int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
 extern int system_argv(char *const argv[]);
-extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
+extern int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[]);
-extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+extern pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
-			    char *const argvp2[], int stdoutp2, int stderrp2);
+extern void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void move_fd(int oldfd, int newfd);
+extern int wait_process_terminate(pid_t pid);
 extern int num_threads(float scaling);
 extern bool file_is_debuginfo(const char *path);
 

src/fullfiles.c

@@ -50,7 +50,6 @@ static void create_fullfile(struct file *file)
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
 	char *param1, *param2;
-	int stderrfd;
 
 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -98,17 +97,13 @@ static void create_fullfile(struct file *file)
 		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
 		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
 
-		stderrfd = open("/dev/null", O_WRONLY);
+		int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
-		if (stderrfd == -1) {
+		if (tarcmdresult != 0) {
-			LOG(NULL, "Failed to open /dev/null", "");
+			LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
-			assert(0);
-		}
-		if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
 			assert(0);
 		}
 		free(param1);
 		free(param2);
-		close(stderrfd);
 
 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);

src/helpers.c

@@ -149,130 +149,117 @@ void concat_str_array(char **output, char *const argv[])
 	}
 }
 
+int system_argv_pipe(char *const lhscmd[], char *const rhscmd[])
+{
+	return system_argv_pipe_fd(-1, -1, lhscmd, -1, -1, rhscmd);
+}
+
+int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	pid_t monitorpid = fork();
+	if (monitorpid == -1) {
+		LOG(NULL, "Failed to create child process to monitor pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		return -1;
+	} else if (monitorpid == 0) {
+		pipe_monitor(lnewstdinfd, lnewstderrfd, lhscmd, rnewstdoutfd, rnewstderrfd, rhscmd);
+	}
+	return wait_process_terminate(monitorpid);
+}
+
+void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+		  int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	int pipefd[2];
+	if (pipe(pipefd) == -1) {
+		LOG(NULL, "Failed to create a pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		assert(0);
+	}
+
+	pid_t lhspid = system_argv_fd_nowait(lnewstdinfd, pipefd[1], lnewstderrfd, pipefd[0], lhscmd);
+	pid_t rhspid = system_argv_fd_nowait(pipefd[0], rnewstdoutfd, rnewstderrfd, pipefd[1], rhscmd);
+
+	if (close(pipefd[1]) == -1) {
+		LOG(NULL, "Could not close write end of pipe file descriptor", "%d", pipefd[1]);
+		assert(0);
+	}
+	if (close(pipefd[0]) == -1) {
+		LOG(NULL, "Could not close read end of pipe file descriptor", "%d", pipefd[0]);
+		assert(0);
+	}
+
+	int lhsresult = wait_process_terminate(lhspid);
+	int rhsresult = wait_process_terminate(rhspid);
+	exit(rhsresult != EXIT_SUCCESS ? rhsresult : lhsresult);
+}
+
 int system_argv(char *const argv[])
 {
-	int child_exit_status;
+	return system_argv_fd(-1, -1, -1, argv);
-	pid_t pid;
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
 }
 
-int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr)
+int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[])
 {
-	int child_exit_status;
+	pid_t cmdpid = system_argv_fd_nowait(newstdinfd, newstdoutfd, newstderrfd, -1, cmd);
-	pid_t pid;
+	return wait_process_terminate(cmdpid);
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		if (newstdin >= 0) {
-			if (dup2(newstdin, STDIN_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdin", "");
-				assert(0);
-			}
-			close(newstdin);
-		}
-		if (newstdout >= 0) {
-			if (dup2(newstdout, STDOUT_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdout", "");
-				assert(0);
-			}
-			close(newstdout);
-		}
-		if (newstderr >= 0) {
-			if (dup2(newstderr, STDERR_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stderr", "");
-				assert(0);
-			}
-			close(newstderr);
-		}
-
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
 }
 
-int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
-		     char *const argvp2[], int stdoutp2, int stderrp2)
 {
-	int statusp2;
+	pid_t cmdpid = fork();
-	int pipefd[2];
+	if (cmdpid == -1) {
-
+		LOG(NULL, "Failed to fork to execute command", "%s", cmd[0]);
-	if (pipe(pipefd)) {
+		assert(0);
-		LOG(NULL, "Failed to create a pipe", "");
+	} else if (cmdpid == 0) {
-		return -1;
+		exec_cmd_fd(newstdinfd, newstdoutfd, newstderrfd, closefd, cmd);
 	}
-	system_argv_fd(argvp1, stdinp1, pipefd[1], stderrp1);
+	return cmdpid;
-	close(pipefd[1]);
+}
-	statusp2 = system_argv_fd(argvp2, pipefd[0], stdoutp2, stderrp2);
-	close(pipefd[0]);
 
-	/* Returns the status of the failed process if any
+void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
-       If both processes failed returns the status of first one */
+{
-	return statusp2;
+	move_fd(newstdinfd, STDIN_FILENO);
+	move_fd(newstdoutfd, STDOUT_FILENO);
+	move_fd(newstderrfd, STDERR_FILENO);
+	if (closefd >= 0 && close(closefd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", closefd);
+		assert(0);
+	}
+	execvp(*cmd, cmd);
+	LOG(NULL, "Command", "%s failed", cmd[0]);
+	assert(0);
+}
+
+void move_fd(int oldfd, int newfd)
+{
+	if (oldfd < 0 || newfd < 0 || oldfd == newfd) {
+		return;
+	}
+	if (dup2(oldfd, newfd) == -1) {
+		LOG(NULL, "Could not create duplicate file descriptor", "%d from %d", newfd, oldfd);
+		assert(0);
+	}
+	if (close(oldfd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", oldfd);
+		assert(0);
+	}
+}
+
+int wait_process_terminate(pid_t pid)
+{
+	int status;
+	do {
+		if (waitpid(pid, &status, 0) == -1) {
+			LOG(NULL, "Failed to wait for PID", "%d", pid);
+			return -1;
+		}
+	} while (!WIFEXITED(status) && !WIFSIGNALED(status));
+	// Exit statuses fall in the range of [0, 255].  Make signal statuses fall in a non-overlapping range starting with 256.
+	if (WIFEXITED(status)) {
+		return WEXITSTATUS(status);
+	} else {
+		return 256 + WTERMSIG(status);
+	}
 }
 
 void check_root(void)

test/functional/file-name-debuginfo/test.bats

@@ -8,6 +8,8 @@ setup() {
   init_test_dir
 
   init_server_ini
+  # unban debuginfo
+  sed -i "s|banned=true|banned=false|" $DIR/server.ini
   set_latest_ver 0
   init_groups_ini os-core
   init_groups_ini test-bundle
@@ -20,6 +22,12 @@ setup() {
   gen_file_plain 10 test-bundle "/usr/lib/debug/foo"
   gen_file_plain 10 test-bundle "/usr/src/debug/bar"
   gen_file_plain 10 test-bundle "/usr/bin/foobar"
+
+  track_bundle 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 test-bundle
+
+  gen_file_plain 20 test-bundle "/usr/foo"
 }
 
 @test "debuginfo files pruned" {
@@ -29,10 +37,20 @@ setup() {
   [[ 1 -eq $(grep '/usr/bin/foobar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
   [[ 1 -eq $(grep '/usr/lib/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
   [[ 1 -eq $(grep '/usr/src/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
-  # These should be pruned
+  # These should not be pruned right now, but should be pruned with the next
-  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  # update after being banned in server.ini again
-  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
 
+  # grab the server.ini that bans debuginfo again
+  init_server_ini
+  set_os_release 20 os-core
+  run sudo sh -c "$CREATE_UPDATE --osversion 20 --statedir $DIR --format 3"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  # These should be pruned
+  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80