Release v3.6.3

This release improves full-file creation to make it thread-safe,
allowing parallelized runs of many instances of swupd_make_fullfiles.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.travis.yml

@@ -11,9 +11,9 @@ before_install:
         - sudo apt-get install -y libmagic-dev
 
 install:
-        - wget http://downloads.sourceforge.net/project/check/check/0.10.0/check-0.10.0.tar.gz
-        - tar -xvf check-0.10.0.tar.gz
-        - pushd check-0.10.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
+        - tar -xvf check-0.11.0.tar.gz
+        - pushd check-0.11.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
         - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
         - tar -xvf bsdiff-1.0.2.tar.xz
         - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd

Makefile.am

@@ -108,8 +108,11 @@ TESTS = $(dist_check_SCRIPTS)
 
 dist_check_SCRIPTS = \
 	test/functional/basic/test.bats \
+	test/functional/contentsize-across-versions-includes/test.bats \
 	test/functional/delete-no-version-bump/test.bats \
 	test/functional/file-name-blacklisted/test.bats \
+	test/functional/file-name-debuginfo/test.bats \
+	test/functional/format-no-decrement/test.bats \
 	test/functional/full-run-delta/test.bats \
 	test/functional/full-run/test.bats \
 	test/functional/fullfiles/test.bats \
@@ -119,7 +122,8 @@ dist_check_SCRIPTS = \
 	test/functional/pack/test.bats \
 	test/functional/state-file/test.bats \
 	test/functional/subtract-delete/test.bats \
-	test/functional/update/test.bats
+	test/functional/update/test.bats \
+	test/functional/format-bump/test.bats
 
 if RENAMES
 dist_check_SCRIPTS += \

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.4.0, tudor.marcu@intel.com)
+AC_INIT(swupd-server, 3.6.3, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -85,6 +85,8 @@ struct manifest {
 	GList *submanifests; /* as struct manifest */
 
 	GList *includes; /* struct manifests for all bundles included into this one */
+
+	GList *actions; /* post-update actions */
 };
 
 struct file;
@@ -173,7 +175,9 @@ extern void release_configuration_data(void);
 extern char *config_image_base(void);
 extern char *config_output_dir(void);
 extern char *config_empty_dir(void);
+extern char *config_debuginfo_path(const char *path);
 extern int config_initial_version(void);
+extern bool config_ban_debuginfo(void);
 
 extern void read_current_version(char *filename);
 extern void write_new_version(char *filename, int version);
@@ -184,7 +188,7 @@ extern GList *get_last_versions_list(int next_version, int max_versions);
 extern char *file_type_to_string(struct file *file);
 extern struct manifest *manifest_from_file(int version, char *module);
 extern void free_manifest(struct manifest *manifest);
-extern struct manifest *alloc_manifest(int version, char *module);
+extern struct manifest *alloc_manifest(int version, char *module, GList *actions);
 extern int match_manifests(struct manifest *m1, struct manifest *m2);
 extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
@@ -258,10 +262,18 @@ extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
 extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
+extern int system_argv_pipe(char *const lhscmd[], char *const rhscmd[]);
+extern int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			       int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
+extern void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			 int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
 extern int system_argv(char *const argv[]);
-extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
-extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
-			    char *const argvp2[], int stdoutp2, int stderrp2);
+extern int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[]);
+extern pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void move_fd(int oldfd, int newfd);
+extern int wait_process_terminate(pid_t pid);
 extern int num_threads(float scaling);
+extern bool file_is_debuginfo(const char *path);
 
 #endif

packfsck.pl

@@ -1,82 +0,0 @@
-#!/usr/bin/perl -w
-#
-#
-# arguments:
-#     perl packfsck.pl <target version>
-
-
-my $target = $ARGV[0];
-system("rm /tmp/Manifest");
-system("wget --quiet --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core");
-
-
-my $from = $target;
-while ($from > $target - 100) {
- $from = $from - 10;
- print "Testing the $from-$target pack\n";
- system("rm /tmp/pack.tar");
- system("wget --quiet ---output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar");
-
-
-
- open FILE, "</tmp/Manifest";
-
- my %expected_hashes;
- my %files;
-
- while (<FILE>) {
-   my $line = $_;
-  
-  
-   if ($line =~ /(^[FDLr\.]+)\s*([0-9a-f]+)\s*([0-9]+)\s*+(.*)\n/) {
-      my $type = $1;
-      my $hash = $2;
-      my $version = $3;
-      my $filename = $4;
-      if ($version > $from) {
-        $expected_hashes{"$hash"} = 1;
-        $files{"$hash"} = $filename;
-#        print "$1 - $2 - $3 - $4\n";
-      }
-   } 
- }
- close FILE;
-
-
- open FILE2, "-|", "tar -tf /tmp/pack.tar 2> /dev/null";
-
- while (<FILE2>) {
-   my $line = $_;
-  
-
-  if ($line =~ /^\.\/delta\/[0-9]+.[0-9]+.([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected delta hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  } 
-  if ($line =~ /\.\/staged\/([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected staged hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  }
- }
- close FILE2;
-
- my $count = 0;
- foreach my $key ( keys %expected_hashes )
- {
-  my $value = $expected_hashes{$key};
-  if ($value eq 1) {
-     my $fn = $files{$key};
-     print "\tFile $fn ($key) is not in the pack\n";
-     $count = $count = 1;
-  }
- }
- if ($count < 1) {
-  print "\tThe $from-$target pack has no files missing\n";
- }
-}

server.ini

@@ -2,3 +2,8 @@
 emptydir=/var/lib/update/empty/
 imagebase=/var/lib/update/image/
 outputdir=/var/lib/update/www/
+
+[Debuginfo]
+banned=true
+lib=/usr/lib/debug/
+src=/usr/src/debug/

src/analyze_fs.c

@@ -314,6 +314,13 @@ static struct file *add_file(struct manifest *manifest,
 	GError *err = NULL;
 	struct file *file;
 
+	if (config_ban_debuginfo() && file_is_debuginfo(sub_filename)) {
+		printf("WARNING: File %s is banned ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
 	if (illegal_characters(entry_name)) {
 		printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
 		free(sub_filename);
@@ -481,7 +488,7 @@ struct manifest *full_manifest_from_directory(int version)
 
 	LOG(NULL, "Computing hashes", "for %i/full", version);
 
-	manifest = alloc_manifest(version, "full");
+	manifest = alloc_manifest(version, "full", NULL);
 
 	string_or_die(&dir, "%s/%i/full", image_dir, version);
 
@@ -557,7 +564,7 @@ struct manifest *sub_manifest_from_directory(char *component, int version)
 
 	LOG(NULL, "Creating component manifest", "for %i/%s", version, component);
 
-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);
 
 	string_or_die(&dir, "%s/%i/%s", image_dir, version, component);
 

src/config.c

@@ -67,6 +67,20 @@ int config_initial_version(void)
 	return version;
 }
 
+bool config_ban_debuginfo(void)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_boolean(keyfile, "Debuginfo", "banned", NULL);
+}
+
+char *config_debuginfo_path(const char *comp)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_value(keyfile, "Debuginfo", comp, NULL);
+}
+
 bool read_configuration_file(char *filename)
 {
 	GError *error = NULL;

src/create_update.c

@@ -340,7 +340,25 @@ int main(int argc, char **argv)
 
 	/* Step 2: Make a manifest for the os-core set */
 	old_MoM = manifest_from_file(current_version, "MoM");
-	new_MoM = alloc_manifest(newversion, "MoM");
+
+	/* Verify that the new format is not older than the previous format. It
+	 * never makes sense to decrease the format number for the next build. */
+	if (format < old_MoM->format) {
+		LOG(NULL, "", "Current format (%llu) must be greater than or equal to previous format (%llu). Exiting",
+		    format, old_MoM->format);
+		printf("Current format (%llu) must be greater than or equal to previous format (%llu). Exiting\n",
+		       format, old_MoM->format);
+		goto exit;
+	}
+
+	/* Detect a format bump and add the "update" action to the manifest
+	 * "actions:" field */
+	GList *actions = NULL;
+	if (format > old_MoM->format) {
+		actions = g_list_prepend(actions, "update");
+	}
+
+	new_MoM = alloc_manifest(newversion, "MoM", actions);
 	old_core = manifest_from_file(manifest_subversion(old_MoM, "os-core"), "os-core");
 	new_core = sub_manifest_from_directory("os-core", newversion);
 	add_component_hashes_to_manifest(new_core, new_full);

src/fullfiles.c

@@ -50,7 +50,6 @@ static void create_fullfile(struct file *file)
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
 	char *param1, *param2;
-	int stderrfd;
 
 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -98,17 +97,13 @@ static void create_fullfile(struct file *file)
 		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
 		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
 
-		stderrfd = open("/dev/null", O_WRONLY);
-		if (stderrfd == -1) {
-			LOG(NULL, "Failed to open /dev/null", "");
-			assert(0);
-		}
-		if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
+		int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
+		if (tarcmdresult != 0) {
+			LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
 			assert(0);
 		}
 		free(param1);
 		free(param2);
-		close(stderrfd);
 
 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);

src/helpers.c

@@ -149,130 +149,117 @@ void concat_str_array(char **output, char *const argv[])
 	}
 }
 
+int system_argv_pipe(char *const lhscmd[], char *const rhscmd[])
+{
+	return system_argv_pipe_fd(-1, -1, lhscmd, -1, -1, rhscmd);
+}
+
+int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	pid_t monitorpid = fork();
+	if (monitorpid == -1) {
+		LOG(NULL, "Failed to create child process to monitor pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		return -1;
+	} else if (monitorpid == 0) {
+		pipe_monitor(lnewstdinfd, lnewstderrfd, lhscmd, rnewstdoutfd, rnewstderrfd, rhscmd);
+	}
+	return wait_process_terminate(monitorpid);
+}
+
+void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+		  int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	int pipefd[2];
+	if (pipe(pipefd) == -1) {
+		LOG(NULL, "Failed to create a pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		assert(0);
+	}
+
+	pid_t lhspid = system_argv_fd_nowait(lnewstdinfd, pipefd[1], lnewstderrfd, pipefd[0], lhscmd);
+	pid_t rhspid = system_argv_fd_nowait(pipefd[0], rnewstdoutfd, rnewstderrfd, pipefd[1], rhscmd);
+
+	if (close(pipefd[1]) == -1) {
+		LOG(NULL, "Could not close write end of pipe file descriptor", "%d", pipefd[1]);
+		assert(0);
+	}
+	if (close(pipefd[0]) == -1) {
+		LOG(NULL, "Could not close read end of pipe file descriptor", "%d", pipefd[0]);
+		assert(0);
+	}
+
+	int lhsresult = wait_process_terminate(lhspid);
+	int rhsresult = wait_process_terminate(rhspid);
+	exit(rhsresult != EXIT_SUCCESS ? rhsresult : lhsresult);
+}
+
 int system_argv(char *const argv[])
 {
-	int child_exit_status;
-	pid_t pid;
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
+	return system_argv_fd(-1, -1, -1, argv);
 }
 
-int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr)
+int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[])
 {
-	int child_exit_status;
-	pid_t pid;
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		if (newstdin >= 0) {
-			if (dup2(newstdin, STDIN_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdin", "");
-				assert(0);
-			}
-			close(newstdin);
-		}
-		if (newstdout >= 0) {
-			if (dup2(newstdout, STDOUT_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdout", "");
-				assert(0);
-			}
-			close(newstdout);
-		}
-		if (newstderr >= 0) {
-			if (dup2(newstderr, STDERR_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stderr", "");
-				assert(0);
-			}
-			close(newstderr);
-		}
-
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
+	pid_t cmdpid = system_argv_fd_nowait(newstdinfd, newstdoutfd, newstderrfd, -1, cmd);
+	return wait_process_terminate(cmdpid);
 }
 
-int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
-		     char *const argvp2[], int stdoutp2, int stderrp2)
+pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
 {
-	int statusp2;
-	int pipefd[2];
-
-	if (pipe(pipefd)) {
-		LOG(NULL, "Failed to create a pipe", "");
-		return -1;
+	pid_t cmdpid = fork();
+	if (cmdpid == -1) {
+		LOG(NULL, "Failed to fork to execute command", "%s", cmd[0]);
+		assert(0);
+	} else if (cmdpid == 0) {
+		exec_cmd_fd(newstdinfd, newstdoutfd, newstderrfd, closefd, cmd);
 	}
-	system_argv_fd(argvp1, stdinp1, pipefd[1], stderrp1);
-	close(pipefd[1]);
-	statusp2 = system_argv_fd(argvp2, pipefd[0], stdoutp2, stderrp2);
-	close(pipefd[0]);
+	return cmdpid;
+}
 
-	/* Returns the status of the failed process if any
-       If both processes failed returns the status of first one */
-	return statusp2;
+void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
+{
+	move_fd(newstdinfd, STDIN_FILENO);
+	move_fd(newstdoutfd, STDOUT_FILENO);
+	move_fd(newstderrfd, STDERR_FILENO);
+	if (closefd >= 0 && close(closefd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", closefd);
+		assert(0);
+	}
+	execvp(*cmd, cmd);
+	LOG(NULL, "Command", "%s failed", cmd[0]);
+	assert(0);
+}
+
+void move_fd(int oldfd, int newfd)
+{
+	if (oldfd < 0 || newfd < 0 || oldfd == newfd) {
+		return;
+	}
+	if (dup2(oldfd, newfd) == -1) {
+		LOG(NULL, "Could not create duplicate file descriptor", "%d from %d", newfd, oldfd);
+		assert(0);
+	}
+	if (close(oldfd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", oldfd);
+		assert(0);
+	}
+}
+
+int wait_process_terminate(pid_t pid)
+{
+	int status;
+	do {
+		if (waitpid(pid, &status, 0) == -1) {
+			LOG(NULL, "Failed to wait for PID", "%d", pid);
+			return -1;
+		}
+	} while (!WIFEXITED(status) && !WIFSIGNALED(status));
+	// Exit statuses fall in the range of [0, 255].  Make signal statuses fall in a non-overlapping range starting with 256.
+	if (WIFEXITED(status)) {
+		return WEXITSTATUS(status);
+	} else {
+		return 256 + WTERMSIG(status);
+	}
 }
 
 void check_root(void)
@@ -309,3 +296,31 @@ int num_threads(float scaling)
 
 	return result;
 }
+
+/* This function is called when configuration specifies a ban on debuginfo files
+ * from manifests. Returns true if the passed file path matches the src or lib
+ * debuginfo configuration. */
+bool file_is_debuginfo(const char *path)
+{
+	bool ret = false;
+	char *lib;
+	char *src;
+
+	lib = config_debuginfo_path("lib");
+	src = config_debuginfo_path("src");
+
+	if (lib && (strncmp(path, lib, strlen(lib)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+	if (src && (strncmp(path, src, strlen(src)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+out:
+	free(lib);
+	free(src);
+	return ret;
+}

src/manifest.c

@@ -92,7 +92,7 @@ int file_sort_filename(gconstpointer a, gconstpointer b)
 	return 0;
 }
 
-struct manifest *alloc_manifest(int version, char *component)
+struct manifest *alloc_manifest(int version, char *component, GList *actions)
 {
 	struct manifest *manifest;
 
@@ -104,6 +104,7 @@ struct manifest *alloc_manifest(int version, char *component)
 	manifest->version = version;
 	manifest->component = strdup(component);
 	manifest->format = format;
+	manifest->actions = actions;
 
 	return manifest;
 }
@@ -133,7 +134,7 @@ struct manifest *manifest_from_file(int version, char *component)
 	if (infile == NULL) {
 		LOG(NULL, "Cannot read manifest", "%s (%s)\n", filename, strerror(errno));
 		free(filename);
-		return alloc_manifest(version, component);
+		return alloc_manifest(version, component, NULL);
 	}
 
 	/* line 1: MANIFEST\t<version> */
@@ -193,7 +194,7 @@ struct manifest *manifest_from_file(int version, char *component)
 		}
 	}
 
-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);
 	manifest->format = format_number;
 	manifest->prevversion = previous;
 	manifest->includes = includes;
@@ -674,19 +675,29 @@ char *file_type_to_string(struct file *file)
 	return type;
 }
 
+/* Calculate the contentsize for the manifest based on file sizes.
+ *
+ * This should calculate the files uniquely included in this manifest, but none
+ * of its submanifests, which will allow calculation of sizes of all bundles on
+ * a system by adding all manifest->contentsizes of installed bundles.
+ *
+ * However, if two bundles not in the same include chain have overlapping
+ * content, summing the include chain of each bundle in the client will result
+ * in an over-estimation of the total size on the system. The more content is
+ * shared, the higher the over-estimation. In reality this overlap will not be
+ * large, but it is currently impossible to calculate the exact installed size
+ * using just the contentsize.
+ */
 static void compute_content_size(struct manifest *manifest)
 {
-	/* FIXME: this is a temporary implementation based on worst case */
-
 	GList *list;
 	struct file *file;
-	struct manifest *submanifest;
 
 	list = g_list_first(manifest->files);
 	while (list) {
 		file = list->data;
 		list = g_list_next(list);
-		if (!file->is_deleted && (file->last_change == manifest->version)) {
+		if (!file->is_deleted) {
 			if (file->is_file) {
 				manifest->contentsize += file->stat.st_size;
 			} else if (file->is_link) {
@@ -696,17 +707,6 @@ static void compute_content_size(struct manifest *manifest)
 			}
 		}
 	}
-
-	list = g_list_first(manifest->submanifests);
-	while (list) {
-		submanifest = list->data;
-		list = g_list_next(list);
-
-		/* Do not take into account groups not included in download content */
-		if (create_download_content_for_group(submanifest->component)) {
-			manifest->contentsize += submanifest->contentsize;
-		}
-	}
 }
 
 /* Returns 0 == success, -1 == failure */
@@ -714,6 +714,7 @@ static int write_manifest_plain(struct manifest *manifest)
 {
 	GList *includes;
 	GList *list;
+	GList *actions;
 	struct file *file;
 	FILE *out = NULL;
 	char *base = NULL, *dir;
@@ -763,6 +764,14 @@ static int write_manifest_plain(struct manifest *manifest)
 		includes = g_list_next(includes);
 		fprintf(out, "includes:\t%s\n", sub->component);
 	}
+
+	actions = manifest->actions;
+	while (actions) {
+		char *action = actions->data;
+		fprintf(out, "actions:\t%s\n", action);
+		actions = g_list_next(actions);
+	}
+
 	fprintf(out, "\n");
 
 	list = g_list_first(manifest->files);
@@ -1046,6 +1055,14 @@ int prune_manifest(struct manifest *manifest)
 			// LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component);
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
+		} else if (config_ban_debuginfo() && file_is_debuginfo(file->filename)) {
+			/* The configuration option to ban debuginfo from the manifests was
+			 * set in server.ini via the [Debuginfo][banned] option. Although
+			 * debuginfo additions are banned via analyze_fs, prune it here
+			 * to insure mistakenly included debuginfo from old versions is
+			 * removed from the manifests. */
+			manifest->files = g_list_delete_link(manifest->files, list);
+			manifest->count--;
 		}
 		list = next;
 	}

src/pack.c

@@ -58,11 +58,11 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m
 		// (re)create module/version/{delta,staged}
 		string_or_die(&path, "%s/%s/%i_to_%i/delta", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 		string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 	}
 }
@@ -79,7 +79,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 
 	string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 		      from_version, to_version);
-	g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+	g_mkdir_with_parents(path, S_IRWXU);
 	dir = opendir(path);
 	if (!dir) {
 		fprintf(stderr, "There are problems accessing %s, exiting\n", path);

test/functional/contentsize-across-versions-includes/test.bats

@@ -0,0 +1,64 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+  gen_file_plain 10 test-bundle1 foo
+  gen_file_plain 10 test-bundle1 foobar
+  gen_file_plain 10 test-bundle2 foo2
+  gen_includes_file test-bundle2 10 test-bundle1
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+  gen_file_plain 20 test-bundle1 foo
+  gen_file_plain 20 test-bundle1 foobar
+  gen_file_plain 20 test-bundle1 foobarbaz
+  gen_file_plain 20 test-bundle2 foo2
+  gen_file_plain 20 test-bundle2 foo2bar
+  gen_includes_file test-bundle2 20 test-bundle1
+}
+
+@test "correct contentsize" {
+  # create a couple updates to both check that contentsize does not add included
+  # bundles and to verify that files changed in previous updates are counted.
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # contentsize for test-bundle2 should not include test-bundle1's contentsize
+  [[ 1 -eq $(grep '^contentsize:	11$' $DIR/www/10/Manifest.test-bundle1 | wc -l) ]]
+  [[ 1 -eq $(grep '^contentsize:	5$'  $DIR/www/10/Manifest.test-bundle2 | wc -l) ]]
+  # os-core is large because it includes /usr/*
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # 5134 + 11 + 5 = 5150
+  [[ 1 -eq $(grep '^contentsize:	5150$' $DIR/www/10/Manifest.full | wc -l) ]]
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  set_latest_ver 20
+
+  # one new file: foobarbaz (10 bytes)
+  [[ 1 -eq $(grep '^contentsize:	21$' $DIR/www/20/Manifest.test-bundle1 | wc -l) ]]
+  # one new file: foo2bar (8 bytes)
+  [[ 1 -eq $(grep '^contentsize:	13$'  $DIR/www/20/Manifest.test-bundle2 | wc -l) ]]
+  # os-core should not change size
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # contentsize for full should be all files, including ones not changed in this release
+  # two new files: foo2bar (8 bytes) and foobarbaz (10 bytes)
+  # 5150 + 10 + 8  = 5168
+  # 5134 + 21 + 13 = 5168
+  [[ 1 -eq $(grep '^contentsize:	5168$' $DIR/www/20/Manifest.full | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/file-name-debuginfo/test.bats

@@ -0,0 +1,56 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  # unban debuginfo
+  sed -i "s|banned=true|banned=false|" $DIR/server.ini
+  set_latest_ver 0
+  init_groups_ini os-core
+  init_groups_ini test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 test-bundle
+
+  gen_file_plain 10 test-bundle "/usr/lib/debug/foo"
+  gen_file_plain 10 test-bundle "/usr/src/debug/bar"
+  gen_file_plain 10 test-bundle "/usr/bin/foobar"
+
+  track_bundle 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 test-bundle
+
+  gen_file_plain 20 test-bundle "/usr/foo"
+}
+
+@test "debuginfo files pruned" {
+  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
+  echo "$output"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/bin/foobar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/src/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  # These should not be pruned right now, but should be pruned with the next
+  # update after being banned in server.ini again
+  [[ 1 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+
+  # grab the server.ini that bans debuginfo again
+  init_server_ini
+  set_os_release 20 os-core
+  run sudo sh -c "$CREATE_UPDATE --osversion 20 --statedir $DIR --format 3"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  # These should be pruned
+  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/format-bump/test.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+}
+
+@test "full run update creation with delta packs over format bump" {
+  # build the first version
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+
+  set_latest_ver 10
+
+  # then the second version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+
+  set_latest_ver 20
+
+  # then the third version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 4
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  sudo $MAKE_PACK --statedir $DIR 0 30 os-core
+
+  # zero packs should exist (non-zero size) for all versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/30/pack-os-core-from-0.tar ]
+
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/10/Manifest.MoM | wc -l) ]]
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/20/Manifest.MoM | wc -l) ]]
+  [[ 1 -eq $(grep '^actions:	update$' $DIR/www/30/Manifest.MoM | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/format-no-decrement/test.bats

@@ -0,0 +1,35 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  gen_file_plain 10 os-core foo
+  gen_file_plain 10 os-core bar
+  gen_file_plain 20 os-core foo
+  gen_file_plain 20 os-core baz
+}
+
+@test "ensure format numbers cannot be decremented" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # Now, decrement the format number. Should result in an EXIT_FAILURE.
+  run sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 2
+  echo "$output"
+  [ $status -eq 1 ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run/test.bats

@@ -40,6 +40,12 @@ setup() {
   [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
   [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+
+  # extract test-bundle pack to make sure staged and delta are created with the
+  # correct permissions
+  sudo tar -xf $DIR/www/10/pack-test-bundle-from-0.tar --directory $DIR/www/10/
+  [[ $(stat -c %a $DIR/www/10/staged) -eq 700 ]]
+  [[ $(stat -c %a $DIR/www/10/delta) -eq 700 ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80