Release v3.7.1

This release fixes the bug introduced in v3.7.0 that caused valid renames from old versions to be dropped. Instead of pruning orphaned renames, they are now marked as deleted as well. Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>
Add test cases for orphaned renames
2026-10-04 07:58:32 +00:00 · 2017-11-28 09:19:14 -08:00 · 2017-11-27 17:22:50 -08:00 · 2017-11-27 17:22:50 -08:00 · 2017-11-21 14:10:00 -08:00 · 2017-11-21 14:06:14 -08:00
47 changed files with 2117 additions and 803 deletions
@@ -0,0 +1,25 @@
+sudo: required
+dist: trusty
+language: c
+
+# Pre-install missing build dependencies:
+# - libmagic (pull from repo)
+# - libcheck 0.9.10 is slightly too old, since 0.9.12 adds TAP support
+# - bsdiff 1.* is the Clear Linux OS fork
+before_install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y libmagic-dev
+
+install:
+        - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
+        - tar -xvf check-0.11.0.tar.gz
+        - pushd check-0.11.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
+        - tar -xvf bsdiff-1.0.2.tar.xz
+        - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd
+
+# Ubuntu's default umask is 0002, but this break's swupd hash calculations.
+script:
+        - sudo find test/functional -exec chmod g-w {} \;
+        - autoreconf --verbose --warnings=none --install --force && ./configure --enable-rename-detection && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
+after_failure: cat test-suite.log
@@ -26,7 +26,6 @@ swupd_create_update_SOURCES = \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/type_change.c \
 	src/versions.c \
@@ -44,7 +43,6 @@ swupd_make_pack_SOURCES = \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c

@@ -61,7 +59,6 @@ swupd_make_fullfiles_SOURCES = \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c

@@ -111,14 +108,29 @@ TESTS = $(dist_check_SCRIPTS)

 dist_check_SCRIPTS = \
 	test/functional/basic/test.bats \
+	test/functional/contentsize-across-versions-includes/test.bats \
 	test/functional/delete-no-version-bump/test.bats \
-	test/functional/include-version-bump/test.bats \
-	test/functional/update/test.bats \
-	test/functional/fullfiles/test.bats \
-	test/functional/pack/test.bats \
-	test/functional/full-run/test.bats \
+	test/functional/file-name-blacklisted/test.bats \
+	test/functional/file-name-debuginfo/test.bats \
+	test/functional/format-no-decrement/test.bats \
 	test/functional/full-run-delta/test.bats \
-	test/functional/file-name-blacklisted/test.bats
+	test/functional/full-run/test.bats \
+	test/functional/fullfiles/test.bats \
+	test/functional/ghosting/test.bats \
+	test/functional/include-version-bump/test.bats \
+	test/functional/includes-deduplicate/test.bats \
+	test/functional/no-delta/test.bats \
+	test/functional/pack/test.bats \
+	test/functional/state-file/test.bats \
+	test/functional/subtract-delete/test.bats \
+	test/functional/update/test.bats \
+	test/functional/format-bump/test.bats
+
+if RENAMES
+dist_check_SCRIPTS += \
+	test/functional/orphaned-renames/test.bats \
+	test/functional/renames/test.bats
+endif
 endif

 if COVERAGE
@@ -136,6 +148,27 @@ endif
 distclean-local:
 	rm -rf aclocal.m4 autogen.sh autom4te.cache config.h.in config.h.in~ configure depcomp install-sh Makefile.in missing compile

+compliant:
+	@git diff --quiet --exit-code include src; ret=$$?; \
+	if [ $$ret -eq 1 ]; then \
+		echo "Error: can only check code style when include/ and src/ are clean."; \
+		echo "Stash or commit your changes and try again."; \
+		exit $$ret; \
+	elif [ $$ret -gt 1 ]; then \
+		exit $$ret; \
+	fi; \
+	clang-format -i -style=file include/*.h src/*.c; ret=$$?; \
+	if [ $$ret -ne 0 ]; then \
+		exit $$ret; \
+	fi; \
+	git diff --quiet --exit-code include src; ret=$$?; \
+	if [ $$ret -eq 1 ]; then \
+		echo "Code style issues found. Run 'git diff' to view issues."; \
+	elif [ $$ret -eq 0 ]; then \
+		echo "No code style issues found."; \
+	fi; \
+	exit $$ret
+
 release:
 	@git rev-parse v$(PACKAGE_VERSION) &> /dev/null; \
 	if [ "$$?" -eq 0 ]; then \
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.

 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.2.9, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.7.1, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC
@@ -82,6 +82,11 @@ AS_IF([test "$enable_lzma" != "no"], [
 ])
 AM_CONDITIONAL([ENABLE_LZMA], [test "$enable_lzma" != "no"])

+AC_ARG_ENABLE([rename-detection], [AS_HELP_STRING([--enable-rename-detection], [enable rename detection feature])])
+
+AS_IF([test "$enable_rename_detection" = "yes"], [AC_DEFINE(RENAMES,1,[Use rename detection])])
+AM_CONDITIONAL([RENAMES], [test "$enable_rename_detection" = "yes"])
+
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT
@@ -20,12 +20,12 @@
 #define TAR_COMMAND "bsdtar"
 #define TAR_XATTR_ARGS ""
 #define TAR_XATTR_ARGS_STRLIST
-#define TAR_WARN_ARGS ""
+#define TAR_WARN_ARGS_STRLIST
 #else
 #define TAR_COMMAND "tar"
 #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'"
 #define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'",
-#define TAR_WARN_ARGS "--warning=no-timestamp"
+#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp",
 #endif

 #if SWUPD_WITH_SELINUX
@@ -65,6 +65,13 @@
 #include <lzma.h>
 #endif

+/* Approximatly the smallest size of a pair of input files which
+ * differ by a single bit that bsdiff can produce a more compact
+ * deltafile. Files smaller than this are always marked as different.
+ * See the magic 200 value in the bsdiff/src/diff.c code.
+ */
+#define BSDIFFSIZE 200
+
 struct manifest {
 	unsigned long long int format;
 	int version;
@@ -78,6 +85,8 @@ struct manifest {
 	GList *submanifests; /* as struct manifest */

 	GList *includes; /* struct manifests for all bundles included into this one */
+
+	GList *actions; /* post-update actions */
 };

 struct file;
@@ -108,6 +117,10 @@ struct file {
 	unsigned int is_file : 1;
 	unsigned int is_link : 1;
 	unsigned int is_deleted : 1;
+	/* a ghosted file is treated as deleted so that it can be used as a rename,
+	 * but is not actually deleted in the client due to a 3rd-party program
+	 * cleaning it up. This happens with boot files managed by a boot manager */
+	unsigned int is_ghosted : 1;
 	unsigned int is_manifest : 1;

 	/* and these are modifiers */
@@ -122,7 +135,7 @@ struct file {
 	double rename_score;
 	struct file *rename_peer;
 	char *alpha_only_filename; /* filename minus all numerics/etc */
-	char *filetype;
+	char *filetype;		   /* The output of 'file', truncated */
 	char *basename;
 	char *dirname;
 	/* end of rename detection fields */
@@ -143,7 +156,6 @@ extern int current_version;
 extern int newversion;
 extern int minversion;
 extern unsigned long long int format;
-extern bool enable_signing;

 extern char *state_dir;
 extern char *packstage_dir;
@@ -167,7 +179,9 @@ extern void release_configuration_data(void);
 extern char *config_image_base(void);
 extern char *config_output_dir(void);
 extern char *config_empty_dir(void);
+extern char *config_debuginfo_path(const char *path);
 extern int config_initial_version(void);
+extern bool config_ban_debuginfo(void);

 extern void read_current_version(char *filename);
 extern void write_new_version(char *filename, int version);
@@ -178,13 +192,14 @@ extern GList *get_last_versions_list(int next_version, int max_versions);
 extern char *file_type_to_string(struct file *file);
 extern struct manifest *manifest_from_file(int version, char *module);
 extern void free_manifest(struct manifest *manifest);
-extern struct manifest *alloc_manifest(int version, char *module);
+extern struct manifest *alloc_manifest(int version, char *module, GList *actions);
 extern int match_manifests(struct manifest *m1, struct manifest *m2);
 extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
 extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
-extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
+extern void clean_renames(struct manifest *manifest);
+extern int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *, struct file *));
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
 extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);
@@ -219,8 +234,6 @@ extern void chroot_create_full(int newversion);

 extern void read_group_file(char *filename);
 extern void release_group_file(void);
-extern char *group_groups(char *group);
-extern char *group_packages(char *group);
 extern char *group_status(char *group);
 extern char *next_group(void);

@@ -243,8 +256,9 @@ extern int previous_version_manifest(struct manifest *mom, char *name);

 extern void type_change_detection(struct manifest *manifest);

-extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
-extern void link_renames(GList *newfiles, struct manifest *from_manifest);
+extern void rename_detection(struct manifest *manifest);
+extern void link_renames(GList *newfiles, int to_version);
+extern void final_link(GList *files);
 extern void __create_delta(struct file *file, int from_version, char *from_hash);

 extern void account_delta_hit(void);
@@ -254,13 +268,18 @@ extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
 extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
+extern int system_argv_pipe(char *const lhscmd[], char *const rhscmd[]);
+extern int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			       int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
+extern void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			 int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
 extern int system_argv(char *const argv[]);
-extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
-extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
-			    char *const argvp2[], int stdoutp2, int stderrp2);
-
-extern bool signature_initialize(void);
-extern void signature_terminate(void);
-extern bool signature_sign(const char *filename);
+extern int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[]);
+extern pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void move_fd(int oldfd, int newfd);
+extern int wait_process_terminate(pid_t pid);
+extern int num_threads(float scaling);
+extern bool file_is_debuginfo(const char *path);

 #endif
@@ -1,82 +0,0 @@
-#!/usr/bin/perl -w
-#
-#
-# arguments:
-#     perl packfsck.pl <target version>
-
-
-my $target = $ARGV[0];
-system("rm /tmp/Manifest");
-system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core");
-
-
-my $from = $target;
-while ($from > $target - 100) {
- $from = $from - 10;
- print "Testing the $from-$target pack\n";
- system("rm /tmp/pack.tar");
- system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar");
-
-
-
- open FILE, "</tmp/Manifest";
-
- my %expected_hashes;
- my %files;
-
- while (<FILE>) {
-   my $line = $_;
-  
-  
-   if ($line =~ /(^[FDLr\.]+)\s*([0-9a-f]+)\s*([0-9]+)\s*+(.*)\n/) {
-      my $type = $1;
-      my $hash = $2;
-      my $version = $3;
-      my $filename = $4;
-      if ($version > $from) {
-        $expected_hashes{"$hash"} = 1;
-        $files{"$hash"} = $filename;
-#        print "$1 - $2 - $3 - $4\n";
-      }
-   } 
- }
- close FILE;
-
-
- open FILE2, "-|", "tar -tf /tmp/pack.tar 2> /dev/null";
-
- while (<FILE2>) {
-   my $line = $_;
-  
-
-  if ($line =~ /^\.\/delta\/[0-9]+.[0-9]+.([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected delta hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  } 
-  if ($line =~ /\.\/staged\/([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected staged hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  }
- }
- close FILE2;
-
- my $count = 0;
- foreach my $key ( keys %expected_hashes )
- {
-  my $value = $expected_hashes{$key};
-  if ($value eq 1) {
-     my $fn = $files{$key};
-     print "\tFile $fn ($key) is not in the pack\n";
-     $count = $count = 1;
-  }
- }
- if ($count < 1) {
-  print "\tThe $from-$target pack has no files missing\n";
- }
-}
@@ -2,3 +2,8 @@
 emptydir=/var/lib/update/empty/
 imagebase=/var/lib/update/image/
 outputdir=/var/lib/update/www/
+
+[Debuginfo]
+banned=true
+lib=/usr/lib/debug/
+src=/usr/src/debug/
@@ -42,6 +42,10 @@

 static GThreadPool *threadpool;

+/* Why not strcpy? Looks like the hash was going to be stored in
+ * binary at one stage. Should use g_string_chunk_insert_const to
+ * change hash_compare to a pointer compare
+ */
 void hash_assign(char *src, char *dst)
 {
 	memcpy(dst, src, SWUPD_HASH_LEN - 1);
@@ -147,7 +151,13 @@ void populate_file_struct(struct file *file, char *filename)
 	ret = lstat(filename, &stat);
 	if (ret < 0) {
 		LOG(NULL, "stat error ", "%s: %s", filename, strerror(errno));
-		file->is_deleted = 1;
+		/* delete the file if it isn't found and isn't a boot file,
+		 * mark as ghosted if this is a boot file */
+		if (file->is_boot) {
+			file->is_ghosted = 1;
+		} else {
+			file->is_deleted = 1;
+		}
 		return;
 	}
 	file->stat.st_mode = stat.st_mode;
@@ -196,7 +206,7 @@ int compute_hash(struct file *file, char *filename)
 		return 0;
 	}

-	hash_set_zeros(key);
+	hash_set_zeros(key); /* Set to 64 '0' (not '\0') characters */

 	if (file->is_link) {
 		char link[PATH_MAX];
@@ -275,7 +285,7 @@ static void get_hash(gpointer data, gpointer user_data)
 /* disallow characters which can do unexpected things when the filename is
 * used on a tar command line via system("tar [args] filename [more args]");
 */
-static bool illegal_characters(char *filename)
+static bool illegal_characters(const char *filename)
 {
 	char c;
 	int i;
@@ -301,27 +311,156 @@ static bool illegal_characters(char *filename)
 	return false;
 }

+static struct file *add_file(struct manifest *manifest,
+			     const char *entry_name,
+			     char *sub_filename,
+			     char *fullname,
+			     bool do_hash)
+{
+	GError *err = NULL;
+	struct file *file;
+
+	if (config_ban_debuginfo() && file_is_debuginfo(sub_filename)) {
+		printf("WARNING: File %s is banned ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
+	if (illegal_characters(entry_name)) {
+		printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
+	file = calloc(1, sizeof(struct file));
+	assert(file);
+
+	file->last_change = manifest->version;
+	file->filename = sub_filename;
+
+	populate_file_struct(file, fullname);
+	if (file->is_deleted) {
+		/*
+		 * populate_file_struct() logs a stat() failure, but
+		 * does not abort. When adding files that should
+		 * exist, this case is an error.
+		 */
+		LOG(NULL, "file not found", "%s", fullname);
+		assert(0);
+	}
+
+	/* if for some reason there is a file in the official build
+	 * which should not be included in the Manifest, then open a bug
+	 * to get it removed, and work around its presence by
+	 * excluding it here, eg:
+	 if (strncmp(file->filename, "/dev/", 5) == 0) {
+	 continue;
+	 }
+	*/
+
+	if (do_hash) {
+		/* compute the hash from a thread */
+		int ret;
+		ret = g_thread_pool_push(threadpool, file, &err);
+		if (ret == FALSE) {
+			printf("GThread hash computation push error\n");
+			printf("%s\n", err->message);
+			assert(0);
+		}
+	}
+	manifest->files = g_list_prepend(manifest->files, file);
+	manifest->count++;
+	return file;
+}
+
 static void iterate_directory(struct manifest *manifest, char *pathprefix,
 			      char *subpath, bool do_hash)
 {
 	DIR *dir;
 	struct dirent *entry;
 	char *fullpath;
-	int ret;
-	GError *err = NULL;

 	string_or_die(&fullpath, "%s/%s", pathprefix, subpath);

 	dir = opendir(fullpath);
 	if (!dir) {
+		bool fatal_error = errno != ENOENT;
+		FILE *content;
+
 		free(fullpath);
+		if (fatal_error) {
+			return;
+		}
+		/*
+		 * If there is a <dir>.content.txt instead of
+		 * the actual directory, then read that
+		 * file. It has a list of path names,
+		 * including all directories. The
+		 * corresponding file system entry is then
+		 * expected to be in a pre-populated "full"
+		 * directory.
+		 *
+		 * Only supported at top level (i.e. empty
+		 * subpath) to keep the code and testing
+		 * simpler.
+		 */
+		assert(!subpath[0]);
+		string_or_die(&fullpath, "%s.content.txt", pathprefix);
+		content = fopen(fullpath, "r");
+		free(fullpath);
+		fullpath = NULL;
+		if (content) {
+			char *line = NULL;
+			size_t len = 0;
+			ssize_t read;
+			const char *full;
+			int full_len;
+			/*
+			 * determine path to "full" directory: it is assumed to be alongside
+			 * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit,
+			 * so we have to strip the last path component.
+			 */
+			full = strrchr(pathprefix, '/');
+			if (full) {
+				full_len = full - pathprefix + 1;
+				full = pathprefix;
+			} else {
+				full = "";
+				full_len = 0;
+			}
+			while ((read = getline(&line, &len, content)) != -1) {
+				if (read) {
+					const char *entry_name = strrchr(line, '/');
+					if (entry_name) {
+						entry_name++;
+					} else {
+						entry_name = line;
+					}
+					if (line[read - 1] == '\n') {
+						line[read - 1] = 0;
+					}
+					string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line);
+					add_file(manifest,
+						 entry_name,
+						 strdup(line),
+						 fullpath,
+						 do_hash);
+				}
+			}
+			free(line);
+		}
+
+		// If both directory and content file are missing, silently (?)
+		// don't add anything to the manifest.
 		return;
 	}

 	while (dir) {
-		struct file *file;
 		char *sub_filename;
 		char *fullname;
+		struct file *file;

 		entry = readdir(dir);
 		if (!entry) {
@@ -334,50 +473,14 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}

 		string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
-
-		if (illegal_characters(entry->d_name)) {
-			printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
-			free(sub_filename);
-			continue;
-		}
-
-		file = calloc(1, sizeof(struct file));
-		if (!file) {
-			break;
-		}
-
-		file->last_change = manifest->version;
-		file->filename = sub_filename;
-
 		string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
-		populate_file_struct(file, fullname);
-		free(fullname);

-		if (entry->d_type == DT_DIR) {
+		/* takes ownership of the strings, so we don't need to free it */
+		file = add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
+
+		if (file && file->is_dir) {
 			iterate_directory(manifest, pathprefix, file->filename, do_hash);
 		}
-
-		/* if for some reason there is a file in the official build
-		 * which should not be included in the Manifest, then open a bug
-		 * to get it removed, and work around its presence by
-		 * excluding it here, eg:
-		if (strncmp(file->filename, "/dev/", 5) == 0) {
-			continue;
-		}
-		 */
-
-		if (do_hash) {
-			/* compute the hash from a thread */
-			ret = g_thread_pool_push(threadpool, file, &err);
-			if (ret == FALSE) {
-				printf("GThread hash computation push error\n");
-				printf("%s\n", err->message);
-				closedir(dir);
-				return;
-			}
-		}
-		manifest->files = g_list_prepend(manifest->files, file);
-		manifest->count++;
 	}
 	closedir(dir);
 	free(fullpath);
@@ -387,14 +490,15 @@ struct manifest *full_manifest_from_directory(int version)
 {
 	struct manifest *manifest;
 	char *dir;
+	int numthreads = num_threads(1.0);

 	LOG(NULL, "Computing hashes", "for %i/full", version);

-	manifest = alloc_manifest(version, "full");
+	manifest = alloc_manifest(version, "full", NULL);

 	string_or_die(&dir, "%s/%i/full", image_dir, version);

-	threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL);
+	threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL);

 	iterate_directory(manifest, dir, "", true);

@@ -466,7 +570,7 @@ struct manifest *sub_manifest_from_directory(char *component, int version)

 	LOG(NULL, "Creating component manifest", "for %i/%s", version, component);

-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);

 	string_or_die(&dir, "%s/%i/%s", image_dir, version, component);

@@ -39,15 +39,21 @@ void chroot_create_full(int newversion)
 	char *full_dir;

 	string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
+	if (!access(full_dir, R_OK | X_OK)) {
+		free(full_dir);
+		return;
+	}

 	g_mkdir_with_parents(full_dir, S_IRWXU);

 	/* start with base */
-	LOG(NULL, "Copying chroot os-core to full", "");
 	string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
-	char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
-	if (system_argv(rsynccmd) != 0) {
-		assert(0);
+	if (!access(param, F_OK)) {
+		LOG(NULL, "Copying chroot os-core to full", "");
+		char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
+		if (system_argv(rsynccmd) != 0) {
+			assert(0);
+		}
 	}
 	free(param);

@@ -58,11 +64,13 @@ void chroot_create_full(int newversion)
 			break;
 		}

-		LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
 		string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
-		char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
-		if (system_argv(rsynccmd) != 0) {
-			assert(0);
+		if (!access(param, F_OK)) {
+			LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
+			char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
+			if (system_argv(rsynccmd) != 0) {
+				assert(0);
+			}
 		}
 		free(param);
 	}
@@ -67,6 +67,20 @@ int config_initial_version(void)
 	return version;
 }

+bool config_ban_debuginfo(void)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_boolean(keyfile, "Debuginfo", "banned", NULL);
+}
+
+char *config_debuginfo_path(const char *comp)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_value(keyfile, "Debuginfo", comp, NULL);
+}
+
 bool read_configuration_file(char *filename)
 {
 	GError *error = NULL;
@@ -29,6 +29,7 @@
 #include <errno.h>
 #include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -49,12 +50,12 @@ static void banner(void)
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
 	{ "version", no_argument, 0, 'v' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "osversion", required_argument, 0, 'o' },
 	{ "minversion", required_argument, 0, 'm' },
 	{ "format", required_argument, 0, 'F' },
 	{ "getformat", no_argument, 0, 'g' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };

@@ -67,12 +68,12 @@ static void print_help(const char *name)
 	printf("   -v, --version           Show software version\n");
 	printf("\n");
 	printf("Application Options:\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -o, --osversion         The OS version for which to create an update\n");
 	printf("   -m, --minversion        Optional minimum file version to write into manifests per file\n");
 	printf("   -F, --format            Format number for the update\n");
 	printf("   -g, --getformat         Print current format string and exit\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }

@@ -80,12 +81,15 @@ static bool parse_options(int argc, char **argv)
 {
 	int opt;

-	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:s", prog_opts, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:", prog_opts, NULL)) != -1) {
 		switch (opt) {
 		case '?':
 		case 'h':
 			print_help(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'v':
 			banner();
 			return false;
@@ -124,9 +128,6 @@ static bool parse_options(int argc, char **argv)
 				free_globals();
 			}
 			exit(0);
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}

@@ -140,6 +141,7 @@ static bool parse_options(int argc, char **argv)
 static void populate_dirs(int version)
 {
 	char *newversiondir;
+	char *newversiondircontent = NULL;

 	string_or_die(&newversiondir, "%s/%d", image_dir, version);

@@ -181,9 +183,11 @@ static void populate_dirs(int version)
 			}

 			string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group);
+			string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group);

 			/* Create the bundle directory(s) as needed */
-			if (access(newversiondir, F_OK | R_OK) != 0) {
+			if (access(newversiondir, F_OK | R_OK) != 0 &&
+			    access(newversiondircontent, F_OK | R_OK) != 0) {
 				printf("%s does not exist...creating\n", group);
 				if (mkdir(newversiondir, 0755) != 0) {
 					printf("Failed to create %s subdirectory\n", group);
@@ -192,6 +196,7 @@ static void populate_dirs(int version)
 		}
 	}
 	free(newversiondir);
+	free(newversiondircontent);
 }

 static int check_build_env(void)
@@ -212,6 +217,35 @@ static int check_build_env(void)
 	return 0;
 }

+static int check_group_file(void)
+{
+	int ret = -1;
+
+	// Ensure the os-core group is defined
+	while (1) {
+		char *group = next_group();
+		if (!group) {
+			break;
+		}
+		if (strcmp(group, "os-core") == 0) {
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static bool both_ghosted(struct file *file1, struct file *file2)
+{
+	return (file1->is_ghosted && file2->is_ghosted);
+}
+
+static bool both_deleted(struct file *file1, struct file *file2)
+{
+	return (file1->is_deleted && file2->is_deleted);
+}
+
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -228,6 +262,7 @@ int main(int argc, char **argv)
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
 	int old_deleted = 0;
+	int old_ghosted = 0;

 	struct timeval current_time;
 	struct timeval previous_time;
@@ -240,6 +275,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);

+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_globals();
 		return EXIT_FAILURE;
@@ -253,12 +293,6 @@ int main(int argc, char **argv)
 		goto exit;
 	}

-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		goto exit;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	if (!read_configuration_file(file_path)) {
 		printf("Failed to read %s configuration file!\n", state_dir);
@@ -270,6 +304,11 @@ int main(int argc, char **argv)
 	string_or_die(&file_path, "%s/groups.ini", state_dir);
 	read_group_file(file_path);
 	free(file_path);
+	ret = check_group_file();
+	if (ret != 0) {
+		printf("os-core bundle is not listed in groups.ini, this is required.\n");
+		goto exit;
+	}

 	read_current_version("LAST_VER");
 	printf("Last processed version is %i\n", current_version);
@@ -297,15 +336,19 @@ int main(int argc, char **argv)
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);

-	old_deleted = remove_old_deleted_files(old_full, new_full);
+	old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
 	if (old_deleted > 0) {
 		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
 		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
 	}

+	old_ghosted = remove_deprecated_files(old_full, new_full, both_ghosted);
+	if (old_ghosted > 0) {
+		LOG(NULL, "", "Old ghosted files (%d) removed from full manifest", old_ghosted);
+		printf("Old ghosted files (%d) removed from full manifest", old_ghosted);
+	}
+
 	apply_heuristics(new_full);
-#warning disabled rename detection for some simplicity
-	// rename_detection(new_full);

 	print_elapsed_time("full manifest creation", &previous_time, &current_time);

@@ -314,7 +357,25 @@ int main(int argc, char **argv)

 	/* Step 2: Make a manifest for the os-core set */
 	old_MoM = manifest_from_file(current_version, "MoM");
-	new_MoM = alloc_manifest(newversion, "MoM");
+
+	/* Verify that the new format is not older than the previous format. It
+	 * never makes sense to decrease the format number for the next build. */
+	if (format < old_MoM->format) {
+		LOG(NULL, "", "Current format (%llu) must be greater than or equal to previous format (%llu). Exiting",
+		    format, old_MoM->format);
+		printf("Current format (%llu) must be greater than or equal to previous format (%llu). Exiting\n",
+		       format, old_MoM->format);
+		goto exit;
+	}
+
+	/* Detect a format bump and add the "update" action to the manifest
+	 * "actions:" field */
+	GList *actions = NULL;
+	if (format > old_MoM->format) {
+		actions = g_list_prepend(actions, "update");
+	}
+
+	new_MoM = alloc_manifest(newversion, "MoM", actions);
 	old_core = manifest_from_file(manifest_subversion(old_MoM, "os-core"), "os-core");
 	new_core = sub_manifest_from_directory("os-core", newversion);
 	add_component_hashes_to_manifest(new_core, new_full);
@@ -337,21 +398,23 @@ int main(int argc, char **argv)
 		apply_heuristics(new_core);
 		/* Step 3c: ... else save the manifest */
 		type_change_detection(new_core);
-
-#warning disabled rename detection for some simplicity
-		/* Detect renamed files specifically for each pack */
-		// rename_detection(...);
-
-		old_deleted = remove_old_deleted_files(old_core, new_core);
-		sort_manifest_by_version(new_core);
+#ifdef RENAMES
+		/* Detect renamed files specifically for os-core */
+		rename_detection(new_core);
+#endif
+		old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
+		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
+		sort_manifest_by_version(new_core); /* sorts by filename */
+		/* clean up orphaned renames by marking them as deleted */
+		clean_renames(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
 			LOG(NULL, "", "Core component has not changed (after pruning), exiting");
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
-		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", newfiles, old_deleted, old_ghosted);
+		printf("Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", newfiles, old_deleted, old_ghosted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
@@ -378,25 +441,40 @@ int main(int argc, char **argv)
 		GList *name_includes;
 		char *group = next_group();
 		struct manifest *manifest;
+		struct manifest *current = NULL;

 		if (!group) {
 			break;
 		}
 		manifest = g_hash_table_lookup(new_manifests, group);
 		name_includes = manifest->includes;
+
 		while (name_includes) {
 			char *name = name_includes->data;
 			name_includes = g_list_next(name_includes);
-			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(new_manifests, name));
+
+			current = g_hash_table_lookup(new_manifests, name);
+
+			// Avoid adding duplicate includes to the list
+			if (g_list_find(manifest_includes, current) == NULL) {
+				manifest_includes = g_list_prepend(manifest_includes, current);
+			}
 		}
 		manifest->includes = manifest_includes;
 		manifest_includes = NULL;
 		manifest = g_hash_table_lookup(old_manifests, group);
 		name_includes = manifest->includes;
+
 		while (name_includes) {
 			char *name = name_includes->data;
 			name_includes = g_list_next(name_includes);
-			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(old_manifests, name));
+
+			current = g_hash_table_lookup(old_manifests, name);
+
+			// Avoid adding duplicate includes to the list
+			if (g_list_find(manifest_includes, current) == NULL) {
+				manifest_includes = g_list_prepend(manifest_includes, current);
+			}
 		}
 		manifest->includes = manifest_includes;
 	}
@@ -443,15 +521,21 @@ int main(int argc, char **argv)
 			newm->version = oldm->version;
 		} else {
 			apply_heuristics(newm);
-#warning missing rename_detection here
+#ifdef RENAMES
+			/* Detect renamed files specifically for this bundle */
+			rename_detection(newm);
+#endif
 			/* Step 6b: otherwise, write out the manifest */
-			old_deleted = remove_old_deleted_files(oldm, newm);
+			old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
+			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
+			/* clean up orphaned renames by marking them as deleted */
+			clean_renames(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
-				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
-				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
+			if (newfiles > 0 || old_deleted > 0 || old_ghosted > 0 || changed_includes(oldm, newm)) {
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", group, newfiles, old_deleted, old_ghosted);
+				printf("%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", group, newfiles, old_deleted, old_ghosted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);
@@ -488,6 +572,8 @@ int main(int argc, char **argv)
 	maximize_to_full(new_MoM, new_full);

 	sort_manifest_by_version(new_full);
+	/* clean up orphaned renames by marking them as deleted */
+	clean_renames(new_full);
 	prune_manifest(new_full);
 	if (write_manifest(new_full) != 0) {
 		goto exit;
@@ -506,7 +592,6 @@ exit:
 	}
 	release_configuration_data();
 	release_group_file();
-	signature_terminate();
 	g_list_free(manifests_last_versions_list);

 	close_log(newversion, exit_status);
@@ -40,8 +40,8 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
 	char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf;
 	int ret;

-	if (file->is_link) {
-		return;
+	if (!file->is_file || !file->peer->is_file) {
+		return; /* only support deltas between two regular files right now */
 	}

 	if (file->is_deleted) {
@@ -97,7 +97,7 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
 		ret = 0;
 		goto out;
 	}
-	xattrs_copy(original, newfile);
+	xattrs_copy(original, testnewfile);

 	/* does xattrs have been correctly copied?*/
 	if (xattrs_compare(original, testnewfile) != 0) {
@@ -50,7 +50,6 @@ static void create_fullfile(struct file *file)
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
 	char *param1, *param2;
-	int stderrfd;

 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -72,7 +71,7 @@ static void create_fullfile(struct file *file)
 	string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
 	if (lstat(origin, &sbuf) < 0) {
 		/* no input file: means earlier phase of update creation failed */
-		LOG(NULL, "Failed to stat %s\n", origin);
+		LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno));
 		assert(0);
 	}

@@ -98,17 +97,13 @@ static void create_fullfile(struct file *file)
 		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
 		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };

-		stderrfd = open("/dev/null", O_WRONLY);
-		if (stderrfd == -1) {
-			LOG(NULL, "Failed to open /dev/null", "");
-			assert(0);
-		}
-		if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
+		int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
+		if (tarcmdresult != 0) {
+			LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
 			assert(0);
 		}
 		free(param1);
 		free(param2);
-		close(stderrfd);

 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
@@ -249,22 +244,20 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest)
 	struct file *tmp;

 	// presort by hash for easy deduplication
-	manifest->files = g_list_sort(manifest->files, file_sort_hash);
+	list = manifest->files = g_list_sort(manifest->files, file_sort_hash);

-	list = g_list_first(manifest->files);
-	while (prev == NULL) {
+	for (; list; list = g_list_next(list)) {
 		tmp = list->data;
-		list = g_list_next(list);

 		// find first new file
 		if (tmp->last_change == manifest->version) {
 			prev = tmp;
 			outfiles = g_list_prepend(outfiles, tmp);
+			break;
 		}
 	}
-	while (list) {
+	for (; list; list = g_list_next(list)) {
 		file = list->data;
-		list = g_list_next(list);

 		// add any new file having a unique hash
 		//FIXME: rename logic will be needed here
@@ -291,10 +284,11 @@ static void submit_fullfile_tasks(GList *files)
 	int ret;
 	int count = 0;
 	GError *err = NULL;
+	int numthreads = num_threads(3.0);

-	LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
+	LOG(NULL, "fullfile threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_fullfile_task, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN) * 3,
+				       numthreads,
 				       TRUE, NULL);

 	printf("Starting downloadable fullfiles data creation\n");
@@ -304,6 +298,11 @@ static void submit_fullfile_tasks(GList *files)
 		file = item->data;
 		item = g_list_next(item);

+		/* do not push ghosted files */
+		if (file->is_ghosted) {
+			continue;
+		}
+
 		ret = g_thread_pool_push(threadpool, file, &err);
 		if (ret == FALSE) {
 			printf("GThread create_fullfile_task push error\n");
@@ -34,7 +34,6 @@
 int newversion = -1;
 int minversion = 0;
 unsigned long long int format = 0;
-bool enable_signing = false;

 char *state_dir = NULL;
 char *packstage_dir = NULL;
@@ -37,6 +37,7 @@ static char **groups = NULL;
 static gsize groupcount = 0;
 static unsigned int groupcursor = 0;

+#if 0
 char *group_packages(char *group)
 {
 	assert(groupfile != NULL);
@@ -50,6 +51,7 @@ char *group_groups(char *group)

 	return g_key_file_get_value(groupfile, group, "groups", NULL);
 }
+#endif

 char *group_status(char *group)
 {
@@ -149,130 +149,117 @@ void concat_str_array(char **output, char *const argv[])
 	}
 }

+int system_argv_pipe(char *const lhscmd[], char *const rhscmd[])
+{
+	return system_argv_pipe_fd(-1, -1, lhscmd, -1, -1, rhscmd);
+}
+
+int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	pid_t monitorpid = fork();
+	if (monitorpid == -1) {
+		LOG(NULL, "Failed to create child process to monitor pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		return -1;
+	} else if (monitorpid == 0) {
+		pipe_monitor(lnewstdinfd, lnewstderrfd, lhscmd, rnewstdoutfd, rnewstderrfd, rhscmd);
+	}
+	return wait_process_terminate(monitorpid);
+}
+
+void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+		  int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	int pipefd[2];
+	if (pipe(pipefd) == -1) {
+		LOG(NULL, "Failed to create a pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		assert(0);
+	}
+
+	pid_t lhspid = system_argv_fd_nowait(lnewstdinfd, pipefd[1], lnewstderrfd, pipefd[0], lhscmd);
+	pid_t rhspid = system_argv_fd_nowait(pipefd[0], rnewstdoutfd, rnewstderrfd, pipefd[1], rhscmd);
+
+	if (close(pipefd[1]) == -1) {
+		LOG(NULL, "Could not close write end of pipe file descriptor", "%d", pipefd[1]);
+		assert(0);
+	}
+	if (close(pipefd[0]) == -1) {
+		LOG(NULL, "Could not close read end of pipe file descriptor", "%d", pipefd[0]);
+		assert(0);
+	}
+
+	int lhsresult = wait_process_terminate(lhspid);
+	int rhsresult = wait_process_terminate(rhspid);
+	exit(rhsresult != EXIT_SUCCESS ? rhsresult : lhsresult);
+}
+
 int system_argv(char *const argv[])
 {
-	int child_exit_status;
-	pid_t pid;
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
+	return system_argv_fd(-1, -1, -1, argv);
 }

-int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr)
+int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[])
 {
-	int child_exit_status;
-	pid_t pid;
-	int status = -1;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		if (newstdin >= 0) {
-			if (dup2(newstdin, STDIN_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdin", "");
-				assert(0);
-			}
-			close(newstdin);
-		}
-		if (newstdout >= 0) {
-			if (dup2(newstdout, STDOUT_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stdout", "");
-				assert(0);
-			}
-			close(newstdout);
-		}
-		if (newstderr >= 0) {
-			if (dup2(newstderr, STDERR_FILENO) == -1) {
-				LOG(NULL, "Could not redirect stderr", "");
-				assert(0);
-			}
-			close(newstderr);
-		}
-
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
-	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-	}
-
-	return status;
+	pid_t cmdpid = system_argv_fd_nowait(newstdinfd, newstdoutfd, newstderrfd, -1, cmd);
+	return wait_process_terminate(cmdpid);
 }

-int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
-		     char *const argvp2[], int stdoutp2, int stderrp2)
+pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
 {
-	int statusp2;
-	int pipefd[2];
-
-	if (pipe(pipefd)) {
-		LOG(NULL, "Failed to create a pipe", "");
-		return -1;
+	pid_t cmdpid = fork();
+	if (cmdpid == -1) {
+		LOG(NULL, "Failed to fork to execute command", "%s", cmd[0]);
+		assert(0);
+	} else if (cmdpid == 0) {
+		exec_cmd_fd(newstdinfd, newstdoutfd, newstderrfd, closefd, cmd);
 	}
-	system_argv_fd(argvp1, stdinp1, pipefd[1], stderrp1);
-	close(pipefd[1]);
-	statusp2 = system_argv_fd(argvp2, pipefd[0], stdoutp2, stderrp2);
-	close(pipefd[0]);
+	return cmdpid;
+}

-	/* Returns the status of the failed process if any
-       If both processes failed returns the status of first one */
-	return statusp2;
+void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
+{
+	move_fd(newstdinfd, STDIN_FILENO);
+	move_fd(newstdoutfd, STDOUT_FILENO);
+	move_fd(newstderrfd, STDERR_FILENO);
+	if (closefd >= 0 && close(closefd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", closefd);
+		assert(0);
+	}
+	execvp(*cmd, cmd);
+	LOG(NULL, "Command", "%s failed", cmd[0]);
+	assert(0);
+}
+
+void move_fd(int oldfd, int newfd)
+{
+	if (oldfd < 0 || newfd < 0 || oldfd == newfd) {
+		return;
+	}
+	if (dup2(oldfd, newfd) == -1) {
+		LOG(NULL, "Could not create duplicate file descriptor", "%d from %d", newfd, oldfd);
+		assert(0);
+	}
+	if (close(oldfd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", oldfd);
+		assert(0);
+	}
+}
+
+int wait_process_terminate(pid_t pid)
+{
+	int status;
+	do {
+		if (waitpid(pid, &status, 0) == -1) {
+			LOG(NULL, "Failed to wait for PID", "%d", pid);
+			return -1;
+		}
+	} while (!WIFEXITED(status) && !WIFSIGNALED(status));
+	// Exit statuses fall in the range of [0, 255].  Make signal statuses fall in a non-overlapping range starting with 256.
+	if (WIFEXITED(status)) {
+		return WEXITSTATUS(status);
+	} else {
+		return 256 + WTERMSIG(status);
+	}
 }

 void check_root(void)
@@ -284,3 +271,56 @@ void check_root(void)
 		exit(EXIT_FAILURE);
 	}
 }
+
+int num_threads(float scaling)
+{
+	const char *var = getenv("SWUPD_NUM_THREADS");
+	int result = sysconf(_SC_NPROCESSORS_ONLN) * scaling;
+
+	if (var && *var) {
+		char *endptr;
+		long int value;
+
+		errno = 0;
+		value = strtol(var, &endptr, 0);
+
+		if ((errno != 0 && value == 0) || *endptr) {
+			LOG(NULL, "SWUPD_NUM_THREADS must be an integer", "%s", var);
+		} else if ((errno == ERANGE && (value == LONG_MAX || value == LONG_MIN)) ||
+			   value < 1 || value > INT_MAX) {
+			LOG(NULL, "SWUPD_NUM_THREADS out of range", "%s", var);
+		} else {
+			result = (int)value;
+		}
+	}
+
+	return result;
+}
+
+/* This function is called when configuration specifies a ban on debuginfo files
+ * from manifests. Returns true if the passed file path matches the src or lib
+ * debuginfo configuration. */
+bool file_is_debuginfo(const char *path)
+{
+	bool ret = false;
+	char *lib;
+	char *src;
+
+	lib = config_debuginfo_path("lib");
+	src = config_debuginfo_path("src");
+
+	if (lib && (strncmp(path, lib, strlen(lib)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+	if (src && (strncmp(path, src, strlen(src)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+out:
+	free(lib);
+	free(src);
+	return ret;
+}
@@ -44,15 +44,15 @@ static void runtime_state_heuristics(struct file *file)
 {
 	/* these are shipped directories that are not themselves state,
 	 * rather only their contents are state */
-	if ((strncmp(file->filename, "/usr/src/debug", 14) == 0) ||
-	    (strncmp(file->filename, "/dev", 4) == 0) ||
-	    (strncmp(file->filename, "/home", 5) == 0) ||
-	    (strncmp(file->filename, "/proc", 5) == 0) ||
-	    (strncmp(file->filename, "/root", 5) == 0) ||
-	    (strncmp(file->filename, "/run", 4) == 0) ||
-	    (strncmp(file->filename, "/sys", 4) == 0) ||
-	    (strncmp(file->filename, "/tmp", 4) == 0) ||
-	    (strncmp(file->filename, "/var", 4) == 0)) {
+	if ((strcmp(file->filename, "/usr/src/debug") == 0) ||
+	    (strcmp(file->filename, "/dev") == 0) ||
+	    (strcmp(file->filename, "/home") == 0) ||
+	    (strcmp(file->filename, "/proc") == 0) ||
+	    (strcmp(file->filename, "/root") == 0) ||
+	    (strcmp(file->filename, "/run") == 0) ||
+	    (strcmp(file->filename, "/sys") == 0) ||
+	    (strcmp(file->filename, "/tmp") == 0) ||
+	    (strcmp(file->filename, "/var") == 0)) {
 		return;
 	}

@@ -33,7 +33,7 @@

 #include "swupd.h"

-static FILE *logfile;
+static FILE *logfile[2];

 static struct timeval start_time;

@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end)
 {
 	char *filename;
 	string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end);
-	logfile = fopen(filename, "w");
+	logfile[0] = fopen(filename, "w");
 	free(filename);
 	gettimeofday(&start_time, NULL);
 }
 void init_log_stdout(void)
 {
-	logfile = stdout;
+	logfile[1] = stdout;
 	gettimeofday(&start_time, NULL);
 }

@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 	char *logstring = NULL;
 	char filebuf[4096];
 	char filebuf2[4096];
+	int i;

-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}

@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 		strcat(filebuf2, " ");
 	}

-	fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
-		(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+	for (i = 0; i < 2; i++) {
+		if (logfile[i]) {
+			fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
+				(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+			fflush(logfile[i]);
+		}
+	}

 	free(logstring);
 	free(buf);
-	fflush(logfile);
 }

 void close_log(int version, int exit_status)
@@ -133,7 +138,7 @@ void close_log(int version, int exit_status)
 	int t_sec;
 	int t_msec;

-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}

@@ -159,6 +164,8 @@ void close_log(int version, int exit_status)
 		printf("Update build failed for version %i\n", version);
 	}

-	fclose(logfile);
-	logfile = NULL;
+	if (logfile[0]) {
+		fclose(logfile[0]);
+		logfile[0] = NULL;
+	}
 }
@@ -23,6 +23,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <getopt.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,6 +33,7 @@

 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
 	{ 0, 0, 0, 0 }
 };
@@ -42,6 +44,7 @@ static void usage(const char *name)
 	printf("   %s <version>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
 	printf("\n");
 }
@@ -56,6 +59,9 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument '%s'\n\n", optarg);
@@ -88,6 +94,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);

+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;
@@ -25,8 +25,8 @@
 #include <dirent.h>
 #include <errno.h>
 #include <getopt.h>
-#include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -45,8 +45,8 @@ static void banner(void)

 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };

@@ -56,8 +56,8 @@ static void usage(const char *name)
 	printf("   %s <start version> <latest version> <bundle>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }

@@ -71,15 +71,15 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument ''%s'\n\n", optarg);
 				return false;
 			}
 			break;
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}

@@ -101,6 +101,11 @@ int main(int argc, char **argv)
 	int exit_status = EXIT_FAILURE;
 	char *file_path = NULL;

+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;
@@ -114,12 +119,6 @@ int main(int argc, char **argv)
 	banner();
 	check_root();

-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		return exit_status;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	read_configuration_file(file_path);
 	free(file_path);
@@ -152,8 +151,6 @@ int main(int argc, char **argv)
 		exit_status = EXIT_SUCCESS;
 	}

-	signature_terminate();
-
 	printf("Pack creation %s (pack-%s %i to %li)\n",
 	       exit_status == EXIT_SUCCESS ? "complete" : "failed",
 	       module, start_version, end_version);
@@ -51,6 +51,10 @@ int file_sort_hash(gconstpointer a, gconstpointer b)
 	return memcmp(A->hash, B->hash, SWUPD_HASH_LEN - 1);
 }

+/* Standard sort compare function which sorts
+ * first on the version where the file changed and then
+ * on the name
+ */
 int file_sort_version(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
@@ -64,7 +68,7 @@ int file_sort_version(gconstpointer a, gconstpointer b)
 		return 1;
 	}

-	return strcmp(B->filename, A->filename);
+	return strcmp(A->filename, B->filename);
 }

 int file_sort_filename(gconstpointer a, gconstpointer b)
@@ -88,7 +92,7 @@ int file_sort_filename(gconstpointer a, gconstpointer b)
 	return 0;
 }

-struct manifest *alloc_manifest(int version, char *component)
+struct manifest *alloc_manifest(int version, char *component, GList *actions)
 {
 	struct manifest *manifest;

@@ -100,6 +104,7 @@ struct manifest *alloc_manifest(int version, char *component)
 	manifest->version = version;
 	manifest->component = strdup(component);
 	manifest->format = format;
+	manifest->actions = actions;

 	return manifest;
 }
@@ -129,7 +134,7 @@ struct manifest *manifest_from_file(int version, char *component)
 	if (infile == NULL) {
 		LOG(NULL, "Cannot read manifest", "%s (%s)\n", filename, strerror(errno));
 		free(filename);
-		return alloc_manifest(version, component);
+		return alloc_manifest(version, component, NULL);
 	}

 	/* line 1: MANIFEST\t<version> */
@@ -189,7 +194,7 @@ struct manifest *manifest_from_file(int version, char *component)
 		}
 	}

-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);
 	manifest->format = format_number;
 	manifest->prevversion = previous;
 	manifest->includes = includes;
@@ -235,10 +240,22 @@ struct manifest *manifest_from_file(int version, char *component)
 			assert(0); /* unknown file type */
 		}

-		if (c[1] == 'd') {
+		switch (c[1]) {
+		case 'd':
+			/* file is deleted */
 			file->is_deleted = 1;
-		} else if (c[1] != '.') {
-			assert(0); /* unknown deleted status */
+			break;
+		case 'g':
+			/* file is ghosted */
+			file->is_ghosted = 1;
+			break;
+		case '.':
+			/* no flag at this index */
+			break;
+		default:
+			/* unknown deleted status */
+			LOG(NULL, "Invalid flag at index 1", "%s", c[1]);
+			assert(0);
 		}

 		if (c[2] == 'C') {
@@ -423,15 +440,20 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			file3->is_config = file1->is_config;
 			file3->is_state = file1->is_state;
 			file3->is_boot = file1->is_boot;
+			/* ghost deleted boot files */
+			file3->is_ghosted = file1->is_boot && file1->is_deleted;

-			if (!file1->is_deleted) {
-				file3->last_change = m2->version;
-			} else {
-				file3->last_change = file1->last_change;
+			if (file3->is_ghosted || file1->is_deleted) {
+				/* if the new file is ghosted or the file was deleted, preserve
+				 * hash (all zeros if file1->is_deleted) and rename status */
 				file3->is_rename = file1->is_rename;
 				hash_assign(file1->hash, file3->hash);
 			}

+			/* for deleted files last_change remains the same.
+			 * otherwise last change is now */
+			file3->last_change = file1->is_deleted ? file1->last_change : m2->version;
+
 			file3->peer = file1;
 			file1->peer = file3;

@@ -582,6 +604,16 @@ void subtract_manifests(struct manifest *m1, struct manifest *m2)
 			list1 = g_list_next(list1);
 			list2 = g_list_next(list2);

+			/* When both files are marked deleted, skip
+			 * subtraction. Preserving the deleted entries in both
+			 * manifests is required for 'swupd update' to know
+			 * when to delete the file, because the m2 bundle may
+			 * be installed with or without the m1 bundle.
+			 */
+			if (file1->is_deleted && file2->is_deleted) {
+				continue;
+			}
+
 			if (file1->is_deleted == file2->is_deleted && file1->is_file == file2->is_file) {
 				m1->files = g_list_delete_link(m1->files, todel);
 				m1->count--;
@@ -643,6 +675,10 @@ char *file_type_to_string(struct file *file)
 		type[1] = 'd';
 	}

+	if (file->is_ghosted) {
+		type[1] = 'g';
+	}
+
 	if (file->is_config) {
 		type[2] = 'C';
 	}
@@ -660,19 +696,29 @@ char *file_type_to_string(struct file *file)
 	return type;
 }

+/* Calculate the contentsize for the manifest based on file sizes.
+ *
+ * This should calculate the files uniquely included in this manifest, but none
+ * of its submanifests, which will allow calculation of sizes of all bundles on
+ * a system by adding all manifest->contentsizes of installed bundles.
+ *
+ * However, if two bundles not in the same include chain have overlapping
+ * content, summing the include chain of each bundle in the client will result
+ * in an over-estimation of the total size on the system. The more content is
+ * shared, the higher the over-estimation. In reality this overlap will not be
+ * large, but it is currently impossible to calculate the exact installed size
+ * using just the contentsize.
+ */
 static void compute_content_size(struct manifest *manifest)
 {
-	/* FIXME: this is a temporary implementation based on worst case */
-
 	GList *list;
 	struct file *file;
-	struct manifest *submanifest;

 	list = g_list_first(manifest->files);
 	while (list) {
 		file = list->data;
 		list = g_list_next(list);
-		if (!file->is_deleted && (file->last_change == manifest->version)) {
+		if (!file->is_deleted) {
 			if (file->is_file) {
 				manifest->contentsize += file->stat.st_size;
 			} else if (file->is_link) {
@@ -682,40 +728,6 @@ static void compute_content_size(struct manifest *manifest)
 			}
 		}
 	}
-
-	list = g_list_first(manifest->submanifests);
-	while (list) {
-		submanifest = list->data;
-		list = g_list_next(list);
-
-		/* Do not take into account groups not included in download content */
-		if (create_download_content_for_group(submanifest->component)) {
-			manifest->contentsize += submanifest->contentsize;
-		}
-	}
-}
-
-/* Returns 0 == success, -1 == failure */
-static int write_manifest_signature(struct manifest *manifest, const char *suffix)
-{
-	char *conf = config_output_dir();
-	char *filename = NULL;
-	int ret = -1;
-
-	if (conf == NULL) {
-		assert(0);
-	}
-	string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version,
-		      manifest->component, suffix);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	free(conf);
-	return ret;
 }

 /* Returns 0 == success, -1 == failure */
@@ -723,6 +735,7 @@ static int write_manifest_plain(struct manifest *manifest)
 {
 	GList *includes;
 	GList *list;
+	GList *actions;
 	struct file *file;
 	FILE *out = NULL;
 	char *base = NULL, *dir;
@@ -772,6 +785,14 @@ static int write_manifest_plain(struct manifest *manifest)
 		includes = g_list_next(includes);
 		fprintf(out, "includes:\t%s\n", sub->component);
 	}
+
+	actions = manifest->actions;
+	while (actions) {
+		char *action = actions->data;
+		fprintf(out, "actions:\t%s\n", action);
+		actions = g_list_next(actions);
+	}
+
 	fprintf(out, "\n");

 	list = g_list_first(manifest->files);
@@ -851,7 +872,7 @@ exit:
 static int write_manifest_tar(struct manifest *manifest)
 {
 	char *conf = config_output_dir();
-	char *directory, *manifesttar, *manifestcomp, *manifestsigned;
+	char *directory, *manifesttar, *manifestcomp;
 	int ret = 0;

 	if (conf == NULL) {
@@ -861,19 +882,11 @@ static int write_manifest_tar(struct manifest *manifest)
 	string_or_die(&directory, "--directory=%s/%i", conf, manifest->version);
 	string_or_die(&manifesttar, "%s/%i/Manifest.%s.tar", conf, manifest->version, manifest->component);
 	string_or_die(&manifestcomp, "Manifest.%s", manifest->component);
-	string_or_die(&manifestsigned, "Manifest.%s.signed", manifest->component);

 	/* now, tar the thing up for efficient full file download */
-	/* and put the signature of the plain manifest into the archive, too */
-	if (enable_signing) {
-		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-					 manifesttar, manifestcomp, manifestsigned, NULL };
-		ret = system_argv(tarcmd);
-	} else {
-		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-					 manifesttar, manifestcomp, NULL };
-		ret = system_argv(tarcmd);
-	}
+	char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
+				 manifesttar, manifestcomp, NULL };
+	ret = system_argv(tarcmd);
 	if (ret) {
 		fprintf(stderr, "Creation of Manifest.tar failed\n");
 	}
@@ -881,7 +894,6 @@ static int write_manifest_tar(struct manifest *manifest)
 	free(directory);
 	free(manifesttar);
 	free(manifestcomp);
-	free(manifestsigned);
 	free(conf);
 	return ret;
 }
@@ -911,9 +923,7 @@ bool compute_hash_with_xattrs(const char *filename)
 int write_manifest(struct manifest *manifest)
 {
 	if (write_manifest_plain(manifest) == 0 &&
-	    write_manifest_signature(manifest, "") == 0 &&
-	    write_manifest_tar(manifest) == 0 &&
-	    write_manifest_signature(manifest, ".tar") == 0) {
+	    write_manifest_tar(manifest) == 0) {
 		return 0;
 	}
 	return -1;
@@ -979,7 +989,7 @@ bool changed_includes(struct manifest *old, struct manifest *new)
 *
 * Note: this function should be called after match_manifests().
 */
-int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
+int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *file1, struct file *file2))
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
@@ -1017,7 +1027,9 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)

 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->is_deleted && file2->is_deleted) {
+			/* use the comparison function passed in to determine if this file
+			 * should be removed */
+			if (compfunc(file1, file2)) {
 				GList *to_delete = list2;
 				list1 = g_list_next(list1);
 				list2 = g_list_next(list2);
@@ -1038,6 +1050,32 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 	return count;
 }

+/* Performs a final link of renames in the manifest to catch any renames from
+ * old versions and then removes any orphaned renames */
+void clean_renames(struct manifest *manifest)
+{
+	GList *list;
+	struct file *file;
+
+	/* make sure all renames are linked, this is necessary for renames
+	 * from old manifests that carry over to the current one */
+	final_link(manifest->files);
+
+	list = g_list_first(manifest->files);
+	while (list) {
+		file = list->data;
+		/* if the file is marked as deleted and renamed it is a
+		 * renamed-from file. If the rename_peer field is still NULL
+		 * then it has been orphaned. Mark these as deleted */
+		if (file->is_deleted && file->is_rename && !file->rename_peer) {
+			hash_set_zeros(file->hash);
+			file->is_rename = 0;
+		}
+
+		list = g_list_next(list);
+	}
+}
+
 /* Conditionally remove some things from a manifest.
 * Returns > 0 when the pruned manifest has new files.
 * Returns 0 when the pruned manifest no longer has new files.
@@ -1062,8 +1100,16 @@ int prune_manifest(struct manifest *manifest)
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		} else if (file->is_boot && file->is_deleted) {
-			// only expose the current best boot files, a client side entity can manage /boot's actual contents
-			// LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component);
+			/* mark boot files that are going away as ghosted, these will be
+			 * cleaned up with the next update */
+			file->is_deleted = 0;
+			file->is_ghosted = 1;
+		} else if (config_ban_debuginfo() && file_is_debuginfo(file->filename)) {
+			/* The configuration option to ban debuginfo from the manifests was
+			 * set in server.ini via the [Debuginfo][banned] option. Although
+			 * debuginfo additions are banned via analyze_fs, prune it here
+			 * to insure mistakenly included debuginfo from old versions is
+			 * removed from the manifests. */
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		}
@@ -1115,9 +1161,6 @@ void create_manifest_delta(int oldversion, int newversion, char *module)
 			}
 			LOG(NULL, "Failed to rename", "");
 		}
-		if (!signature_sign(outfile)) {
-			fprintf(stderr, "Creating signature for '%s' failed\n", outfile);
-		}
 	} else {
 		sleep(1); /* we raced. whatever. sleep for a bit to get the other guy to make progress */
 	}
@@ -58,11 +58,11 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m
 		// (re)create module/version/{delta,staged}
 		string_or_die(&path, "%s/%s/%i_to_%i/delta", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 		string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 	}
 }
@@ -79,7 +79,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)

 	string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 		      from_version, to_version);
-	g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+	g_mkdir_with_parents(path, S_IRWXU);
 	dir = opendir(path);
 	if (!dir) {
 		fprintf(stderr, "There are problems accessing %s, exiting\n", path);
@@ -116,7 +116,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 		 * time on the client...
 		 */
 		string_or_die(&param, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version);
-		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
 		if (system_argv(tarcmd) == 0) {
 			unlink(path);
 		}
@@ -144,7 +144,7 @@ static void prepare_pack(struct packdata *pack)

 	match_manifests(manifest, pack->end_manifest);

-	link_renames(pack->end_manifest->files, manifest);
+	link_renames(pack->end_manifest->files, pack->to);
 }

 static void make_pack_full_files(struct packdata *pack)
@@ -285,10 +285,11 @@ static void make_pack_deltas(GList *files)
 	struct file *file;
 	int ret;
 	GError *err = NULL;
+	int numthreads = num_threads(1.0);

-	LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
+	LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_delta, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
+				       numthreads, FALSE, NULL);

 	item = g_list_first(files);
 	while (item) {
@@ -307,24 +308,6 @@ static void make_pack_deltas(GList *files)
 	g_thread_pool_free(threadpool, FALSE, TRUE);
 }

-/* Returns 0 == success, -1 == failure */
-static int write_pack_signature(struct packdata *pack)
-{
-	char *filename = NULL;
-	int ret = -1;
-
-	string_or_die(&filename, "%s/%i/pack-%s-from-%i.tar",
-		      staging_dir, pack->to, pack->module, pack->from);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	return ret;
-}
-
 /* Returns 0 == success, other == failure */
 static int make_final_pack(struct packdata *pack)
 {
@@ -507,13 +490,6 @@ static int make_final_pack(struct packdata *pack)
 	if ((ret != 0) && (ret != 1)) {
 		fprintf(stderr, "Unexpected return value (%d) creating tar of pack %s from %i to %i\n",
 			ret, pack->module, pack->from, pack->to);
-	} else {
-		/* Write the signature file */
-		ret = write_pack_signature(pack);
-		if (ret != 0) {
-			fprintf(stderr, "Failure creating signature of pack %s from %i to %i\n",
-				pack->module, pack->from, pack->to);
-		}
 	}

 	/* and clean up */
@@ -21,6 +21,9 @@
 *
 */

+/* Rename detection and support.
+ */
+
 #define _GNU_SOURCE
 #include <assert.h>
 #include <ctype.h>
@@ -37,8 +40,55 @@

 #include <magic.h>

-static magic_t mcookie;
+static bool samefiletype(char *t1, char *t2)
+{
+	if (t1 && (t1 == t2) && *t1) {
+		return true;
+	}
+	return false;
+}
+/* For an elf binary, we get the BuildID in it, which
+ * is essentially a hash of the original loaded sections
+ * and hence is unique. This hardly makes for a 'type'
+ */
+static char *getmagic(char *filename)
+{
+	static magic_t mcookie;
+	static GStringChunk *typestore;
+	char *c2;
+	char *c1;
+	if (mcookie == NULL) {
+		mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS);
+		magic_load(mcookie, NULL);
+		typestore = g_string_chunk_new(200);
+	}

+	c1 = (char *)magic_file(mcookie, filename);
+	if (!c1) {
+		LOG(NULL, "Cannot find file type", "%s", filename);
+		c1 = "";
+	}
+	c1 = strdup(c1);
+	c2 = strstr(c1, ", BuildID[");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = strstr(c1, "not stripped");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = strstr(c1, "stripped");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = g_string_chunk_insert_const(typestore, c1);
+	free(c1);
+	return c2;
+}
+
+/* Assign a score roughly in the range -100 to 1000 to express how similar
+ * two files are.
+ */
 double rename_score(struct file *old, struct file *new)
 {
 	double score = 0.0;
@@ -50,6 +100,16 @@ double rename_score(struct file *old, struct file *new)
 		score += 400;
 	}

+	/* If the files are smaller than about 200 bytes then even a
+	 * single byte change using bsdiff is going to work out as
+	 * bigger than just shipping the new file, so stop if they are
+	 * not the same. No point in running up bsdiff just for the
+	 * sake of it.
+	 */
+	if (new->stat.st_size < BSDIFFSIZE) {
+		return -99.0;
+	}
+
 	/* points for being in the same directory */
 	if (strcmp(old->dirname, new->dirname) == 0) {
 		score += 10;
@@ -86,12 +146,14 @@ double rename_score(struct file *old, struct file *new)
 		in--;
 	}

+#if 0
 	/* if both start with /boot/vmlinuz give it a boost; this is a local hack due to vmlinuz being very short */
 	if (strncmp(old->filename, "/boot/vmlinuz", 13) == 0 && strncmp(new->filename, "/boot/vmlinuz", 13) == 0) {
 		score += 80;
 	}

 	/* if ELF, points for sharing the same soname to the first dot */
+#endif

 	/* negative points for not being within 25%+/-1Kb of the same file size */
 	if (old->stat.st_size > ((new->stat.st_size * 1.25) + 1024)) {
@@ -115,66 +177,40 @@ double rename_score(struct file *old, struct file *new)
 	}
 	/* negative points for not having the same 'file' type */

-	if (old->filetype && new->filetype && strcmp(old->filetype, new->filetype) != 0) {
+	if (!samefiletype(old->filetype, new->filetype)) {
 		score -= 60;
 	}

 	return score;
 }

-static void precompute_file_data(struct manifest *manifest, struct file *file, int old_rename, GList *last_versions_list)
+static void precompute_file_data(int version, const char *component, struct file *file)
 {
-	GList *item;
 	char *c1, *c2;
 	char *filename = NULL;
-	int last_change;
-	struct stat buf;

+	assert(file);
 	/* fill in the filename-minus-the-numbers field */
 	file->alpha_only_filename = calloc(strlen(file->filename) + 1, sizeof(char));

 	c1 = file->filename;
 	c2 = file->alpha_only_filename;
-	while (*c1) {
-		while (*c1 && !isalpha(*c1))
-			c1++;
-		if (!*c1) {
-			break;
-		}
-		if (c2 == NULL) {
-			break;
-		}
-		*c2 = *c1;
-		c1++;
-		c2++;
-	}
-
-	if (manifest) {
-		string_or_die(&filename, "%s/%i/%s/%s", image_dir, manifest->version, manifest->component, file->filename);
-	} else if (old_rename) {
-		item = g_list_first(last_versions_list);
-		while (item) {
-			last_change = GPOINTER_TO_INT(item->data);
-			item = g_list_next(item);
-
-			free(filename);
-			string_or_die(&filename, "%s/%i/full/%s", image_dir, last_change, file->filename);
-			if (!lstat(filename, &buf)) {
-				break;
+	if (c2) {
+		for (; *c1; c1++) {
+			if (isalpha(*c1)) { /* Only copy letters */
+				*c2++ = *c1;
 			}
 		}
-	} else {
-		string_or_die(&filename, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+		/* alpha_only_filename is NUL terminated by calloc */
 	}
+	string_or_die(&filename, "%s/%i/%s/%s", image_dir, version, component, file->filename);

 	/* make sure file->stat.st_size is valid */
 	if (file->stat.st_size == 0) {
 		int ret;
+		struct stat buf;

-		if (filename == NULL) {
-			printf("filename is null...impossible to stat\n");
-			assert(0);
-		}
+		assert(filename);
 		ret = lstat(filename, &buf);
 		if (!ret) {
 			file->stat.st_size = buf.st_size;
@@ -182,27 +218,19 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i
 			printf("Stat failure on %s\n", filename);
 		}
 	}
-
-	c1 = (char *)magic_file(mcookie, filename);
-	if (c1) {
-		char *c2;
-		file->filetype = strdup(c1);
-		c2 = strstr(file->filetype, "not stripped");
-		if (c2) {
-			*c2 = 0;
-		}
-		c2 = strstr(file->filetype, "stripped");
-		if (c2) {
-			*c2 = 0;
-		}
+	if (file->stat.st_size < BSDIFFSIZE) {
+		/* thing is too small, always will be regenerated
+		 * so no point in trying to figure out what kind
+		 * of file it is
+		 */
 	} else {
-		LOG(file, "Cannot find file type", "%s", filename);
+		file->filetype = getmagic(filename);
 	}

 	free(filename);

-	file->basename = strdup(basename(file->filename));
-	file->dirname = strdup(dirname(file->filename));
+	file->basename = g_path_get_basename(file->filename);
+	file->dirname = g_path_get_dirname(file->filename);
 }

 int file_sort_score(gconstpointer a, gconstpointer b)
@@ -222,6 +250,10 @@ int file_sort_score(gconstpointer a, gconstpointer b)
 	return 0;
 }

+/* compare file to each deleted file.
+ * set file->rename_peer to best matched deleted file
+ * set file->rename_score to the score
+ */
 static void score_file(GList *deleted_files, struct file *file)
 {
 	GList *list2;
@@ -246,61 +278,107 @@ static void score_file(GList *deleted_files, struct file *file)
 	}
 }

-void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list)
+/* delete the first element of the list and return the new head */
+static GList *del_first(GList *list)
 {
-	GList *new_files = NULL;
-	GList *deleted_files = NULL;
+	/* The first list is the pointer to the list, the second is
+	 * the pointer to what to delete */
+	return g_list_delete_link(list, list);
+}
+
+/* Take a list, return a new list where the filter function returns true */
+static GList *new_filtered_list(GList *list, int version, int (*f)(struct file *file, int version))
+{
+	/* make a list of new files, no peer */
+	GList *newlist = NULL;
+	list = g_list_first(list);
+	for (; list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		if (f(file, version)) {
+			newlist = g_list_prepend(newlist, file);
+		}
+	}
+	return newlist;
+}
+
+static int renamed_file_p(struct file *file, int unused __attribute__((unused)))
+{
+	return file->is_rename;
+}
+/* Return a new list of renamed files */
+static GList *new_list_renamed_files(GList *infiles)
+{
+	return new_filtered_list(infiles, 0, renamed_file_p);
+}
+
+/* Predicate that returns true if this is a new file in the stated version */
+static int new_file_p(struct file *file, int version)
+{
+	if ((file->last_change != version) ||
+	    (file->is_deleted) ||
+	    (!file->is_file) ||
+	    (file->peer)) {
+		return 0;
+	}
+	return 1;
+}
+
+/* return a new list of the new files */
+static GList *list_new_files(struct manifest *manifest)
+{
+	GList *list = new_filtered_list(manifest->files, manifest->version, new_file_p);
+	/* call precompute_file_data for each file on list, return the list */
+	GList *ret = list;
+	for (list = g_list_first(list); list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		precompute_file_data(manifest->version, manifest->component, file);
+	}
+	return ret;
+}
+
+static int deleted_p(struct file *file, int version)
+{
+	if ((!file->is_deleted) ||
+	    (!file->peer) ||
+	    (file->last_change != version) ||
+	    (file->peer->is_dir || file->peer->is_link)) {
+		return 0;
+	}
+	return 1;
+}
+
+static GList *list_deleted_files(struct manifest *manifest)
+{
+	GList *list = new_filtered_list(manifest->files, manifest->version, deleted_p);
+	GList *ret = list;
+	/* call precompute_file_data for each peer of file on list */
+	for (list = g_list_first(list); list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		struct file *peer = file->peer;
+		/* Need to get things from the /full/ as we do not know
+		 * which  component may be coming from? */
+		precompute_file_data(peer->last_change, "full", peer);
+	}
+	return ret;
+}
+
+void rename_detection(struct manifest *manifest)
+{
+	GList *new_files;
+	GList *deleted_files;

 	GList *list;
 	struct file *file;
-	int old_rename = 0;

-	if (last_change != manifest->version) {
-		old_rename = 1;
-	}
+	new_files = list_new_files(manifest);

-	if (mcookie == NULL) {
-		mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS);
-		magic_load(mcookie, NULL);
-	}
-
-	/* make a list of new files, no peer */
-	list = g_list_first(manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
-		if ((file->last_change != manifest->version) ||
-		    (file->is_deleted) ||
-		    (!file->is_file) ||
-		    (file->peer)) {
-			continue;
-		}
-
-		new_files = g_list_prepend(new_files, file);
-		precompute_file_data(manifest, file, old_rename, last_versions_list);
-	}
-
-	/* if there are no new files, we're not having any renames -- early exit */
+	/* no new files --> no renames -- early exit */
 	if (!new_files) {
 		LOG(NULL, "No new files, no rename detection", "%s", manifest->component);
 		return;
 	}
-	/* make a list of newly deleted files that have a peer */

-	list = g_list_first(manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
-		if ((!file->is_deleted) ||
-		    (!file->peer) ||
-		    (file->last_change != last_change) ||
-		    (file->peer->is_dir || file->peer->is_link)) {
-			continue;
-		}
-
-		deleted_files = g_list_prepend(deleted_files, file);
-		precompute_file_data(NULL, file->peer, old_rename, last_versions_list);
-	}
+	deleted_files = list_deleted_files(manifest);

 	/* nothing got deleted --> no renames --> early exit */
 	if (!deleted_files) {
@@ -309,7 +387,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		return;
 	}

-	/* for each new file, find the deleted file with the highest score, and store the score */
+	/* for each new file, find the deleted file with the highest score,
+	 * store it in file->rename_peer and store the score */
 	list = g_list_first(new_files);
 	while (list) {
 		file = list->data;
@@ -317,20 +396,24 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		score_file(deleted_files, file);
 	}

+redo:
 	/* sort all new files by score */
-
+	/* walk the sorted score list.
+	 * pick the top score,
+	 * check if the score is still valid,
+	 * if not, recompute the score and resort
+	 * This is probably an O(n^3).
+	 */
 	new_files = g_list_sort(new_files, file_sort_score);

-	/* pick the top score, check if the score is still valid, if not, recompute the score and resort */
-
-	while (new_files) {
+	for (; new_files; new_files = del_first(new_files)) {
 		file = new_files->data;
+		if (file->rename_peer == NULL) {
+			continue;
+		}

-		if (file->rename_score < 15.0 || file->rename_peer == NULL) {
-			new_files = g_list_delete_link(new_files, new_files);
-			if (file->rename_peer) {
-				LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
-			}
+		if (file->rename_score < 15.0) {
+			LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
 			continue;
 		}

@@ -338,10 +421,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 			/* the candidate peer got already taken by another file! */
 			LOG(NULL, "Rename not done due to target already taken", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
 			file->rename_peer = NULL;
-			file->rename_score = -100;
 			score_file(deleted_files, file);
-			new_files = g_list_sort(new_files, file_sort_score);
-			continue;
+			goto redo;
 		}

 		/* if valid and score is high enough, make the link by setting the flag and storing the hash */
@@ -349,13 +430,14 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		LOG(NULL, "Rename detected!", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);

 		file->rename_peer->rename_peer = file;
-		/* must delete the file from the deleted list */
+		/* must remove the file from the deleted list */
 		deleted_files = g_list_remove(deleted_files, file->rename_peer);
 		hash_assign(file->hash, file->rename_peer->hash);
 		file->is_rename = 1;
 		file->rename_peer->is_rename = 1;
-
-		new_files = g_list_delete_link(new_files, new_files);
+		if (!deleted_files) {
+			break; /* No more deleted files to rename */
+		}

 	} /* lather, rinse, repeat until all files have a target */

@@ -364,65 +446,89 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 	g_list_free(deleted_files);
 }

-static int file_found_in_older_manifest(struct manifest *from_manifest, struct file *searched_file)
+void final_link(GList *files)
 {
-	GList *list;
-	struct file *file;
+	GList *list1, *list2;
+	struct file *file1, *file2;

-	list = g_list_first(from_manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
+	list1 = new_list_renamed_files(files);
+	list1 = list2 = g_list_sort(list1, file_sort_version);

-		if (file->is_deleted) {
+	for (; list1; list1 = g_list_next(list1)) {
+		file1 = list1->data;
+		/* do not re-link already-linked renames */
+		if (file1->rename_peer != NULL) {
 			continue;
 		}
-		if (!strcmp(file->filename, searched_file->filename)) {
-			return 1;
+
+		/* check the rest of the list */
+		list2 = g_list_next(list1);
+		for (; list2; list2 = g_list_next(list2)) {
+			file2 = list2->data;
+			/* must be the same version to link */
+			if (file1->last_change != file2->last_change) {
+				continue;
+			}
+
+			/* need one and only one to be deleted to link as this
+			 * indicates a renamed-from/renamed-to relationship
+			 * (.d.r -> F..r) */
+			if (file1->is_deleted == file2->is_deleted) {
+				continue;
+			}
+
+			/* must have same hash to link */
+			if (!hash_compare(file2->hash, file1->hash)) {
+				continue;
+			}
+
+			file1->rename_peer = file2;
+			file2->rename_peer = file1;
 		}
 	}
-
-	return 0;
 }

-void link_renames(GList *newfiles, struct manifest *from_manifest)
+/* What do we need this for?
+ *
+ * rename_detection has already set up the links in the manifest it
+ * was given.
+ *
+ */
+void link_renames(GList *newfiles, int to_version)
 {
 	GList *list1, *list2;
 	GList *targets;
 	struct file *file1, *file2;

-	targets = newfiles = g_list_sort(newfiles, file_sort_version);
+	targets = new_list_renamed_files(newfiles);
+	/* TODO: Check that g_list_sort is reasonable speed */
+	targets = newfiles = g_list_sort(targets, file_sort_version);

-	list1 = g_list_first(newfiles);
-
-	/* todo: sort newfiles and targets by hash */
-
-	while (list1) {
+	for (list1 = newfiles; list1; list1 = g_list_next(list1)) {
 		file1 = list1->data;
-		list1 = g_list_next(list1);

-		if ((file1->peer || !file1->is_rename) ||
-		    (file1->is_deleted)) {
+		if (file1->peer || file1->is_deleted) {
 			continue;
 		}
-		/* now, file1 is the new file that got renamed. time to search the rename targets */
+		/* now, file1 is the new file that got renamed.
+		 * time to search the rename targets */
 		list2 = g_list_first(targets);
-		while (list2) {
+		for (; list2; list2 = g_list_next(list2)) {
 			file2 = list2->data;
-			list2 = g_list_next(list2);
-
-			if ((!file2->peer || !file2->is_rename) ||
+			/* This is like deleted_p but not quite */
+			/* deleted_p returns false for directories and links */
+			if (!file2->peer ||
 			    (!file2->is_deleted) ||
-			    (!file_found_in_older_manifest(from_manifest, file2))) {
+			    (file2->last_change != to_version)) {
 				continue;
 			}
 			if (hash_compare(file2->hash, file1->hash)) {
 				file1->rename_peer = file2->peer;
 				file1->peer = file2->peer;
 				file2->peer->rename_peer = file1;
-				list2 = NULL;
+				break;
 			}
 		}
 	}
-	free(from_manifest);
+	g_list_free(targets);
 }
@@ -1,173 +0,0 @@
-/*
- *   Software Updater - server side
- *
- *      Copyright © 2012-2016 Intel Corporation.
- *
- *   This program is free software: you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, version 2 or later of the License.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *   Authors:
- *         Tom Keel <thomas.keel@intel.com>
- *
- */
-
-#define _GNU_SOURCE
-#include <err.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "swupd.h"
-
-static char *make_filename(const char *, const char *, const char *);
-
-static char *leaf_key = NULL;
-static char *leaf_cert = NULL;
-static char *ca_chain_cert = NULL;
-static char *passphrase = NULL;
-
-static bool initialized = false;
-
-/*
- * Initialize this module.
- * @return true <=> success
- */
-bool signature_initialize(void)
-{
-	if (!enable_signing) {
-		return true;
-	}
-
-	char *cdir;
-	char *pphr;
-	struct stat s;
-
-	if (initialized) {
-		return true;
-	}
-	cdir = getenv("SWUPD_CERTS_DIR");
-	if (cdir == NULL || cdir[0] == '\0') {
-		printf("No certificates directory specified\n");
-		goto err;
-	}
-	if (stat(cdir, &s)) {
-		printf("Can't stat certificates directory '%s' (%s)\n", cdir,
-		       strerror(errno));
-		goto err;
-	}
-	leaf_key = make_filename(cdir, "LEAF_KEY", "leaf key");
-	if (leaf_key == NULL) {
-		goto err;
-	}
-	leaf_cert = make_filename(cdir, "LEAF_CERT", "leaf certificate");
-	if (leaf_cert == NULL) {
-		goto err;
-	}
-	ca_chain_cert = make_filename(cdir, "CA_CHAIN_CERT", "CA chain certificate");
-	if (ca_chain_cert == NULL) {
-		goto err;
-	}
-	pphr = getenv("PASSPHRASE");
-	if (pphr == NULL || (passphrase = strdup(pphr)) == NULL) {
-		goto err;
-	}
-	if (stat(passphrase, &s)) {
-		printf("Can't stat '%s' (%s)\n", passphrase,
-		       strerror(errno));
-		goto err;
-	}
-	initialized = true;
-	return true;
-err:
-	signature_terminate();
-	return false;
-}
-
-/* Make filename from dir name and env variable containing basename */
-static char *make_filename(const char *dir, const char *env, const char *desc)
-{
-	char *fn = getenv(env);
-	char *result = NULL;
-	struct stat s;
-
-	if (fn == NULL || fn[0] == '\0') {
-		printf("No %s file specified\n", desc);
-		return NULL;
-	}
-	string_or_die(&result, "%s/%s", dir, fn);
-	if (stat(result, &s)) {
-		printf("Can't stat %s '%s' (%s)\n", desc, result, strerror(errno));
-		free(result);
-		return NULL;
-	}
-	return result;
-}
-
-/*
- * Terminate this module, free resources.
- */
-void signature_terminate(void)
-{
-	if (!enable_signing) {
-		return;
-	}
-
-	free(leaf_key);
-	free(leaf_cert);
-	free(ca_chain_cert);
-	free(passphrase);
-
-	leaf_key = NULL;
-	leaf_cert = NULL;
-	ca_chain_cert = NULL;
-	passphrase = NULL;
-
-	initialized = false;
-}
-
-/*
- * Write the signature file corresponding to the given data file.
- * The name of the signature file is the name of the data file with suffix
- * ".signed" appended.
- */
-bool signature_sign(const char *filename)
-{
-	char *param1, *param2;
-	int status;
-
-	if (!enable_signing) {
-		return true;
-	}
-
-	if (!initialized) {
-		return false;
-	}
-	string_or_die(&param1, "%s.signed", filename);
-	string_or_die(&param2, "file:%s", passphrase);
-	char *const opensslcmd[] = { "openssl", "smime", "-sign", "-in", (char *)filename, "-binary",
-				     "-out", param1, "-outform", " PEM", "-md", "sha256", "-inkey",
-				     leaf_key, "-signer", leaf_cert, "-certfile", ca_chain_cert,
-				     "-passin", param2, NULL };
-	status = system_argv(opensslcmd);
-	if (status != 0) {
-		printf("Bad status %d from signing command\n", status);
-	}
-	free(param1);
-	free(param2);
-
-	return status == 0;
-}
@@ -235,8 +235,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 	DIR *dir;
 	GList *list = NULL;
 	GList *cur_item, *next_item;
-	struct dirent entry;
-	struct dirent *result;
+	struct dirent *entry;
 	struct stat stat;
 	char *filename = NULL;
 	int idx, build_num, build_type, jump_point;
@@ -249,13 +248,13 @@ GList *get_last_versions_list(int next_version, int max_versions)
 		return NULL;
 	}

-	while (readdir_r(dir, &entry, &result) == 0 && result != NULL) {
-		if (strspn(entry.d_name, "0123456789") != strlen(entry.d_name)) {
+	while ((entry = readdir(dir))) {
+		if (strspn(entry->d_name, "0123456789") != strlen(entry->d_name)) {
 			continue;
 		}

 		free(filename);
-		string_or_die(&filename, "%s/%s", staging_dir, entry.d_name);
+		string_or_die(&filename, "%s/%s", staging_dir, entry->d_name);

 		if (lstat(filename, &stat)) {
 			LOG(NULL, "lstat failed", "path= %s, strerror= %s",
@@ -267,11 +266,11 @@ GList *get_last_versions_list(int next_version, int max_versions)
 			continue;
 		}

-		if (atoi(entry.d_name) >= next_version) {
+		if (atoi(entry->d_name) >= next_version) {
 			continue;
 		}

-		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry.d_name)));
+		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry->d_name)));
 	}
 	free(filename);
 	closedir(dir);
@@ -60,8 +60,7 @@ static int xattr_get_value(const char *path, const char *name, char **blob,

 	/* realloc needed len + 1 in case we need to add final zero
 	 * to ensure consistent blob */
-	value = realloc(*blob, *blob_len + len +
-				   (action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
+	value = realloc(*blob, *blob_len + len + (action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
 	assert(value);

 	*blob = value;
@@ -0,0 +1,64 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+  gen_file_plain 10 test-bundle1 foo
+  gen_file_plain 10 test-bundle1 foobar
+  gen_file_plain 10 test-bundle2 foo2
+  gen_includes_file test-bundle2 10 test-bundle1
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+  gen_file_plain 20 test-bundle1 foo
+  gen_file_plain 20 test-bundle1 foobar
+  gen_file_plain 20 test-bundle1 foobarbaz
+  gen_file_plain 20 test-bundle2 foo2
+  gen_file_plain 20 test-bundle2 foo2bar
+  gen_includes_file test-bundle2 20 test-bundle1
+}
+
+@test "correct contentsize" {
+  # create a couple updates to both check that contentsize does not add included
+  # bundles and to verify that files changed in previous updates are counted.
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # contentsize for test-bundle2 should not include test-bundle1's contentsize
+  [[ 1 -eq $(grep '^contentsize:	11$' $DIR/www/10/Manifest.test-bundle1 | wc -l) ]]
+  [[ 1 -eq $(grep '^contentsize:	5$'  $DIR/www/10/Manifest.test-bundle2 | wc -l) ]]
+  # os-core is large because it includes /usr/*
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # 5134 + 11 + 5 = 5150
+  [[ 1 -eq $(grep '^contentsize:	5150$' $DIR/www/10/Manifest.full | wc -l) ]]
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  set_latest_ver 20
+
+  # one new file: foobarbaz (10 bytes)
+  [[ 1 -eq $(grep '^contentsize:	21$' $DIR/www/20/Manifest.test-bundle1 | wc -l) ]]
+  # one new file: foo2bar (8 bytes)
+  [[ 1 -eq $(grep '^contentsize:	13$'  $DIR/www/20/Manifest.test-bundle2 | wc -l) ]]
+  # os-core should not change size
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # contentsize for full should be all files, including ones not changed in this release
+  # two new files: foo2bar (8 bytes) and foobarbaz (10 bytes)
+  # 5150 + 10 + 8  = 5168
+  # 5134 + 21 + 13 = 5168
+  [[ 1 -eq $(grep '^contentsize:	5168$' $DIR/www/20/Manifest.full | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,56 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  # unban debuginfo
+  sed -i "s|banned=true|banned=false|" $DIR/server.ini
+  set_latest_ver 0
+  init_groups_ini os-core
+  init_groups_ini test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 test-bundle
+
+  gen_file_plain 10 test-bundle "/usr/lib/debug/foo"
+  gen_file_plain 10 test-bundle "/usr/src/debug/bar"
+  gen_file_plain 10 test-bundle "/usr/bin/foobar"
+
+  track_bundle 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 test-bundle
+
+  gen_file_plain 20 test-bundle "/usr/foo"
+}
+
+@test "debuginfo files pruned" {
+  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
+  echo "$output"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/bin/foobar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/src/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  # These should not be pruned right now, but should be pruned with the next
+  # update after being banned in server.ini again
+  [[ 1 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+
+  # grab the server.ini that bans debuginfo again
+  init_server_ini
+  set_os_release 20 os-core
+  run sudo sh -c "$CREATE_UPDATE --osversion 20 --statedir $DIR --format 3"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  # These should be pruned
+  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+}
+
+@test "full run update creation with delta packs over format bump" {
+  # build the first version
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+
+  set_latest_ver 10
+
+  # then the second version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+
+  set_latest_ver 20
+
+  # then the third version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 4
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  sudo $MAKE_PACK --statedir $DIR 0 30 os-core
+
+  # zero packs should exist (non-zero size) for all versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/30/pack-os-core-from-0.tar ]
+
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/10/Manifest.MoM | wc -l) ]]
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/20/Manifest.MoM | wc -l) ]]
+  [[ 1 -eq $(grep '^actions:	update$' $DIR/www/30/Manifest.MoM | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,35 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  gen_file_plain 10 os-core foo
+  gen_file_plain 10 os-core bar
+  gen_file_plain 20 os-core foo
+  gen_file_plain 20 os-core baz
+}
+
+@test "ensure format numbers cannot be decremented" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # Now, decrement the format number. Should result in an EXIT_FAILURE.
+  run sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 2
+  echo "$output"
+  [ $status -eq 1 ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -21,7 +21,7 @@ setup() {
  track_bundle 20 included-two
  track_bundle 20 included-nested

-  gen_file_to_delta 10 4096 20 4 test-bundle
+  gen_file_to_delta 10 4096 20 4 test-bundle randomfile

  gen_file_plain 10 test-bundle foo
  gen_file_plain 10 test-bundle foobarbaz
@@ -40,6 +40,12 @@ setup() {
  [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
  [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
  [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+
+  # extract test-bundle pack to make sure staged and delta are created with the
+  # correct permissions
+  sudo tar -xf $DIR/www/10/pack-test-bundle-from-0.tar --directory $DIR/www/10/
+  [[ $(stat -c %a $DIR/www/10/staged) -eq 700 ]]
+  [[ $(stat -c %a $DIR/www/10/delta) -eq 700 ]]
 }

 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -14,6 +14,7 @@ load "../swupdlib"
 }

@test "make_fullfiles root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
  run $MAKE_FULLFILES foo
  [ "$status" -eq 1 ]
  [[ "$output" =~ "not being run as root.. exiting" ]]
@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  gen_file_plain_with_content 10 test-bundle usr/lib/kernel/bar testfile_contents
+  gen_file_plain_with_content 20 test-bundle usr/lib/kernel/baz new_testfile_contents
+  # make sure /usr/lib/kernel stays around
+  mkdir $DIR/image/30/test-bundle/usr/lib/kernel
+}
+
+@test "create updates while ghosting boot files before removing them" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  # version 10: add boot file to 10
+  [ 1 -eq $(grep 'F\.b\.	.*	10	/usr/lib/kernel/bar' $DIR/www/10/Manifest.full | wc -l) ]
+  # version 20: ghost boot file instead of deleting it when it doesn't exist
+  [ 1 -eq $(grep '\.gb\.	.*	20	/usr/lib/kernel/bar' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 20: add a new boot file
+  [ 1 -eq $(grep 'F\.b\.	.*	20	/usr/lib/kernel/baz' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 30: old ghosted file /usr/lib/kernel/bar cleaned up
+  [ 0 -eq $(grep '10	/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
+  # version 30: boot file added in version 20 ghosted
+  [ 1 -eq $(grep '\.gb\.	.*	30	/usr/lib/kernel/baz' $DIR/www/30/Manifest.full | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,42 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+
+  gen_file_plain 10 os-core test0
+  gen_file_plain 10 test-bundle1 test1
+  gen_file_plain 10 test-bundle2 test2
+
+  gen_includes_file test-bundle2 10 test-bundle1 test-bundle1
+  gen_includes_file test-bundle2 20 test-bundle1 test-bundle1
+}
+
+@test "deduplicate bundle includes" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+
+  # includes list should be deduplicated in both the old and new manifests
+  [[ 1 -eq $(grep '^includes:	test-bundle1$' $DIR/www/10/Manifest.test-bundle2 | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	test-bundle1$' $DIR/www/20/Manifest.test-bundle2 | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,73 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  # symlink -> regular file type change (L -> F)
+  gen_file_to_delta 10 4096 20 4 os-core testfile1
+  gen_symlink_to_file 10 os-core testsym1 testfile1
+  copy_file 10 os-core testfile1 20 os-core testsym1
+
+  # regular file -> symlink type change (F -> L)
+  gen_file_to_delta 10 4096 20 4 os-core testfile2
+  copy_file 10 os-core testfile2 10 os-core testsym2
+  gen_symlink_to_file 20 os-core testsym2 testfile2
+
+  # symlink change + symlink target change; delta should be created for
+  # testfile3, but not for the dereferenced testsym3
+  gen_file_to_delta 10 4096 20 4 os-core testfile3
+  copy_file 20 os-core testfile3 20 os-core testfile4
+  gen_symlink_to_file 10 os-core testsym3 testfile3
+  gen_symlink_to_file 20 os-core testsym3 testfile4
+}
+
+@test "no deltas created for type changes or dereferenced symlinks" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+
+  # attempt to create some deltas
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+
+  # F -> F deltas should exist
+  hash1=$(hash_for 10 os-core "/testfile1")
+  hash2=$(hash_for 20 os-core "/testfile1")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testfile2")
+  hash2=$(hash_for 20 os-core "/testfile2")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testfile3")
+  hash2=$(hash_for 20 os-core "/testfile3")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  # deltas for symlink type changes should not be created
+  hash1=$(hash_for 10 os-core "/testsym1")
+  hash2=$(hash_for 20 os-core "/testsym1")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testsym2")
+  hash2=$(hash_for 20 os-core "/testsym2")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testsym3")
+  hash2=$(hash_for 20 os-core "/testsym3")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+}
+
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,95 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  set_os_release 40 os-core
+  set_os_release 40 test-bundle
+  track_bundle 40 os-core
+  track_bundle 40 test-bundle
+
+  # /usr/lib/bar and /one will be renamed to /usr/lib/baz and /two
+  gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
+  gen_file_plain_with_content 10 test-bundle /one "$(printf 'a%.0s' {1..200})"
+
+  gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
+  gen_file_plain_with_content 20 test-bundle /two "$(printf 'a%.0s' {1..200})"
+
+  # different content just to make sure this works with delta renames as well as
+  # direct renames
+  gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
+
+  gen_file_plain_with_content 40 test-bundle /usr/lib/foo "$(seq 100) new"
+  # new file to force manifest generation
+  gen_file_plain_with_content 40 test-bundle /a "testfile"
+}
+
+@test "create updates with renamed-to file getting deleted" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+
+  set_latest_ver 30
+
+  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 40
+  # version 10: add files to 10
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/one' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  # version 20: rename bar to baz and one to two
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/one' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/two' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  # version 30: original renamed-from file (bar) is now orphaned and therefore
+  # deleted, baz is now a renamed-from file and foo is a renamed-to file.
+  # /two was deleted in this version, so both /one and /two should be marked as
+  # deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+
+  # version 40: the existing rename from baz -> foo must persist while all
+  # others remain deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -20,6 +20,7 @@ load "../swupdlib"
 }

@test "make_pack root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
  run $MAKE_PACK foo bar foo
  [ "$status" -eq 1 ]
  [[ "$output" =~ "not being run as root.. exiting" ]]
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export BATS_TEST_DIRNAME=$PWD
+export TIMEFORMAT
+TIMEFORMAT="time %R %P%% (%U+%S)"
+source ../swupdlib.bash
+exec 3< rundata
+
+while read -r -u 3 
+do
+    read -i "$REPLY" -e CMD
+    eval "set -x ; time ( $CMD ) ; set +x "
+done
@@ -0,0 +1,22 @@
+sudo rm -r logs/ web-dir/
+tar xf ~/data/test.tar 
+init_test_dir
+init_server_ini
+set_latest-ver 0
+init_groups_ini os-core test-bundle
+set_os_release 10 os-core
+track_bundle 10 os-core
+track_bundle 10 test-bundle
+set_os_release 20 os-core
+track_bundle 20 os-core
+track_bundle 20 test-bundle
+sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 10
+sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+time  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+set_latest-ver 10
+sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 20
+sudo $MAKE_PACK --statedir $DIR  10 20  os-core
+sudo $MAKE_PACK --statedir $DIR  10 20  test-bundle
@@ -0,0 +1,246 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  maybeskip
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+}
+
+# Generate data files of 3 types in test-bundle
+# All files are big enough that they get rename detection
+gendataA() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 100)"
+}
+gendataB() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 50) $(seq 52 101)"
+}
+gendataC() {
+  # cache string
+  [ -z "$dataC" ] || dataC="$(seq 1000 | gzip | uuencode wombat)"
+  gen_file_plain_with_content "$1" test-bundle "$2" "$dataC"
+}
+# Generate small data files
+gendataAs() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 50)"
+}
+gendataBs() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 24) $(seq 26 49)"
+}
+gendataCs() {
+  # cache string
+  [ -z "$dataCs" ] || dataC="$(seq 50 | gzip | uuencode wombat)"
+  gen_file_plain_with_content "$1" test-bundle "$2" "$dataCs"
+}
+
+checkrenamed(){
+    local flags sh1 ver name fromsha1="bad" tosha1
+    # Check that $1 is renamed to $2
+    exec 9< $DIR/www/20/Manifest.test-bundle
+    # skip the header
+    while read -u9
+    do
+	[ -z "$REPLY" ] && break
+    done
+    while read -r -u9 flags sha1 ver name
+    do
+	case "$flags" in
+	    (?"dr"?) [ "$name" = "$1" ] && fromsha1=$sha1 ;;
+	    (?".r"?) [ "$name" = "$2" ] && tosha1=$sha1 ;;
+	esac
+    done
+    if [ "$fromsh1" = "$tosha1" ] ; then return 0 ; else return 1 ; fi
+}
+
+# Guts of doing an update
+do_an_update() {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+}
+
+
+@test "basic rename detection support" {
+  gendataA 10 foo
+  gendataA 20 bar
+  do_an_update
+  checkrenamed foo bar
+}
+
+@test "ignore rename detection for small files" {
+  gendataAs 10 foo
+  gendataAs 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "ignore rename detection for large to small files" {
+  gendataA 10 foo
+  gendataAs 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "ignore rename detection for small to large files" {
+  gendataAs 10 foo
+  gendataA 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+@test "rename one file to two" {
+  gendataA 10 foo
+  gendataA 20 bar
+  gendataA 20 baz
+  do_an_update  
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two file to one" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 baz
+  do_an_update  
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/baz$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.\..*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two files to two" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataA 20 baz  
+  do_an_update
+  checkrenamed foo bar
+  checkrenamed foz baz
+}
+
+@test "rename two files to two, one slightly different" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataB 20 baz  
+  do_an_update
+  # we don't actually know how the client we do this rename, but don't care
+  checkrenamed foo bar
+  checkrenamed foz baz
+}
+
+@test "rename two files to two, each pair slightly different" {
+  gendataA 10 foo
+  gendataB 10 foz
+  gendataA 20 bar
+  gendataB 20 baz
+  do_an_update
+  checkrenamed foo bar
+  checkrenamed foz baz
+}
+
+@test "rename two files to two, one very different" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataC 20 baz  
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two files to two, one small" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataCs 20 baz  
+  do_an_update
+  run checkrenamed foo bar
+  if [ $status -eq 1 ] ; then
+    checkrenamed foz bar
+  fi
+}
+
+@test "directory name changes" {
+  gendataA 10 dir1/foo
+  gendataA 20 dir2/foo
+  do_an_update
+  checkrenamed /dir1/foo /dir2/foo
+}
+
+@test "directory name changes small files" {
+  gendataAs 10 dir1/foo
+  gendataAs 20 dir2/foo
+  do_an_update
+  if ! checkrenamed /dir1/foo /dir2/foo ; then false ; fi
+}
+
+@test "directory name and small data changes" {
+  gendataA 10 dir1/foo
+  gendataB 20 dir2/foz
+  do_an_update
+  checkrenamed /dir1/foo /dir2/foz
+}
+
+@test "directory name and small data changes, choose same name" {
+  gendataA 10 dir1/foo
+  gendataA 10 dir1/foz
+  gendataB 20 dir2/foz
+  do_an_update
+  checkrenamed /dir1/foz /dir2/foz
+}
+
+@test "same basename test" {
+    gendataA 10 dir1/foo.so.1
+    gendataA 10 dir1/foz.so.1
+    gendataB 20 dir2/foo.so.2
+    do_an_update
+    checkrenamed /dir1/foo.so.1 /dir2/foo.so.2
+}
+
+@test "rename file to dir/file" {
+    gendataA 10 foo
+    gendataA 20 foo/bar
+    do_an_update
+    checkrenamed /foo /foo/bar
+}
+# @test "rename foo/foo to foo" {
+#     gendataA 10 foo/foo
+#     gendataA 20 foo
+#     do_an_update
+#     checkrenamed /foo/foo /foo
+# }
+
+# Emacs and vi support
+# Local variables:
+# sh-indentation: 2
+# End:
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export BATS_TEST_DIRNAME=$PWD
+export TIMEFORMAT
+TIMEFORMAT="time %R %P%% (%U+%S)"
+source ../swupdlib.bash
+exec 3< rundata
+
+while read -r -u 3 
+do
+    read -i "$REPLY" -e CMD
+    eval "set -x ; time ( $CMD ) ; set +x "
+done
@@ -0,0 +1,22 @@
+sudo rm -r logs/ web-dir/
+tar xf ~/data/test.tar 
+init_test_dir
+init_server_ini
+set_latest_ver 0
+init_groups_ini os-core test-bundle
+set_os_release 10 os-core
+track_bundle 10 os-core
+track_bundle 10 test-bundle
+set_os_release 20 os-core
+track_bundle 20 os-core
+track_bundle 20 test-bundle
+sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 10
+sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+time  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+set_latest_ver 10
+sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 20
+sudo $MAKE_PACK --statedir $DIR  10 20  os-core
+sudo $MAKE_PACK --statedir $DIR  10 20  test-bundle
@@ -0,0 +1,144 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+  tar x -C $DIR/.. -f ~/data/test.tar 
+  mv $DIR/image/20/test-bundle/usr/share/bash-completion/completions{,.old}
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  # Files have different names ("foo" vs "bar"), but have the same content
+  gen_file_plain_with_content 10 test-bundle "foo" "data"
+  gen_file_plain_with_content 20 test-bundle "bar" "data"
+}
+
+@test "rename detection support" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+# @test "rename detection support 1 to 2" {
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "baz" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^F\.\.\..*/ba[rz]$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle) ]]
+# }
+
+# @test "rename detection support 2 to 2" {
+#   gen_file_plain_with_content 10 test-bundle "fee" "data"
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo~" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/foo~$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 2 -eq $(grep -c '^\.d\.r.*/f..$' $DIR/www/20/Manifest.test-bundle) ]]
+# }
+
+# @test "rename detection support binary files" {
+#   sudo mkdir -p  $DIR/image/10/test-bundle/bin/
+#   sudo cp /bin/gcc-ar $DIR/image/10/test-bundle/bin/gcc-ar.1
+#   sudo cp /bin/gcc-ar $DIR/image/20/test-bundle/gcc-ar.1
+#   strip $DIR/image/20/test-bundle/gcc-ar.1
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo~" "And now not the same"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+# }
+
+# @test "rename detection support binary files 2" {
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo" "And now not the same"
+#   gen_file_plain_with_content 20 test-bundle "foo.1" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+# }
+
+# # vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  gen_file_plain 10 os-core "/var/lib/test"
+}
+
+@test "state file marked in manifest" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+
+  # a file and dir installed in /var should be marked state
+  grep '^D\.s\..*/var/lib$' $DIR/www/10/Manifest.os-core
+  grep '^F\.s\..*/var/lib/test$' $DIR/www/10/Manifest.os-core
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -0,0 +1,56 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  # start with both bundles containing a file "foo"
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  gen_file_plain 10 os-core foo
+  gen_file_plain 10 test-bundle foo
+
+  # delete "foo" from os-core (the included bundle)
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  gen_file_plain 20 test-bundle foo
+
+  # delete "foo" from test-bundle
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  # make modification (add new file) to test-bundle
+  set_os_release 40 os-core
+  track_bundle 40 os-core
+  track_bundle 40 test-bundle
+  gen_file_plain 40 test-bundle foobar
+}
+
+@test "no subtraction for two deleted files" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  set_latest_ver 20
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  set_latest_ver 30
+  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
+  set_latest_ver 40
+
+  # If the file is absent in test-bundle, this means it was subtracted. If
+  # present, subtraction was not performed.
+  hash1=$(hash_for 40 test-bundle "/foo")
+  [ -n "$hash1" ]
+}
+
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -40,6 +40,11 @@ EOF
  done
 }

+# If the variable RUN_JUST_ONE is set then only run that test
+maybeskip() {
+    [ -z "$RUN_JUST_ONE" ] || [ "$RUN_JUST_ONE" -eq "$BATS_TEST_NUMBER" ] || skip
+}
+
 set_os_release() {
  local ver=$1
  local bundle=$2
@@ -72,16 +77,17 @@ gen_file_to_delta() {
  local newver=$3
  local newbytes=$4
  local bundle=$5
+  local name=$6

  # create some random data for the original version
  mkdir -p $DIR/image/$origver/$bundle
-  dd if=/dev/urandom of=$DIR/image/$origver/$bundle/randomfile bs=1 count=$origsize
+  dd if=/dev/urandom of=$DIR/image/$origver/$bundle/$name bs=1 count=$origsize

  # append more random data to the end of the file in the new version
  TMP=$(mktemp foo.XXXXXX)
  mkdir -p $DIR/image/$newver/$bundle
  dd if=/dev/urandom of=$TMP bs=1 count=$newbytes
-  cat $DIR/image/$origver/$bundle/randomfile $TMP > $DIR/image/$newver/$bundle/randomfile
+  cat $DIR/image/$origver/$bundle/$name $TMP > $DIR/image/$newver/$bundle/$name
  rm $TMP
 }

@@ -91,7 +97,10 @@ gen_file_plain() {
  local name="$3"

  # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
  echo "$name" > $DIR/image/$ver/$bundle/"$name"
 }

@@ -101,8 +110,54 @@ gen_file_plain_change() {
  local name="$3"

  # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
  echo "$ver $name" > $DIR/image/$ver/$bundle/"$name"
 }

+gen_symlink_to_file() {
+  local ver=$1
+  local bundle=$2
+  local symname="$3"
+  local filename="$4"
+
+  mkdir -p $DIR/image/$ver/$bundle/$(dirname "$symname")
+  ln -s "$filename" $DIR/image/$ver/$bundle/"$symname"
+}
+
+copy_file() {
+  local origver=$1
+  local origbundle=$2
+  local origname="$3"
+  local newver=$4
+  local newbundle=$5
+  local newname="$6"
+
+  mkdir -p $DIR/image/$newver/$newbundle/$(dirname "$newname")
+  cp -a $DIR/image/$origver/$origbundle/"$origname" $DIR/image/$newver/$newbundle/"$newname"
+}
+
+# Gets the hash for file NAME in BUNDLE manifest for VER
+hash_for() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+
+  awk -F'\t' -v NAME="$name" 'NF == 4 && $4 == NAME { print $2 }' $DIR/www/$ver/Manifest.$bundle
+}
+
+gen_file_plain_with_content() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+  local content="$4"
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
+  echo "$content" > $DIR/image/$ver/$bundle/"$name"
+}
+
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80
@@ -15,6 +15,7 @@ load "../swupdlib"
 }

@test "create_update root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
  run $CREATE_UPDATE -F 3 -o 10
  [ "$status" -eq 1 ]
  [[ "$output" =~ "not being run as root.. exiting" ]]