Call checkrenamed() with correct arguments

Some of the renames tests omitted the leading forward slash for the two
function arguments, so add them here.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

.travis.yml

@@ -0,0 +1,25 @@
+sudo: required
+dist: trusty
+language: c
+
+# Pre-install missing build dependencies:
+# - libmagic (pull from repo)
+# - libcheck 0.9.10 is slightly too old, since 0.9.12 adds TAP support
+# - bsdiff 1.* is the Clear Linux OS fork
+before_install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y libmagic-dev
+
+install:
+        - wget https://github.com/libcheck/check/releases/download/0.11.0/check-0.11.0.tar.gz
+        - tar -xvf check-0.11.0.tar.gz
+        - pushd check-0.11.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
+        - tar -xvf bsdiff-1.0.2.tar.xz
+        - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd
+
+# Ubuntu's default umask is 0002, but this break's swupd hash calculations.
+script:
+        - sudo find test/functional -exec chmod g-w {} \;
+        - autoreconf --verbose --warnings=none --install --force && ./configure --enable-rename-detection && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
+after_failure: cat test-suite.log

Makefile.am

@@ -26,7 +26,6 @@ swupd_create_update_SOURCES = \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/type_change.c \
 	src/versions.c \
@@ -37,13 +36,13 @@ swupd_make_pack_SOURCES = \
 	src/config.c \
 	src/delta.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_packs.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c
 
@@ -53,13 +52,13 @@ swupd_make_fullfiles_SOURCES = \
 	src/delta.c \
 	src/fullfiles.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_fullfiles.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c
 
@@ -109,13 +108,29 @@ TESTS = $(dist_check_SCRIPTS)
 
 dist_check_SCRIPTS = \
 	test/functional/basic/test.bats \
+	test/functional/contentsize-across-versions-includes/test.bats \
 	test/functional/delete-no-version-bump/test.bats \
-	test/functional/include-version-bump/test.bats \
-	test/functional/update/test.bats \
-	test/functional/fullfiles/test.bats \
-	test/functional/pack/test.bats \
+	test/functional/file-name-blacklisted/test.bats \
+	test/functional/file-name-debuginfo/test.bats \
+	test/functional/format-no-decrement/test.bats \
+	test/functional/full-run-delta/test.bats \
 	test/functional/full-run/test.bats \
-	test/functional/full-run-delta/test.bats
+	test/functional/fullfiles/test.bats \
+	test/functional/ghosting/test.bats \
+	test/functional/include-version-bump/test.bats \
+	test/functional/includes-deduplicate/test.bats \
+	test/functional/no-delta/test.bats \
+	test/functional/pack/test.bats \
+	test/functional/state-file/test.bats \
+	test/functional/subtract-delete/test.bats \
+	test/functional/update/test.bats \
+	test/functional/format-bump/test.bats
+
+if RENAMES
+dist_check_SCRIPTS += \
+	test/functional/orphaned-renames/test.bats \
+	test/functional/renames/test.bats
+endif
 endif
 
 if COVERAGE
@@ -133,6 +148,27 @@ endif
 distclean-local:
 	rm -rf aclocal.m4 autogen.sh autom4te.cache config.h.in config.h.in~ configure depcomp install-sh Makefile.in missing compile
 
+compliant:
+	@git diff --quiet --exit-code include src; ret=$$?; \
+	if [ $$ret -eq 1 ]; then \
+		echo "Error: can only check code style when include/ and src/ are clean."; \
+		echo "Stash or commit your changes and try again."; \
+		exit $$ret; \
+	elif [ $$ret -gt 1 ]; then \
+		exit $$ret; \
+	fi; \
+	clang-format -i -style=file include/*.h src/*.c; ret=$$?; \
+	if [ $$ret -ne 0 ]; then \
+		exit $$ret; \
+	fi; \
+	git diff --quiet --exit-code include src; ret=$$?; \
+	if [ $$ret -eq 1 ]; then \
+		echo "Code style issues found. Run 'git diff' to view issues."; \
+	elif [ $$ret -eq 0 ]; then \
+		echo "No code style issues found."; \
+	fi; \
+	exit $$ret
+
 release:
 	@git rev-parse v$(PACKAGE_VERSION) &> /dev/null; \
 	if [ "$$?" -eq 0 ]; then \

README

@@ -1,3 +1,12 @@
+With the release of mixer-tools v4.3.0 [1] the swupd-server project is
+deprecated.  All functionality has been re-written into the mixer tool under
+the `mixer build update` command. All development done around creating update
+content consumed by swupd-client is now done in that project.
+
+[1] https://github.com/clearlinux/mixer-tools
+
+---
+
 The swupd-server package provides a reference implementation of a software
 update server-side component that generates update content consumable by a
 software update client (swupd-client). Such content includes manifests that

basic_creator.sh

@@ -15,7 +15,7 @@ error() {
 # eg: all of openstack, pnp, bat, cloud stuff, non-basic scripting language bundles
 ${SWUPDREPO}/mk_groups_ini.sh
 
-PREVREL=`cat ${UPDATEDIR}/image/latest.version`
+PREVREL=`cat ${UPDATEDIR}/image/LAST_VER`
 
 export SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR:-"/root/swupd-certs"}
 export LEAF_KEY="leaf.key.pem"
@@ -45,12 +45,8 @@ for job in $(jobs -p); do
 	fi
 done
 
-# create delta packs for 2 versions back
-NUM_PACKS=2
-${SWUPDREPO}/pack_maker.sh ${VER} ${NUM_PACKS}
-
 # expose the new build to staging / testing
-echo ${VER} > ${UPDATEDIR}/image/latest.version
+echo ${VER} > ${UPDATEDIR}/image/LAST_VER
 
 STAGING_FILE="${UPDATEDIR}/www/version/formatstaging/latest"
 if [ ! -f "${STAGING_FILE}" ]; then

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.2.2, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.7.5, matthew.johnson@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC
@@ -82,6 +82,11 @@ AS_IF([test "$enable_lzma" != "no"], [
 ])
 AM_CONDITIONAL([ENABLE_LZMA], [test "$enable_lzma" != "no"])
 
+AC_ARG_ENABLE([rename-detection], [AS_HELP_STRING([--enable-rename-detection], [enable rename detection feature])])
+
+AS_IF([test "$enable_rename_detection" = "yes"], [AC_DEFINE(RENAMES,1,[Use rename detection])])
+AM_CONDITIONAL([RENAMES], [test "$enable_rename_detection" = "yes"])
+
 AC_CONFIG_FILES([Makefile])
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT

include/swupd.h

@@ -19,19 +19,25 @@
 #if SWUPD_WITH_BSDTAR
 #define TAR_COMMAND "bsdtar"
 #define TAR_XATTR_ARGS ""
-#define TAR_WARN_ARGS ""
+#define TAR_XATTR_ARGS_STRLIST
+#define TAR_WARN_ARGS_STRLIST
 #else
 #define TAR_COMMAND "tar"
 #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'"
-#define TAR_WARN_ARGS "--warning=no-timestamp"
+#define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'",
+#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp",
 #endif
 
 #if SWUPD_WITH_SELINUX
 #define TAR_PERM_ATTR_ARGS "--preserve-permissions --selinux " TAR_XATTR_ARGS
+#define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions", "--selinux"
 #else
 #define TAR_PERM_ATTR_ARGS "--preserve-permissions " TAR_XATTR_ARGS
+#define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions"
 #endif
 
+#define SWUPD_HASH_DIRNAME "DIRECTORY"
+
 #if SWUPD_WITH_STATELESS
 #define OS_IS_STATELESS 1
 #else
@@ -59,6 +65,13 @@
 #include <lzma.h>
 #endif
 
+/* Approximatly the smallest size of a pair of input files which
+ * differ by a single bit that bsdiff can produce a more compact
+ * deltafile. Files smaller than this are always marked as different.
+ * See the magic 200 value in the bsdiff/src/diff.c code.
+ */
+#define BSDIFFSIZE 200
+
 struct manifest {
 	unsigned long long int format;
 	int version;
@@ -72,6 +85,8 @@ struct manifest {
 	GList *submanifests; /* as struct manifest */
 
 	GList *includes; /* struct manifests for all bundles included into this one */
+
+	GList *actions; /* post-update actions */
 };
 
 struct file;
@@ -102,6 +117,10 @@ struct file {
 	unsigned int is_file : 1;
 	unsigned int is_link : 1;
 	unsigned int is_deleted : 1;
+	/* a ghosted file is treated as deleted so that it can be used as a rename,
+	 * but is not actually deleted in the client due to a 3rd-party program
+	 * cleaning it up. This happens with boot files managed by a boot manager */
+	unsigned int is_ghosted : 1;
 	unsigned int is_manifest : 1;
 
 	/* and these are modifiers */
@@ -116,7 +135,7 @@ struct file {
 	double rename_score;
 	struct file *rename_peer;
 	char *alpha_only_filename; /* filename minus all numerics/etc */
-	char *filetype;
+	char *filetype;		   /* The output of 'file', truncated */
 	char *basename;
 	char *dirname;
 	/* end of rename detection fields */
@@ -137,7 +156,6 @@ extern int current_version;
 extern int newversion;
 extern int minversion;
 extern unsigned long long int format;
-extern bool enable_signing;
 
 extern char *state_dir;
 extern char *packstage_dir;
@@ -161,7 +179,9 @@ extern void release_configuration_data(void);
 extern char *config_image_base(void);
 extern char *config_output_dir(void);
 extern char *config_empty_dir(void);
+extern char *config_debuginfo_path(const char *path);
 extern int config_initial_version(void);
+extern bool config_ban_debuginfo(void);
 
 extern void read_current_version(char *filename);
 extern void write_new_version(char *filename, int version);
@@ -172,13 +192,14 @@ extern GList *get_last_versions_list(int next_version, int max_versions);
 extern char *file_type_to_string(struct file *file);
 extern struct manifest *manifest_from_file(int version, char *module);
 extern void free_manifest(struct manifest *manifest);
-extern struct manifest *alloc_manifest(int version, char *module);
+extern struct manifest *alloc_manifest(int version, char *module, GList *actions);
 extern int match_manifests(struct manifest *m1, struct manifest *m2);
 extern void sort_manifest_by_version(struct manifest *manifest);
 extern bool manifest_includes(struct manifest *manifest, char *component);
 extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
-extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
+extern void clean_renames(struct manifest *manifest);
+extern int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *, struct file *));
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
 extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);
@@ -213,8 +234,7 @@ extern void chroot_create_full(int newversion);
 
 extern void read_group_file(char *filename);
 extern void release_group_file(void);
-extern char *group_groups(char *group);
-extern char *group_packages(char *group);
+extern char *group_status(char *group);
 extern char *next_group(void);
 
 extern void apply_heuristics(struct manifest *manifest);
@@ -236,9 +256,10 @@ extern int previous_version_manifest(struct manifest *mom, char *name);
 
 extern void type_change_detection(struct manifest *manifest);
 
-extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
-extern void link_renames(GList *newfiles, struct manifest *from_manifest);
-extern void __create_delta(struct file *file, int from_version);
+extern void rename_detection(struct manifest *manifest);
+extern void link_renames(GList *newfiles, int to_version);
+extern void final_link(GList *files);
+extern void __create_delta(struct file *file, int from_version, char *from_hash);
 
 extern void account_delta_hit(void);
 extern void account_delta_miss(void);
@@ -246,11 +267,19 @@ extern void account_delta_miss(void);
 extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write only */
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
-extern void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time);
+extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
+extern int system_argv_pipe(char *const lhscmd[], char *const rhscmd[]);
+extern int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			       int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
+extern void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			 int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[]);
 extern int system_argv(char *const argv[]);
-
-extern bool signature_initialize(void);
-extern void signature_terminate(void);
-extern bool signature_sign(const char *filename);
+extern int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[]);
+extern pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[]);
+extern void move_fd(int oldfd, int newfd);
+extern int wait_process_terminate(pid_t pid);
+extern int num_threads(float scaling);
+extern bool file_is_debuginfo(const char *path);
 
 #endif

mk_groups_ini.sh

@@ -18,8 +18,8 @@ if [ ! -f "$SWUPD_SERVER_INI" ]; then
 	sed -i "s|/var/lib/update|$UPDATEDIR|" $SWUPD_SERVER_INI
 fi
 
-if [ ! -f "$UPDATEDIR/image/latest.version" ]; then
-	echo "0" > $UPDATEDIR/image/latest.version
+if [ ! -f "$UPDATEDIR/image/LAST_VER" ]; then
+	echo "0" > $UPDATEDIR/image/LAST_VER
 fi
 
 echo "rebuilding $SWUPD_GROUPS_INI based on $BUNDLEREPO"
@@ -27,7 +27,11 @@ rm -f $SWUPD_GROUPS_INI
 
 for bundle in $(ls $BUNDLEREPO/bundles)
 do
+	status=$(awk -F: '/^# .STATUS/ {print $2}' $BUNDLEREPO/$bundle | tr -cd '[[:alnum:]]')
 	echo "[$bundle]" >> $SWUPD_GROUPS_INI
 	echo "group=$bundle" >> $SWUPD_GROUPS_INI
+	if [ -n "$status" ]; then
+		echo "status=$status" >> $SWUPD_GROUPS_INI
+	fi
 	echo "" >> $SWUPD_GROUPS_INI
 done

pack_maker.sh

@@ -1,112 +0,0 @@
-#!/bin/bash -e
-# usage:
-#	pack_maker.sh <to version> <back_count>
-#	pack_maker.sh 2820 2
-#	pack_maker.sh 2730 3
-#
-# For each bundle changed as of the Manifest.MoM at the <to version>,
-# create version-pair packs going back <back_count>
-# versions of that bundle.  For example if bundle "editors" changed in 2650
-# and the script is run as "pack_maker.sh 2650 2", it will find that
-# "editors" last two change points were 2550 and 2260 and thus create a
-# 2550-to-2650 and a 2260-to-2650 version pair pack for the "editors"
-# bundle.
-
-#NOTE: fullfiles must be created before calling this script
-#NOTE: zero packs need similarly created
-
-#NOTE: build numbers are sparse, search back for the next oldest existing build
-#NOTE: in the future isn't simply -10, and when that change happens we must also
-#      consider which version-pair packs are meaningful for intermediate builds
-#      and revert builds.
-
-SWUPDREPO=${SWUPDREPO:-"/usr/src/clear-projects/swupd-server"}
-BUNDLEREPO=${BUNDLEREPO:-"/usr/src/clear-projects/clr-bundles"}
-UPDATEDIR=${UPDATEDIR:-"/var/lib/update"}
-SWUPDWEBDIR="${UPDATEDIR}/www"
-SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR=:-"/root/swupd-certs"}
-
-export XZ_DEFAULTS="--threads 0"
-
-export SWUPD_CERTS_DIR
-export LEAF_KEY="leaf.key.pem"
-export LEAF_CERT="leaf.cert.pem"
-export CA_CHAIN_CERT="ca-chain.cert.pem"
-export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase"
-
-VER=$1
-BACK_COUNT=$2
-
-if [ "${BACK_COUNT}" == "" ]; then
-	echo "missing to pack count"
-	exit 1
-fi
-
-if [ "${VER}" == "" ]; then
-	echo "missing to version"
-	exit 1
-fi
-
-MOM=${SWUPDWEBDIR}/${VER}/Manifest.MoM
-if [ ! -e ${MOM} ]; then
-	echo "invalid version (no ${MOM})"
-	exit 1
-fi
-
-#time ${SWUPDREPO}/swupd_make_fullfiles --statedir ${UPDATEDIR} ${VER}
-
-BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }')
-
-# build packs for all bundles changed in $VER
-for BUNDLE in $BUNDLE_LIST; do
-	#${SWUPDREPO}/swupd_make_pack --statedir ${UPDATEDIR} 0 ${VER} ${BUNDLE} &
-
-	BUNDLE_VER_LIST=""
-	MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$VER/Manifest.MoM | grep "^previous:" | cut -f 2)
-	BUNDLE_VER=""
-	COUNT=0
-	while [ $COUNT -lt $BACK_COUNT ] && [ $MANIFEST_VER -gt 0 ]; do
-		MOM=${SWUPDWEBDIR}/$MANIFEST_VER/Manifest.MoM
-		if [ -e ${MOM} ]; then
-			BUNDLE_VER=$(cat ${MOM} | grep "	${BUNDLE}$" | cut -f 3)
-			if [ "$BUNDLE_VER" != "" ]; then
-				BUNDLE_VER_LIST="$BUNDLE_VER $BUNDLE_VER_LIST"
-				MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$BUNDLE_VER/Manifest.MoM | grep "^previous:" | cut -f 2)
-				let COUNT=$((COUNT+1))
-			else
-				# back in history to where bundle didn't exist yet
-				MANIFEST_VER=0
-			fi
-		else
-			# seek backwards until a manifest if found
-			MANIFEST_VER=$((MANIFEST_VER-10))
-		fi
-	done
-
-	for v in $BUNDLE_VER_LIST; do
-		if [ -e ${SWUPDWEBDIR}/${VER}/pack-${BUNDLE}-from-$v.tar ]; then
-			echo "${VER}/pack-${BUNDLE}-from-$v.tar already exists, skipping."
-		else
-			${SWUPDREPO}/swupd_make_pack --statedir ${UPDATEDIR} $v ${VER} ${BUNDLE} &
-		fi
-	done
-done
-
-for job in $(jobs -p)
-do
-	wait ${job}
-	RET=$?
-	if [ "$RET" != "0" ]; then
-		echo "pack creation failure!"
-		exit $RET
-	fi
-done
-
-# NOTE: your devops will want to expose the completed swupd server build to
-# trial usage at some point.  This would be done via code like:
-#
-#	echo ${VER} > ${SWUPDWEBDIR}/../image/latest.version
-#	STAGING=$(cat ${SWUPDWEBDIR}/version/formatstaging/latest)
-#	if [ "${STAGING}" -lt "${VER}" ]; then
-#		echo ${VER} > ${SWUPDWEBDIR}/version/formatstaging/latest
-#	fi

packfsck.pl

@@ -1,82 +0,0 @@
-#!/usr/bin/perl -w
-#
-#
-# arguments:
-#     perl packfsck.pl <target version>
-
-
-my $target = $ARGV[0];
-system("rm /tmp/Manifest");
-system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core");
-
-
-my $from = $target;
-while ($from > $target - 100) {
- $from = $from - 10;
- print "Testing the $from-$target pack\n";
- system("rm /tmp/pack.tar");
- system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar");
-
-
-
- open FILE, "</tmp/Manifest";
-
- my %expected_hashes;
- my %files;
-
- while (<FILE>) {
-   my $line = $_;
-  
-  
-   if ($line =~ /(^[FDLr\.]+)\s*([0-9a-f]+)\s*([0-9]+)\s*+(.*)\n/) {
-      my $type = $1;
-      my $hash = $2;
-      my $version = $3;
-      my $filename = $4;
-      if ($version > $from) {
-        $expected_hashes{"$hash"} = 1;
-        $files{"$hash"} = $filename;
-#        print "$1 - $2 - $3 - $4\n";
-      }
-   } 
- }
- close FILE;
-
-
- open FILE2, "-|", "tar -tf /tmp/pack.tar 2> /dev/null";
-
- while (<FILE2>) {
-   my $line = $_;
-  
-
-  if ($line =~ /^\.\/delta\/[0-9]+.[0-9]+.([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected delta hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  } 
-  if ($line =~ /\.\/staged\/([0-9a-f]+)/) {
-      my $hash = $1;
-      if (!defined($expected_hashes{"$hash"})) {
-          print "\tUnexpected staged hash found $hash\n";
-      }
-      $expected_hashes{"$hash"} = 2;
-  }
- }
- close FILE2;
-
- my $count = 0;
- foreach my $key ( keys %expected_hashes )
- {
-  my $value = $expected_hashes{$key};
-  if ($value eq 1) {
-     my $fn = $files{$key};
-     print "\tFile $fn ($key) is not in the pack\n";
-     $count = $count = 1;
-  }
- }
- if ($count < 1) {
-  print "\tThe $from-$target pack has no files missing\n";
- }
-}

server.ini

@@ -2,3 +2,8 @@
 emptydir=/var/lib/update/empty/
 imagebase=/var/lib/update/image/
 outputdir=/var/lib/update/www/
+
+[Debuginfo]
+banned=true
+lib=/usr/lib/debug/
+src=/usr/src/debug/

src/analyze_fs.c

@@ -42,6 +42,10 @@
 
 static GThreadPool *threadpool;
 
+/* Why not strcpy? Looks like the hash was going to be stored in
+ * binary at one stage. Should use g_string_chunk_insert_const to
+ * change hash_compare to a pointer compare
+ */
 void hash_assign(char *src, char *dst)
 {
 	memcpy(dst, src, SWUPD_HASH_LEN - 1);
@@ -147,7 +151,13 @@ void populate_file_struct(struct file *file, char *filename)
 	ret = lstat(filename, &stat);
 	if (ret < 0) {
 		LOG(NULL, "stat error ", "%s: %s", filename, strerror(errno));
-		file->is_deleted = 1;
+		/* delete the file if it isn't found and isn't a boot file,
+		 * mark as ghosted if this is a boot file */
+		if (file->is_boot) {
+			file->is_ghosted = 1;
+		} else {
+			file->is_deleted = 1;
+		}
 		return;
 	}
 	file->stat.st_mode = stat.st_mode;
@@ -196,7 +206,7 @@ int compute_hash(struct file *file, char *filename)
 		return 0;
 	}
 
-	hash_set_zeros(key);
+	hash_set_zeros(key); /* Set to 64 '0' (not '\0') characters */
 
 	if (file->is_link) {
 		char link[PATH_MAX];
@@ -222,7 +232,7 @@ int compute_hash(struct file *file, char *filename)
 		hmac_sha256_for_string(file->hash,
 				       (const unsigned char *)key,
 				       key_len,
-				       file->filename); //file->filename not filename
+				       SWUPD_HASH_DIRNAME); // Make independent of dirname
 		return 0;
 	}
 
@@ -275,7 +285,7 @@ static void get_hash(gpointer data, gpointer user_data)
 /* disallow characters which can do unexpected things when the filename is
  * used on a tar command line via system("tar [args] filename [more args]");
  */
-static bool illegal_characters(char *filename)
+static bool illegal_characters(const char *filename)
 {
 	char c;
 	int i;
@@ -301,27 +311,156 @@ static bool illegal_characters(char *filename)
 	return false;
 }
 
+static struct file *add_file(struct manifest *manifest,
+			     const char *entry_name,
+			     char *sub_filename,
+			     char *fullname,
+			     bool do_hash)
+{
+	GError *err = NULL;
+	struct file *file;
+
+	if (config_ban_debuginfo() && file_is_debuginfo(sub_filename)) {
+		printf("WARNING: File %s is banned ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
+	if (illegal_characters(entry_name)) {
+		printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
+	file = calloc(1, sizeof(struct file));
+	assert(file);
+
+	file->last_change = manifest->version;
+	file->filename = sub_filename;
+
+	populate_file_struct(file, fullname);
+	if (file->is_deleted) {
+		/*
+		 * populate_file_struct() logs a stat() failure, but
+		 * does not abort. When adding files that should
+		 * exist, this case is an error.
+		 */
+		LOG(NULL, "file not found", "%s", fullname);
+		assert(0);
+	}
+
+	/* if for some reason there is a file in the official build
+	 * which should not be included in the Manifest, then open a bug
+	 * to get it removed, and work around its presence by
+	 * excluding it here, eg:
+	 if (strncmp(file->filename, "/dev/", 5) == 0) {
+	 continue;
+	 }
+	*/
+
+	if (do_hash) {
+		/* compute the hash from a thread */
+		int ret;
+		ret = g_thread_pool_push(threadpool, file, &err);
+		if (ret == FALSE) {
+			printf("GThread hash computation push error\n");
+			printf("%s\n", err->message);
+			assert(0);
+		}
+	}
+	manifest->files = g_list_prepend(manifest->files, file);
+	manifest->count++;
+	return file;
+}
+
 static void iterate_directory(struct manifest *manifest, char *pathprefix,
 			      char *subpath, bool do_hash)
 {
 	DIR *dir;
 	struct dirent *entry;
 	char *fullpath;
-	int ret;
-	GError *err = NULL;
 
 	string_or_die(&fullpath, "%s/%s", pathprefix, subpath);
 
 	dir = opendir(fullpath);
 	if (!dir) {
+		bool fatal_error = errno != ENOENT;
+		FILE *content;
+
 		free(fullpath);
+		if (fatal_error) {
+			return;
+		}
+		/*
+		 * If there is a <dir>.content.txt instead of
+		 * the actual directory, then read that
+		 * file. It has a list of path names,
+		 * including all directories. The
+		 * corresponding file system entry is then
+		 * expected to be in a pre-populated "full"
+		 * directory.
+		 *
+		 * Only supported at top level (i.e. empty
+		 * subpath) to keep the code and testing
+		 * simpler.
+		 */
+		assert(!subpath[0]);
+		string_or_die(&fullpath, "%s.content.txt", pathprefix);
+		content = fopen(fullpath, "r");
+		free(fullpath);
+		fullpath = NULL;
+		if (content) {
+			char *line = NULL;
+			size_t len = 0;
+			ssize_t read;
+			const char *full;
+			int full_len;
+			/*
+			 * determine path to "full" directory: it is assumed to be alongside
+			 * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit,
+			 * so we have to strip the last path component.
+			 */
+			full = strrchr(pathprefix, '/');
+			if (full) {
+				full_len = full - pathprefix + 1;
+				full = pathprefix;
+			} else {
+				full = "";
+				full_len = 0;
+			}
+			while ((read = getline(&line, &len, content)) != -1) {
+				if (read) {
+					const char *entry_name = strrchr(line, '/');
+					if (entry_name) {
+						entry_name++;
+					} else {
+						entry_name = line;
+					}
+					if (line[read - 1] == '\n') {
+						line[read - 1] = 0;
+					}
+					string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line);
+					add_file(manifest,
+						 entry_name,
+						 strdup(line),
+						 fullpath,
+						 do_hash);
+				}
+			}
+			free(line);
+		}
+
+		// If both directory and content file are missing, silently (?)
+		// don't add anything to the manifest.
 		return;
 	}
 
 	while (dir) {
-		struct file *file;
 		char *sub_filename;
 		char *fullname;
+		struct file *file;
 
 		entry = readdir(dir);
 		if (!entry) {
@@ -334,50 +473,14 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
-
-		if (illegal_characters(entry->d_name)) {
-			printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
-			free(sub_filename);
-			continue;
-		}
-
-		file = calloc(1, sizeof(struct file));
-		if (!file) {
-			break;
-		}
-
-		file->last_change = manifest->version;
-		file->filename = sub_filename;
-
 		string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
-		populate_file_struct(file, fullname);
-		free(fullname);
 
-		if (entry->d_type == DT_DIR) {
+		/* takes ownership of the strings, so we don't need to free it */
+		file = add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
+
+		if (file && file->is_dir) {
 			iterate_directory(manifest, pathprefix, file->filename, do_hash);
 		}
-
-		/* if for some reason there is a file in the official build
-		 * which should not be included in the Manifest, then open a bug
-		 * to get it removed, and work around its presence by
-		 * excluding it here, eg:
-		if (strncmp(file->filename, "/dev/", 5) == 0) {
-			continue;
-		}
-		 */
-
-		if (do_hash) {
-			/* compute the hash from a thread */
-			ret = g_thread_pool_push(threadpool, file, &err);
-			if (ret == FALSE) {
-				printf("GThread hash computation push error\n");
-				printf("%s\n", err->message);
-				closedir(dir);
-				return;
-			}
-		}
-		manifest->files = g_list_prepend(manifest->files, file);
-		manifest->count++;
 	}
 	closedir(dir);
 	free(fullpath);
@@ -387,14 +490,15 @@ struct manifest *full_manifest_from_directory(int version)
 {
 	struct manifest *manifest;
 	char *dir;
+	int numthreads = num_threads(1.0);
 
 	LOG(NULL, "Computing hashes", "for %i/full", version);
 
-	manifest = alloc_manifest(version, "full");
+	manifest = alloc_manifest(version, "full", NULL);
 
 	string_or_die(&dir, "%s/%i/full", image_dir, version);
 
-	threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL);
+	threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL);
 
 	iterate_directory(manifest, dir, "", true);
 
@@ -418,7 +522,7 @@ GList *get_sub_manifest_includes(char *component, int version)
 	char *included;
 	char line[8192];
 
-	conf = config_output_dir();
+	conf = config_image_base();
 	if (conf == NULL) {
 		assert(0);
 	}
@@ -466,7 +570,7 @@ struct manifest *sub_manifest_from_directory(char *component, int version)
 
 	LOG(NULL, "Creating component manifest", "for %i/%s", version, component);
 
-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);
 
 	string_or_die(&dir, "%s/%i/%s", image_dir, version, component);
 

src/chroot.c

@@ -34,22 +34,28 @@
 
 void chroot_create_full(int newversion)
 {
-	int ret;
 	char *group;
-	char *command;
+	char *param;
 	char *full_dir;
 
 	string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
+	if (!access(full_dir, R_OK | X_OK)) {
+		free(full_dir);
+		return;
+	}
 
 	g_mkdir_with_parents(full_dir, S_IRWXU);
 
 	/* start with base */
-	LOG(NULL, "Copying chroot os-core to full", "");
-	string_or_die(&command, "rsync -aAX %s/%i/os-core/ %s",
-		      image_dir, newversion, full_dir);
-	ret = system(command);
-	assert(ret == 0);
-	free(command);
+	string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
+	if (!access(param, F_OK)) {
+		LOG(NULL, "Copying chroot os-core to full", "");
+		char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
+		if (system_argv(rsynccmd) != 0) {
+			assert(0);
+		}
+	}
+	free(param);
 
 	/* overlay any new files from each group */
 	while (1) {
@@ -58,32 +64,15 @@ void chroot_create_full(int newversion)
 			break;
 		}
 
-		LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
-		string_or_die(&command, "rsync -aAX --ignore-existing %s/%i/%s/ %s",
-			      image_dir, newversion, group, full_dir);
-		ret = system(command);
-		assert(ret == 0);
-		free(command);
-	}
-
-	/* and since some dumb files have things like install timestamps in
-	 * them, we pull files from full back to bundles so each instance
-	 * of the "same" file is truly identical across bundles.  It may
-	 * make sense at some point to add more logic here for conflict
-	 * detection and resolution, including work to insure renames are
-	 * consistent across groups. */
-	while (1) {
-		group = next_group();
-		if (!group) {
-			break;
+		string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
+		if (!access(param, F_OK)) {
+			LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
+			char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
+			if (system_argv(rsynccmd) != 0) {
+				assert(0);
+			}
 		}
-
-		LOG(NULL, "Recopy bundle chroot out of full", "%s", group);
-		string_or_die(&command, "rsync -aAX --existing %s %s/%i/%s",
-			      full_dir, image_dir, newversion, group);
-		ret = system(command);
-		assert(ret == 0);
-		free(command);
+		free(param);
 	}
 	free(full_dir);
 }

src/config.c

@@ -67,6 +67,20 @@ int config_initial_version(void)
 	return version;
 }
 
+bool config_ban_debuginfo(void)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_boolean(keyfile, "Debuginfo", "banned", NULL);
+}
+
+char *config_debuginfo_path(const char *comp)
+{
+	assert(keyfile != NULL);
+
+	return g_key_file_get_value(keyfile, "Debuginfo", comp, NULL);
+}
+
 bool read_configuration_file(char *filename)
 {
 	GError *error = NULL;

src/create_update.c

@@ -29,6 +29,7 @@
 #include <errno.h>
 #include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -49,12 +50,12 @@ static void banner(void)
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
 	{ "version", no_argument, 0, 'v' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "osversion", required_argument, 0, 'o' },
 	{ "minversion", required_argument, 0, 'm' },
 	{ "format", required_argument, 0, 'F' },
 	{ "getformat", no_argument, 0, 'g' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -67,12 +68,12 @@ static void print_help(const char *name)
 	printf("   -v, --version           Show software version\n");
 	printf("\n");
 	printf("Application Options:\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -o, --osversion         The OS version for which to create an update\n");
 	printf("   -m, --minversion        Optional minimum file version to write into manifests per file\n");
 	printf("   -F, --format            Format number for the update\n");
 	printf("   -g, --getformat         Print current format string and exit\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }
 
@@ -80,12 +81,15 @@ static bool parse_options(int argc, char **argv)
 {
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:s", prog_opts, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:", prog_opts, NULL)) != -1) {
 		switch (opt) {
 		case '?':
 		case 'h':
 			print_help(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'v':
 			banner();
 			return false;
@@ -124,9 +128,6 @@ static bool parse_options(int argc, char **argv)
 				free_globals();
 			}
 			exit(0);
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}
 
@@ -140,13 +141,14 @@ static bool parse_options(int argc, char **argv)
 static void populate_dirs(int version)
 {
 	char *newversiondir;
+	char *newversiondircontent = NULL;
 
 	string_or_die(&newversiondir, "%s/%d", image_dir, version);
 
 	if ((access(newversiondir, F_OK | R_OK) != 0) && (version == 0)) {
 		char *latestpath = NULL;
 
-		string_or_die(&latestpath, "%s/latest.version", image_dir);
+		string_or_die(&latestpath, "%s/LAST_VER", image_dir);
 
 		printf("** %s does not exist... creating and populating\n", newversiondir);
 		if (mkdir(newversiondir, 0755) != 0) {
@@ -181,9 +183,11 @@ static void populate_dirs(int version)
 			}
 
 			string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group);
+			string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group);
 
 			/* Create the bundle directory(s) as needed */
-			if (access(newversiondir, F_OK | R_OK) != 0) {
+			if (access(newversiondir, F_OK | R_OK) != 0 &&
+			    access(newversiondircontent, F_OK | R_OK) != 0) {
 				printf("%s does not exist...creating\n", group);
 				if (mkdir(newversiondir, 0755) != 0) {
 					printf("Failed to create %s subdirectory\n", group);
@@ -192,6 +196,7 @@ static void populate_dirs(int version)
 		}
 	}
 	free(newversiondir);
+	free(newversiondircontent);
 }
 
 static int check_build_env(void)
@@ -212,6 +217,35 @@ static int check_build_env(void)
 	return 0;
 }
 
+static int check_group_file(void)
+{
+	int ret = -1;
+
+	// Ensure the os-core group is defined
+	while (1) {
+		char *group = next_group();
+		if (!group) {
+			break;
+		}
+		if (strcmp(group, "os-core") == 0) {
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static bool both_ghosted(struct file *file1, struct file *file2)
+{
+	return (file1->is_ghosted && file2->is_ghosted);
+}
+
+static bool both_deleted(struct file *file1, struct file *file2)
+{
+	return (file1->is_deleted && file2->is_deleted);
+}
+
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -228,6 +262,7 @@ int main(int argc, char **argv)
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
 	int old_deleted = 0;
+	int old_ghosted = 0;
 
 	struct timeval current_time;
 	struct timeval previous_time;
@@ -240,6 +275,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_globals();
 		return EXIT_FAILURE;
@@ -253,12 +293,6 @@ int main(int argc, char **argv)
 		goto exit;
 	}
 
-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		goto exit;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	if (!read_configuration_file(file_path)) {
 		printf("Failed to read %s configuration file!\n", state_dir);
@@ -270,8 +304,13 @@ int main(int argc, char **argv)
 	string_or_die(&file_path, "%s/groups.ini", state_dir);
 	read_group_file(file_path);
 	free(file_path);
+	ret = check_group_file();
+	if (ret != 0) {
+		printf("os-core bundle is not listed in groups.ini, this is required.\n");
+		goto exit;
+	}
 
-	read_current_version("latest.version");
+	read_current_version("LAST_VER");
 	printf("Last processed version is %i\n", current_version);
 
 	populate_dirs(newversion);
@@ -288,7 +327,7 @@ int main(int argc, char **argv)
 	printf("Syncing bundle chroots with full chroot\n");
 	chroot_create_full(newversion);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("full chroot creation", &previous_time, &current_time);
 
 	printf("Calculating full manifest (this is expensive/slow)\n");
 	old_full = manifest_from_file(current_version, "full");
@@ -297,24 +336,51 @@ int main(int argc, char **argv)
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
 
-	old_deleted = remove_old_deleted_files(old_full, new_full);
+	if (old_full->format < new_full->format) {
+		old_deleted = remove_deprecated_files(old_full, new_full, both_deleted);
+	} else {
+		old_deleted = 0;
+	}
+
 	if (old_deleted > 0) {
 		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
 		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
 	}
 
-	apply_heuristics(new_full);
-#warning disabled rename detection for some simplicity
-	// rename_detection(new_full);
+	old_ghosted = remove_deprecated_files(old_full, new_full, both_ghosted);
+	if (old_ghosted > 0) {
+		LOG(NULL, "", "Old ghosted files (%d) removed from full manifest", old_ghosted);
+		printf("Old ghosted files (%d) removed from full manifest", old_ghosted);
+	}
 
-	print_elapsed_time(&previous_time, &current_time);
+	apply_heuristics(new_full);
+
+	print_elapsed_time("full manifest creation", &previous_time, &current_time);
 
 	printf("Entering phase 2: Doing the os-core bundle\n");
 	/* Phase 2 : the os-core set */
 
 	/* Step 2: Make a manifest for the os-core set */
 	old_MoM = manifest_from_file(current_version, "MoM");
-	new_MoM = alloc_manifest(newversion, "MoM");
+
+	/* Verify that the new format is not older than the previous format. It
+	 * never makes sense to decrease the format number for the next build. */
+	if (format < old_MoM->format) {
+		LOG(NULL, "", "Current format (%llu) must be greater than or equal to previous format (%llu). Exiting",
+		    format, old_MoM->format);
+		printf("Current format (%llu) must be greater than or equal to previous format (%llu). Exiting\n",
+		       format, old_MoM->format);
+		goto exit;
+	}
+
+	/* Detect a format bump and add the "update" action to the manifest
+	 * "actions:" field */
+	GList *actions = NULL;
+	if (format > old_MoM->format) {
+		actions = g_list_prepend(actions, "update");
+	}
+
+	new_MoM = alloc_manifest(newversion, "MoM", actions);
 	old_core = manifest_from_file(manifest_subversion(old_MoM, "os-core"), "os-core");
 	new_core = sub_manifest_from_directory("os-core", newversion);
 	add_component_hashes_to_manifest(new_core, new_full);
@@ -337,27 +403,33 @@ int main(int argc, char **argv)
 		apply_heuristics(new_core);
 		/* Step 3c: ... else save the manifest */
 		type_change_detection(new_core);
+#ifdef RENAMES
+		/* Detect renamed files specifically for os-core */
+		rename_detection(new_core);
+#endif
+		if (old_core->format < new_core->format) {
+			old_deleted = remove_deprecated_files(old_core, new_core, both_deleted);
+		} else {
+			old_deleted = 0;
+		}
 
-#warning disabled rename detection for some simplicity
-		/* Detect renamed files specifically for each pack */
-		// rename_detection(...);
-
-		old_deleted = remove_old_deleted_files(old_core, new_core);
-		sort_manifest_by_version(new_core);
+		old_ghosted = remove_deprecated_files(old_core, new_core, both_ghosted);
+		sort_manifest_by_version(new_core); /* sorts by filename */
+		/* clean up orphaned renames by marking them as deleted */
+		clean_renames(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
 			LOG(NULL, "", "Core component has not changed (after pruning), exiting");
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
-		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", newfiles, old_deleted, old_ghosted);
+		printf("Core component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", newfiles, old_deleted, old_ghosted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
 			goto exit;
 		}
-		create_manifest_deltas(new_core, manifests_last_versions_list);
 	}
 
 	nest_manifest(new_MoM, new_core);
@@ -379,25 +451,40 @@ int main(int argc, char **argv)
 		GList *name_includes;
 		char *group = next_group();
 		struct manifest *manifest;
+		struct manifest *current = NULL;
 
 		if (!group) {
 			break;
 		}
 		manifest = g_hash_table_lookup(new_manifests, group);
 		name_includes = manifest->includes;
+
 		while (name_includes) {
 			char *name = name_includes->data;
 			name_includes = g_list_next(name_includes);
-			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(new_manifests, name));
+
+			current = g_hash_table_lookup(new_manifests, name);
+
+			// Avoid adding duplicate includes to the list
+			if (g_list_find(manifest_includes, current) == NULL) {
+				manifest_includes = g_list_prepend(manifest_includes, current);
+			}
 		}
 		manifest->includes = manifest_includes;
 		manifest_includes = NULL;
 		manifest = g_hash_table_lookup(old_manifests, group);
 		name_includes = manifest->includes;
+
 		while (name_includes) {
 			char *name = name_includes->data;
 			name_includes = g_list_next(name_includes);
-			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(old_manifests, name));
+
+			current = g_hash_table_lookup(old_manifests, name);
+
+			// Avoid adding duplicate includes to the list
+			if (g_list_find(manifest_includes, current) == NULL) {
+				manifest_includes = g_list_prepend(manifest_includes, current);
+			}
 		}
 		manifest->includes = manifest_includes;
 	}
@@ -444,21 +531,31 @@ int main(int argc, char **argv)
 			newm->version = oldm->version;
 		} else {
 			apply_heuristics(newm);
-#warning missing rename_detection here
+#ifdef RENAMES
+			/* Detect renamed files specifically for this bundle */
+			rename_detection(newm);
+#endif
 			/* Step 6b: otherwise, write out the manifest */
-			old_deleted = remove_old_deleted_files(oldm, newm);
+			if (oldm->format < newm->format) {
+				old_deleted = remove_deprecated_files(oldm, newm, both_deleted);
+			} else {
+				old_deleted = 0;
+			}
+
+			old_ghosted = remove_deprecated_files(oldm, newm, both_ghosted);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
+			/* clean up orphaned renames by marking them as deleted */
+			clean_renames(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
-				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
-				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
+			if (newfiles > 0 || old_deleted > 0 || old_ghosted > 0 || changed_includes(oldm, newm)) {
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest", group, newfiles, old_deleted, old_ghosted);
+				printf("%s component has changes (%d new, %d deleted, %d ghosted), writing out new manifest\n", group, newfiles, old_deleted, old_ghosted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);
 					goto exit;
 				}
-				create_manifest_deltas(newm, manifests_last_versions_list);
 			} else {
 				LOG(NULL, "", "%s component has not changed (after pruning), no new manifest", group);
 				printf("%s component has not changed (after pruning), no new manifest\n", group);
@@ -469,7 +566,7 @@ int main(int argc, char **argv)
 		nest_manifest(new_MoM, newm);
 	}
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("bundle manifest creation", &previous_time, &current_time);
 
 	printf("Entering phase 4: completion manifests \n");
 	/* Phase 4 : manifest completion */
@@ -480,9 +577,8 @@ int main(int argc, char **argv)
 		printf("Failed to write new MoM\n");
 		goto exit;
 	}
-	create_manifest_deltas(new_MoM, manifests_last_versions_list);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("MoM creation", &previous_time, &current_time);
 
 	printf("Entering phase 5: creating download content\n");
 	/* Phase 5: wrapping up */
@@ -491,6 +587,8 @@ int main(int argc, char **argv)
 	maximize_to_full(new_MoM, new_full);
 
 	sort_manifest_by_version(new_full);
+	/* clean up orphaned renames by marking them as deleted */
+	clean_renames(new_full);
 	prune_manifest(new_full);
 	if (write_manifest(new_full) != 0) {
 		goto exit;
@@ -499,17 +597,16 @@ int main(int argc, char **argv)
 	/* Step 8: Prepare delta directory */
 	prepare_delta_dir(new_full);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("finalize update", &previous_time, &current_time);
 
 	exit_status = EXIT_SUCCESS;
 exit:
 	if (exit_status == EXIT_SUCCESS) {
 		write_cookiecrumbs_to_download_area(newversion);
-		//write_new_version("latest.version", newversion);
+		//write_new_version("LAST_VER", newversion);
 	}
 	release_configuration_data();
 	release_group_file();
-	signature_terminate();
 	g_list_free(manifests_last_versions_list);
 
 	close_log(newversion, exit_status);

src/delta.c

@@ -35,14 +35,13 @@
 #include "swupd.h"
 #include "xattrs.h"
 
-void __create_delta(struct file *file, int from_version)
+void __create_delta(struct file *file, int from_version, char *from_hash)
 {
-	char *original, *newfile, *outfile, *dotfile, *testnewfile, *sanitycheck;
-	char *conf;
+	char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf;
 	int ret;
 
-	if (file->is_link) {
-		return;
+	if (!file->is_file || !file->peer->is_file) {
+		return; /* only support deltas between two regular files right now */
 	}
 
 	if (file->is_deleted) {
@@ -62,10 +61,9 @@ void __create_delta(struct file *file, int from_version)
 
 	conf = config_output_dir();
 
-	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&sanitycheck, "cmp -s \"%s\" \"%s\"", newfile, testnewfile);
+	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
+	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
+	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s-%s.testnewfile", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
 
 	LOG(file, "Making delta", "%s->%s", original, newfile);
 
@@ -99,7 +97,7 @@ void __create_delta(struct file *file, int from_version)
 		ret = 0;
 		goto out;
 	}
-	xattrs_copy(original, newfile);
+	xattrs_copy(original, testnewfile);
 
 	/* does xattrs have been correctly copied?*/
 	if (xattrs_compare(original, testnewfile) != 0) {
@@ -110,13 +108,14 @@ void __create_delta(struct file *file, int from_version)
 		goto out;
 	}
 
-	ret = system(sanitycheck);
-	if (ret == -1 || !WIFEXITED(ret) || WEXITSTATUS(ret) == 2) {
+	char *const sanitycheck[] = { "cmp", "-s", newfile, testnewfile, NULL };
+	ret = system_argv(sanitycheck);
+	if (ret == -1 || ret == 2) {
 		printf("Sanity check system command failed %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
 		assert(0);
 		goto out;
-	} else if (WEXITSTATUS(ret) == 1) {
+	} else if (ret == 1) {
 		printf("Delta application resulted in file mismatch %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
 		LOG(file, "Delta mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile);
@@ -137,7 +136,6 @@ void __create_delta(struct file *file, int from_version)
 		LOG(NULL, "Failed to rename", "");
 	}
 out:
-	free(sanitycheck);
 	free(testnewfile);
 	free(conf);
 	free(newfile);

src/fullfiles.c

@@ -24,6 +24,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <glib.h>
 #include <libgen.h>
 #include <stdint.h>
@@ -41,7 +42,6 @@
 static void create_fullfile(struct file *file)
 {
 	char *origin;
-	char *tarcommand = NULL;
 	char *tarname = NULL;
 	char *rename_source = NULL;
 	char *rename_target = NULL;
@@ -49,6 +49,7 @@ static void create_fullfile(struct file *file)
 	int ret;
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
+	char *param1, *param2;
 
 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -70,7 +71,7 @@ static void create_fullfile(struct file *file)
 	string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
 	if (lstat(origin, &sbuf) < 0) {
 		/* no input file: means earlier phase of update creation failed */
-		LOG(NULL, "Failed to stat %s\n", origin);
+		LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno));
 		assert(0);
 	}
 
@@ -91,13 +92,18 @@ static void create_fullfile(struct file *file)
 			assert(0);
 		}
 
-		string_or_die(&tarcommand, TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -cf - --exclude='%s/?*' './%s' 2> /dev/null | " TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -xf - 2> /dev/null",
-			      dir, base, base, rename_tmpdir);
-		if (system(tarcommand) != 0) {
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		string_or_die(&param1, "--exclude=%s/?*", base);
+		string_or_die(&param2, "./%s", base);
+		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
+		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
+
+		int tarcmdresult = system_argv_pipe(tarcfcmd, tarxfcmd);
+		if (tarcmdresult != 0) {
+			LOG(NULL, "Tar command for copying directory full file failed with code %d", tarcmdresult);
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
 
 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
@@ -108,13 +114,13 @@ static void create_fullfile(struct file *file)
 		free(rename_source);
 
 		/* for a directory file, tar up simply with gzip */
-		string_or_die(&tarcommand, TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar %s",
-			      rename_tmpdir, outdir, file->last_change, file->hash, file->hash);
-		if (system(tarcommand) != 0) {
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		string_or_die(&param1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL };
+
+		if (system_argv(tarcmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
 
 		if (rmdir(rename_target)) {
 			LOG(NULL, "rmdir failed for %s", rename_target);
@@ -123,6 +129,7 @@ static void create_fullfile(struct file *file)
 		if (rmdir(rename_tmpdir)) {
 			LOG(NULL, "rmdir failed for %s", rename_tmpdir);
 		}
+		free(rename_tmpdir);
 
 		free(tmp1);
 		free(tmp2);
@@ -143,29 +150,38 @@ static void create_fullfile(struct file *file)
 
 		/* step 2a: tar it with each compression type  */
 		// lzma
-		string_or_die(&tarcommand, TAR_COMMAND " --directory=%s " TAR_PERM_ATTR_ARGS " -Jcf %s/%i/files/%s.tar.xz %s",
-			      empty, outdir, file->last_change, file->hash, file->hash);
-		if (system(tarcommand) != 0) {
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		string_or_die(&param1, "--directory=%s", empty);
+		string_or_die(&param2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
+		char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL };
+
+		if (system_argv(tarlzmacmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 		// gzip
-		string_or_die(&tarcommand, TAR_COMMAND " --directory=%s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar.gz %s",
-			      empty, outdir, file->last_change, file->hash, file->hash);
-		if (system(tarcommand) != 0) {
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		string_or_die(&param1, "--directory=%s", empty);
+		string_or_die(&param2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
+		char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL };
+
+		if (system_argv(targzipcmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 #ifdef SWUPD_WITH_BZIP2
-		string_or_die(&tarcommand, TAR_COMMAND " --directory=%s " TAR_PERM_ATTR_ARGS " -jcf %s/%i/files/%s.tar.bz2 %s",
-			      empty, outdir, file->last_change, file->hash, file->hash);
-		if (system(tarcommand) != 0) {
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		string_or_die(&param1, "--directory=%s", empty);
+		string_or_die(&param2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
+		char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL };
+
+		if (system_argv(tarbzip2cmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 #endif
 
 		/* step 2b: pick the smallest of the three compression formats */
@@ -228,22 +244,20 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest)
 	struct file *tmp;
 
 	// presort by hash for easy deduplication
-	manifest->files = g_list_sort(manifest->files, file_sort_hash);
+	list = manifest->files = g_list_sort(manifest->files, file_sort_hash);
 
-	list = g_list_first(manifest->files);
-	while (prev == NULL) {
+	for (; list; list = g_list_next(list)) {
 		tmp = list->data;
-		list = g_list_next(list);
 
 		// find first new file
 		if (tmp->last_change == manifest->version) {
 			prev = tmp;
 			outfiles = g_list_prepend(outfiles, tmp);
+			break;
 		}
 	}
-	while (list) {
+	for (; list; list = g_list_next(list)) {
 		file = list->data;
-		list = g_list_next(list);
 
 		// add any new file having a unique hash
 		//FIXME: rename logic will be needed here
@@ -270,10 +284,11 @@ static void submit_fullfile_tasks(GList *files)
 	int ret;
 	int count = 0;
 	GError *err = NULL;
+	int numthreads = num_threads(3.0);
 
-	LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
+	LOG(NULL, "fullfile threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_fullfile_task, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN) * 3,
+				       numthreads,
 				       TRUE, NULL);
 
 	printf("Starting downloadable fullfiles data creation\n");
@@ -283,6 +298,11 @@ static void submit_fullfile_tasks(GList *files)
 		file = item->data;
 		item = g_list_next(item);
 
+		/* do not push ghosted files */
+		if (file->is_ghosted) {
+			continue;
+		}
+
 		ret = g_thread_pool_push(threadpool, file, &err);
 		if (ret == FALSE) {
 			printf("GThread create_fullfile_task push error\n");
@@ -301,17 +321,14 @@ void create_fullfiles(struct manifest *manifest)
 {
 	GList *deduped_file_list;
 	char *path;
-	char *cmd;
 	char *conf, *empty;
 
 	empty = config_empty_dir();
 
-	string_or_die(&cmd, "rm -rf %s", empty);
-	if (system(cmd) != 0) {
-		LOG(NULL, "Failed to run command:", "%s", cmd);
+	char *const rmcmd[] = { "rm", "-rf", empty, NULL };
+	if (system_argv(rmcmd) != 0) {
 		assert(0);
 	}
-	free(cmd);
 	g_mkdir_with_parents(empty, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 	free(empty);
 

src/globals.c

@@ -34,7 +34,6 @@
 int newversion = -1;
 int minversion = 0;
 unsigned long long int format = 0;
-bool enable_signing = false;
 
 char *state_dir = NULL;
 char *packstage_dir = NULL;

src/groups.c

@@ -37,6 +37,7 @@ static char **groups = NULL;
 static gsize groupcount = 0;
 static unsigned int groupcursor = 0;
 
+#if 0
 char *group_packages(char *group)
 {
 	assert(groupfile != NULL);
@@ -50,6 +51,14 @@ char *group_groups(char *group)
 
 	return g_key_file_get_value(groupfile, group, "groups", NULL);
 }
+#endif
+
+char *group_status(char *group)
+{
+	assert(groupfile != NULL);
+
+	return g_key_file_get_value(groupfile, group, "status", NULL);
+}
 
 void read_group_file(char *filename)
 {

src/helpers.c

@@ -101,22 +101,27 @@ void string_or_die(char **strp, const char *fmt, ...)
 	va_list ap;
 
 	va_start(ap, fmt);
-	if (vasprintf(strp, fmt, ap) <= 0) {
+	if (vasprintf(strp, fmt, ap) < 0) {
 		assert(0);
 	}
 	va_end(ap);
 }
 
-void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time)
+/* "step" should be a descriptive string for the process that completed before
+ * this function is called */
+void print_elapsed_time(const char *step, struct timeval *previous_time,
+			struct timeval *current_time)
 {
 	char *elapsed;
 
+	assert(strlen(step) > 0);
+
 	gettimeofday(current_time, NULL);
 	elapsed = get_elapsed_time(previous_time, current_time);
 	if (strcmp(elapsed, " ") == 0) {
-		printf("\t~0.0 elapsed for Manifest.full creation\n");
+		printf("\t~0.0 elapsed for '%s' step\n", step);
 	} else {
-		printf("\t%s elapsed for full chroot creation\n", elapsed);
+		printf("\t%s elapsed for '%s' step\n", elapsed, step);
 	}
 	previous_time->tv_sec = current_time->tv_sec;
 	previous_time->tv_usec = current_time->tv_usec;
@@ -144,45 +149,116 @@ void concat_str_array(char **output, char *const argv[])
 	}
 }
 
+int system_argv_pipe(char *const lhscmd[], char *const rhscmd[])
+{
+	return system_argv_pipe_fd(-1, -1, lhscmd, -1, -1, rhscmd);
+}
+
+int system_argv_pipe_fd(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+			int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	pid_t monitorpid = fork();
+	if (monitorpid == -1) {
+		LOG(NULL, "Failed to create child process to monitor pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		return -1;
+	} else if (monitorpid == 0) {
+		pipe_monitor(lnewstdinfd, lnewstderrfd, lhscmd, rnewstdoutfd, rnewstderrfd, rhscmd);
+	}
+	return wait_process_terminate(monitorpid);
+}
+
+void pipe_monitor(int lnewstdinfd, int lnewstderrfd, char *const lhscmd[],
+		  int rnewstdoutfd, int rnewstderrfd, char *const rhscmd[])
+{
+	int pipefd[2];
+	if (pipe(pipefd) == -1) {
+		LOG(NULL, "Failed to create a pipe between", "command %s and command %s", lhscmd[0], rhscmd[0]);
+		assert(0);
+	}
+
+	pid_t lhspid = system_argv_fd_nowait(lnewstdinfd, pipefd[1], lnewstderrfd, pipefd[0], lhscmd);
+	pid_t rhspid = system_argv_fd_nowait(pipefd[0], rnewstdoutfd, rnewstderrfd, pipefd[1], rhscmd);
+
+	if (close(pipefd[1]) == -1) {
+		LOG(NULL, "Could not close write end of pipe file descriptor", "%d", pipefd[1]);
+		assert(0);
+	}
+	if (close(pipefd[0]) == -1) {
+		LOG(NULL, "Could not close read end of pipe file descriptor", "%d", pipefd[0]);
+		assert(0);
+	}
+
+	int lhsresult = wait_process_terminate(lhspid);
+	int rhsresult = wait_process_terminate(rhspid);
+	exit(rhsresult != EXIT_SUCCESS ? rhsresult : lhsresult);
+}
+
 int system_argv(char *const argv[])
 {
-	int child_exit_status;
-	pid_t pid;
+	return system_argv_fd(-1, -1, -1, argv);
+}
+
+int system_argv_fd(int newstdinfd, int newstdoutfd, int newstderrfd, char *const cmd[])
+{
+	pid_t cmdpid = system_argv_fd_nowait(newstdinfd, newstdoutfd, newstderrfd, -1, cmd);
+	return wait_process_terminate(cmdpid);
+}
+
+pid_t system_argv_fd_nowait(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
+{
+	pid_t cmdpid = fork();
+	if (cmdpid == -1) {
+		LOG(NULL, "Failed to fork to execute command", "%s", cmd[0]);
+		assert(0);
+	} else if (cmdpid == 0) {
+		exec_cmd_fd(newstdinfd, newstdoutfd, newstderrfd, closefd, cmd);
+	}
+	return cmdpid;
+}
+
+void exec_cmd_fd(int newstdinfd, int newstdoutfd, int newstderrfd, int closefd, char *const cmd[])
+{
+	move_fd(newstdinfd, STDIN_FILENO);
+	move_fd(newstdoutfd, STDOUT_FILENO);
+	move_fd(newstderrfd, STDERR_FILENO);
+	if (closefd >= 0 && close(closefd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", closefd);
+		assert(0);
+	}
+	execvp(*cmd, cmd);
+	LOG(NULL, "Command", "%s failed", cmd[0]);
+	assert(0);
+}
+
+void move_fd(int oldfd, int newfd)
+{
+	if (oldfd < 0 || newfd < 0 || oldfd == newfd) {
+		return;
+	}
+	if (dup2(oldfd, newfd) == -1) {
+		LOG(NULL, "Could not create duplicate file descriptor", "%d from %d", newfd, oldfd);
+		assert(0);
+	}
+	if (close(oldfd) == -1) {
+		LOG(NULL, "Could not close file descriptor", "%d", oldfd);
+		assert(0);
+	}
+}
+
+int wait_process_terminate(pid_t pid)
+{
 	int status;
-
-	pid = fork();
-
-	if (pid == 0) { /* child */
-		execvp(*argv, argv);
-		LOG(NULL, "This line must not be reached", "");
-		assert(0);
-	} else if (pid < 0) {
-		LOG(NULL, "Failed to fork a child process", "");
-		assert(0);
+	do {
+		if (waitpid(pid, &status, 0) == -1) {
+			LOG(NULL, "Failed to wait for PID", "%d", pid);
+			return -1;
+		}
+	} while (!WIFEXITED(status) && !WIFSIGNALED(status));
+	// Exit statuses fall in the range of [0, 255].  Make signal statuses fall in a non-overlapping range starting with 256.
+	if (WIFEXITED(status)) {
+		return WEXITSTATUS(status);
 	} else {
-		pid_t ws = waitpid(pid, &child_exit_status, 0);
-
-		if (ws == -1) {
-			LOG(NULL, "Failed to wait for child process", "");
-			assert(0);
-		}
-
-		if (WIFEXITED(child_exit_status)) {
-			status = WEXITSTATUS(child_exit_status);
-		} else {
-			LOG(NULL, "Child process didn't exit", "");
-			assert(0);
-		}
-
-		if (status != 0) {
-			char *cmdline = NULL;
-
-			concat_str_array(&cmdline, argv);
-			LOG(NULL, "Failed to run command:", "%s", cmdline);
-			free(cmdline);
-		}
-
-		return status;
+		return 256 + WTERMSIG(status);
 	}
 }
 
@@ -195,3 +271,56 @@ void check_root(void)
 		exit(EXIT_FAILURE);
 	}
 }
+
+int num_threads(float scaling)
+{
+	const char *var = getenv("SWUPD_NUM_THREADS");
+	int result = sysconf(_SC_NPROCESSORS_ONLN) * scaling;
+
+	if (var && *var) {
+		char *endptr;
+		long int value;
+
+		errno = 0;
+		value = strtol(var, &endptr, 0);
+
+		if ((errno != 0 && value == 0) || *endptr) {
+			LOG(NULL, "SWUPD_NUM_THREADS must be an integer", "%s", var);
+		} else if ((errno == ERANGE && (value == LONG_MAX || value == LONG_MIN)) ||
+			   value < 1 || value > INT_MAX) {
+			LOG(NULL, "SWUPD_NUM_THREADS out of range", "%s", var);
+		} else {
+			result = (int)value;
+		}
+	}
+
+	return result;
+}
+
+/* This function is called when configuration specifies a ban on debuginfo files
+ * from manifests. Returns true if the passed file path matches the src or lib
+ * debuginfo configuration. */
+bool file_is_debuginfo(const char *path)
+{
+	bool ret = false;
+	char *lib;
+	char *src;
+
+	lib = config_debuginfo_path("lib");
+	src = config_debuginfo_path("src");
+
+	if (lib && (strncmp(path, lib, strlen(lib)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+	if (src && (strncmp(path, src, strlen(src)) == 0)) {
+		ret = true;
+		goto out;
+	}
+
+out:
+	free(lib);
+	free(src);
+	return ret;
+}

src/heuristics.c

@@ -45,6 +45,7 @@ static void runtime_state_heuristics(struct file *file)
 	/* these are shipped directories that are not themselves state,
 	 * rather only their contents are state */
 	if ((strcmp(file->filename, "/usr/src/debug") == 0) ||
+	    (strcmp(file->filename, "/usr/src/kernel") == 0) ||
 	    (strcmp(file->filename, "/dev") == 0) ||
 	    (strcmp(file->filename, "/home") == 0) ||
 	    (strcmp(file->filename, "/proc") == 0) ||
@@ -56,6 +57,12 @@ static void runtime_state_heuristics(struct file *file)
 		return;
 	}
 
+	/* the contents of these directory are not state,
+	 * but it belongs to a state directory  */
+	if ((strncmp(file->filename, "/usr/src/kernel/", 16) == 0)) {
+		return;
+	}
+
 	/* the contents of these directory are state, ideally this never
 	 * triggers if our package builds are clean */
 	if ((strncmp(file->filename, "/dev/", 5) == 0) ||
@@ -72,7 +79,7 @@ static void runtime_state_heuristics(struct file *file)
 		return;
 	}
 
-	/* these are commonly added directories for user customizatio,
+	/* these are commonly added directories for user customization,
 	 * ideally this never triggers if our package builds are clean */
 	if ((strncmp(file->filename, "/acct", 5) == 0) ||
 	    (strncmp(file->filename, "/cache", 6) == 0) ||

src/log.c

@@ -33,7 +33,7 @@
 
 #include "swupd.h"
 
-static FILE *logfile;
+static FILE *logfile[2];
 
 static struct timeval start_time;
 
@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end)
 {
 	char *filename;
 	string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end);
-	logfile = fopen(filename, "w");
+	logfile[0] = fopen(filename, "w");
 	free(filename);
 	gettimeofday(&start_time, NULL);
 }
 void init_log_stdout(void)
 {
-	logfile = stdout;
+	logfile[1] = stdout;
 	gettimeofday(&start_time, NULL);
 }
 
@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 	char *logstring = NULL;
 	char filebuf[4096];
 	char filebuf2[4096];
+	int i;
 
-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}
 
@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 		strcat(filebuf2, " ");
 	}
 
-	fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
-		(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+	for (i = 0; i < 2; i++) {
+		if (logfile[i]) {
+			fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
+				(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+			fflush(logfile[i]);
+		}
+	}
 
 	free(logstring);
 	free(buf);
-	fflush(logfile);
 }
 
 void close_log(int version, int exit_status)
@@ -133,7 +138,7 @@ void close_log(int version, int exit_status)
 	int t_sec;
 	int t_msec;
 
-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}
 
@@ -159,6 +164,8 @@ void close_log(int version, int exit_status)
 		printf("Update build failed for version %i\n", version);
 	}
 
-	fclose(logfile);
-	logfile = NULL;
+	if (logfile[0]) {
+		fclose(logfile[0]);
+		logfile[0] = NULL;
+	}
 }

src/make_fullfiles.c

@@ -23,6 +23,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <getopt.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,6 +33,7 @@
 
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
 	{ 0, 0, 0, 0 }
 };
@@ -42,6 +44,7 @@ static void usage(const char *name)
 	printf("   %s <version>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
 	printf("\n");
 }
@@ -56,6 +59,9 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument '%s'\n\n", optarg);
@@ -88,6 +94,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;

src/make_packs.c

@@ -25,8 +25,8 @@
 #include <dirent.h>
 #include <errno.h>
 #include <getopt.h>
-#include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -45,8 +45,8 @@ static void banner(void)
 
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -56,8 +56,8 @@ static void usage(const char *name)
 	printf("   %s <start version> <latest version> <bundle>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }
 
@@ -71,15 +71,15 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument ''%s'\n\n", optarg);
 				return false;
 			}
 			break;
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}
 
@@ -101,6 +101,11 @@ int main(int argc, char **argv)
 	int exit_status = EXIT_FAILURE;
 	char *file_path = NULL;
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;
@@ -114,12 +119,6 @@ int main(int argc, char **argv)
 	banner();
 	check_root();
 
-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		return exit_status;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	read_configuration_file(file_path);
 	free(file_path);
@@ -152,8 +151,6 @@ int main(int argc, char **argv)
 		exit_status = EXIT_SUCCESS;
 	}
 
-	signature_terminate();
-
 	printf("Pack creation %s (pack-%s %i to %li)\n",
 	       exit_status == EXIT_SUCCESS ? "complete" : "failed",
 	       module, start_version, end_version);

src/manifest.c

@@ -51,6 +51,10 @@ int file_sort_hash(gconstpointer a, gconstpointer b)
 	return memcmp(A->hash, B->hash, SWUPD_HASH_LEN - 1);
 }
 
+/* Standard sort compare function which sorts
+ * first on the version where the file changed and then
+ * on the name
+ */
 int file_sort_version(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
@@ -64,7 +68,7 @@ int file_sort_version(gconstpointer a, gconstpointer b)
 		return 1;
 	}
 
-	return strcmp(B->filename, A->filename);
+	return strcmp(A->filename, B->filename);
 }
 
 int file_sort_filename(gconstpointer a, gconstpointer b)
@@ -88,7 +92,7 @@ int file_sort_filename(gconstpointer a, gconstpointer b)
 	return 0;
 }
 
-struct manifest *alloc_manifest(int version, char *component)
+struct manifest *alloc_manifest(int version, char *component, GList *actions)
 {
 	struct manifest *manifest;
 
@@ -100,6 +104,7 @@ struct manifest *alloc_manifest(int version, char *component)
 	manifest->version = version;
 	manifest->component = strdup(component);
 	manifest->format = format;
+	manifest->actions = actions;
 
 	return manifest;
 }
@@ -129,7 +134,7 @@ struct manifest *manifest_from_file(int version, char *component)
 	if (infile == NULL) {
 		LOG(NULL, "Cannot read manifest", "%s (%s)\n", filename, strerror(errno));
 		free(filename);
-		return alloc_manifest(version, component);
+		return alloc_manifest(version, component, NULL);
 	}
 
 	/* line 1: MANIFEST\t<version> */
@@ -189,7 +194,7 @@ struct manifest *manifest_from_file(int version, char *component)
 		}
 	}
 
-	manifest = alloc_manifest(version, component);
+	manifest = alloc_manifest(version, component, NULL);
 	manifest->format = format_number;
 	manifest->prevversion = previous;
 	manifest->includes = includes;
@@ -235,10 +240,22 @@ struct manifest *manifest_from_file(int version, char *component)
 			assert(0); /* unknown file type */
 		}
 
-		if (c[1] == 'd') {
+		switch (c[1]) {
+		case 'd':
+			/* file is deleted */
 			file->is_deleted = 1;
-		} else if (c[1] != '.') {
-			assert(0); /* unknown deleted status */
+			break;
+		case 'g':
+			/* file is ghosted */
+			file->is_ghosted = 1;
+			break;
+		case '.':
+			/* no flag at this index */
+			break;
+		default:
+			/* unknown deleted status */
+			LOG(NULL, "Invalid flag at index 1", "%s", c[1]);
+			assert(0);
 		}
 
 		if (c[2] == 'C') {
@@ -306,6 +323,21 @@ struct manifest *manifest_from_file(int version, char *component)
 	return manifest;
 }
 
+/* do not set peer for the case where the old file was deleted and the new file
+ * was added back. This peer is normally set when the names match up and this
+ * triggers an attempt at creating a delta during pack creation. For this case
+ * where the old file is deleted or ghosted and is brought back we do not want
+ * this to be attempted, as the old file will not be present when trying to
+ * create the delta.
+ *
+ * We can go even further and not set peers for files when the old one did not
+ * exist (deleted or ghosted). It is not necessary to check the status of the
+ * new file */
+bool should_have_peer(struct file *file1, struct file *file2)
+{
+	return (!(file1->is_deleted || file1->is_ghosted));
+}
+
 void free_manifest(struct manifest *manifest)
 {
 	struct file *file;
@@ -323,6 +355,62 @@ void free_manifest(struct manifest *manifest)
 	free(manifest);
 }
 
+static bool same_file_contents(struct file *file1, struct file *file2)
+{
+	return (file1->is_dir == file2->is_dir &&
+		file1->is_link == file2->is_link &&
+		file1->is_deleted == file2->is_deleted &&
+		file1->is_file == file2->is_file &&
+		file1->is_config == file2->is_config &&
+		file1->is_state == file2->is_state &&
+		file1->is_boot == file2->is_boot &&
+		hash_compare(file1->hash, file2->hash));
+}
+
+/*
+ * Add a deleted file entry for it in the target list. However, since we're
+ * currently walking the list we HAVE to prepend the entry. Calling function
+ * should track to sort at the end.
+ */
+static void add_deleted_file(struct file *source, struct manifest *manifest)
+{
+	struct file *deleted;
+	deleted = calloc(1, sizeof(struct file));
+	if (deleted == NULL) {
+		assert(0);
+	}
+
+	deleted->filename = strdup(source->filename);
+	hash_set_zeros(deleted->hash);
+	deleted->is_deleted = 1;
+	deleted->is_config = source->is_config;
+	deleted->is_state = source->is_state;
+	deleted->is_boot = source->is_boot;
+	/* ghost deleted boot files */
+	deleted->is_ghosted = source->is_ghosted | source->is_boot & source->is_deleted;
+
+	if (deleted->is_ghosted || source->is_deleted) {
+		/* if the new file is ghosted or the file was deleted, preserve
+		 * hash (all zeros if source->is_deleted) and rename status */
+		deleted->is_rename = source->is_rename;
+		hash_assign(source->hash, deleted->hash);
+	}
+
+	/* for deleted files last_change remains the same.
+	 * otherwise last change is now */
+	deleted->last_change = source->is_deleted ? source->last_change : manifest->version;
+
+	deleted->peer = source;
+	source->peer = deleted;
+
+	/* if we are adding a deleted file we are walking the old and new
+	 * manifest files in-sync. we need to prepend this file in order to
+	 * not process it twice. This is why it is important for the calling
+	 * function to sort the list again at the end */
+	manifest->files = g_list_prepend(manifest->files, deleted);
+	manifest->count++;
+}
+
 /*
 backfill the "last changed" of each file in a manifest
 by comparing the hash against the same file in the previous manifest
@@ -337,7 +425,6 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 	struct file *file1, *file2;
 	int must_sort = 0;
 	int count = 0;
-	int first = 1;
 
 	if (!m1) {
 		printf("Matching manifests up failed: No old manifest!\n");
@@ -365,158 +452,72 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->is_deleted && file2->is_deleted && file1->is_rename) {
-				file2->is_rename = file1->is_rename;
-				hash_assign(file1->hash, file2->hash);
-			}
-
-			if (hash_compare(file1->hash, file2->hash) &&
-			    file1->is_dir == file2->is_dir &&
-			    file1->is_link == file2->is_link &&
-			    file1->is_deleted == file2->is_deleted &&
-			    file1->is_file == file2->is_file &&
-			    file1->is_config == file2->is_config &&
-			    file1->is_state == file2->is_state &&
-			    file1->is_boot == file2->is_boot &&
-			    file1->last_change >= minversion) {
+			/* file is present in both manifests */
+			if (same_file_contents(file1, file2) && file1->last_change >= minversion) {
+				/* file did not change */
 				file2->last_change = file1->last_change;
 				file2->is_rename = file1->is_rename;
 			} else {
+				/* file changed */
 				account_changed_file();
-				if (first) {
-					LOG(file1, "file changed", "");
-					first = 0;
-				}
 				count++;
 			}
 
-			if (!file1->is_deleted || file2->is_deleted) {
+			/* check if these files should be peers */
+			if (should_have_peer(file1, file2)) {
 				file1->peer = file2;
 				file2->peer = file1;
 			}
 
+			/* there was a match, advance both lists */
 			list1 = g_list_next(list1);
 			list2 = g_list_next(list2);
-			continue;
-		}
-		if (first) {
-			LOG(file1, "file added? ", "(file2 is %s)", file2->filename);
-			first = 0;
-		}
-		if (ret < 0) {
-			struct file *file3;
-			/*
-			 * if we get here, file1 got deleted... what we must do
-			 * is add a file entry for it in the target list.
-			 * However, since we're currently walking the list we
-			 * HAVE to prepend the entry.. and mark for sort at the
-			 * end.
-			 */
-			file3 = calloc(1, sizeof(struct file));
-			if (file3 == NULL) {
-				assert(0);
-			}
-
-			file3->filename = strdup(file1->filename);
-			hash_set_zeros(file3->hash);
-			file3->is_deleted = 1;
-			file3->is_config = file1->is_config;
-			file3->is_state = file1->is_state;
-			file3->is_boot = file1->is_boot;
-
-			if (!file1->is_deleted) {
-				file3->last_change = m2->version;
-			} else {
-				file3->last_change = file1->last_change;
-				file3->is_rename = file1->is_rename;
-				hash_assign(file1->hash, file3->hash);
-			}
-
-			file3->peer = file1;
-			file1->peer = file3;
-
-			list1 = g_list_next(list1);
-			m2->files = g_list_prepend(m2->files, file3);
-			m2->count++;
+		} else if (ret < 0) {
+			/* file1 was deleted, create entry for deleted file */
+			add_deleted_file(file1, m2);
 			if (!file1->is_deleted) {
 				account_deleted_file();
 				count++;
-				if (first) {
-					LOG(file1, "file got deleted", "");
-					first = 0;
-				}
 			}
+
 			must_sort = 1;
-			continue;
+			/* advance list1 for next file */
+			list1 = g_list_next(list1);
+		} else {
+			/* if we get here, ret is > 0, which means this is a new file added */
+			/* all we do is advance the pointer */
+			account_new_file();
+			count++;
+			/* advance list2 to check against same file in list1 */
+			list2 = g_list_next(list2);
 		}
-		/* if we get here, ret is > 0, which means this is a new file added */
-		/* all we do is advance the pointer */
-		account_new_file();
-		list2 = g_list_next(list2);
-		count++;
 	}
 
 	/* now deal with the tail ends */
-	while (list1) {
+	/* deleted files from list1 */
+	for (; list1; list1 = g_list_next(list1)) {
 		file1 = list1->data;
-
-		struct file *file3;
-
-		if (first) {
-			LOG(file1, "file changed tail", "");
-			first = 0;
-		}
-		count++;
-		/*
-		 * if we get here, file1 got deleted... what we must do is add
-		 * a file entry for it in the target list.  However, since
-		 * we're currently walking the list we HAVE to prepend the
-		 * entry.. and mark for sort at the end.
-		 */
-		file3 = calloc(1, sizeof(struct file));
-		if (file3 == NULL) {
-			assert(0);
-		}
-
-		file3->filename = strdup(file1->filename);
-		hash_set_zeros(file3->hash);
-		file3->is_deleted = 1;
-		file3->is_config = file1->is_config;
-		file3->is_state = file1->is_state;
-		file3->is_boot = file1->is_boot;
-
-		if (!file1->is_deleted) {
-			file3->last_change = m2->version;
-		} else {
-			file3->last_change = file1->last_change;
-			file3->is_rename = file1->is_rename;
-			hash_assign(file1->hash, file3->hash);
-		}
-
-		file3->peer = file1;
-		file1->peer = file3;
-
-		list1 = g_list_next(list1);
-		m2->files = g_list_prepend(m2->files, file3);
-		m2->count++;
+		add_deleted_file(file1, m2);
 		if (!file1->is_deleted) {
 			account_deleted_file();
+			count++;
 		}
+
 		must_sort = 1;
 	}
 
-	while (list2) {
+	/* added files from list2 */
+	for (; list2; list2 = g_list_next(list2)) {
 		account_new_file();
-		list2 = g_list_next(list2);
-		if (first) {
-			first = 0;
-		}
 		count++;
 	}
+
+	/* finally, sort the list if necessary */
 	if (must_sort) {
 		m2->files = g_list_sort(m2->files, file_sort_filename);
 	}
 
+	/* returned count of changed files */
 	return count;
 }
 
@@ -582,6 +583,16 @@ void subtract_manifests(struct manifest *m1, struct manifest *m2)
 			list1 = g_list_next(list1);
 			list2 = g_list_next(list2);
 
+			/* When both files are marked deleted, skip
+			 * subtraction. Preserving the deleted entries in both
+			 * manifests is required for 'swupd update' to know
+			 * when to delete the file, because the m2 bundle may
+			 * be installed with or without the m1 bundle.
+			 */
+			if (file1->is_deleted && file2->is_deleted) {
+				continue;
+			}
+
 			if (file1->is_deleted == file2->is_deleted && file1->is_file == file2->is_file) {
 				m1->files = g_list_delete_link(m1->files, todel);
 				m1->count--;
@@ -643,6 +654,10 @@ char *file_type_to_string(struct file *file)
 		type[1] = 'd';
 	}
 
+	if (file->is_ghosted) {
+		type[1] = 'g';
+	}
+
 	if (file->is_config) {
 		type[2] = 'C';
 	}
@@ -660,19 +675,29 @@ char *file_type_to_string(struct file *file)
 	return type;
 }
 
+/* Calculate the contentsize for the manifest based on file sizes.
+ *
+ * This should calculate the files uniquely included in this manifest, but none
+ * of its submanifests, which will allow calculation of sizes of all bundles on
+ * a system by adding all manifest->contentsizes of installed bundles.
+ *
+ * However, if two bundles not in the same include chain have overlapping
+ * content, summing the include chain of each bundle in the client will result
+ * in an over-estimation of the total size on the system. The more content is
+ * shared, the higher the over-estimation. In reality this overlap will not be
+ * large, but it is currently impossible to calculate the exact installed size
+ * using just the contentsize.
+ */
 static void compute_content_size(struct manifest *manifest)
 {
-	/* FIXME: this is a temporary implementation based on worst case */
-
 	GList *list;
 	struct file *file;
-	struct manifest *submanifest;
 
 	list = g_list_first(manifest->files);
 	while (list) {
 		file = list->data;
 		list = g_list_next(list);
-		if (!file->is_deleted && (file->last_change == manifest->version)) {
+		if (!file->is_deleted) {
 			if (file->is_file) {
 				manifest->contentsize += file->stat.st_size;
 			} else if (file->is_link) {
@@ -682,40 +707,6 @@ static void compute_content_size(struct manifest *manifest)
 			}
 		}
 	}
-
-	list = g_list_first(manifest->submanifests);
-	while (list) {
-		submanifest = list->data;
-		list = g_list_next(list);
-
-		/* Do not take into account groups not included in download content */
-		if (create_download_content_for_group(submanifest->component)) {
-			manifest->contentsize += submanifest->contentsize;
-		}
-	}
-}
-
-/* Returns 0 == success, -1 == failure */
-static int write_manifest_signature(struct manifest *manifest, const char *suffix)
-{
-	char *conf = config_output_dir();
-	char *filename = NULL;
-	int ret = -1;
-
-	if (conf == NULL) {
-		assert(0);
-	}
-	string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version,
-		      manifest->component, suffix);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	free(conf);
-	return ret;
 }
 
 /* Returns 0 == success, -1 == failure */
@@ -723,12 +714,16 @@ static int write_manifest_plain(struct manifest *manifest)
 {
 	GList *includes;
 	GList *list;
+	GList *actions;
 	struct file *file;
 	FILE *out = NULL;
 	char *base = NULL, *dir;
 	char *conf = config_output_dir();
 	char *filename = NULL;
+	char *status = NULL;
 	char *submanifest_filename = NULL;
+	char *manifest_tempdir = NULL;
+	char *tempmanifest = NULL;
 	int ret = -1;
 
 	if (conf == NULL) {
@@ -760,11 +755,23 @@ static int write_manifest_plain(struct manifest *manifest)
 	compute_content_size(manifest);
 	fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
 	includes = manifest->includes;
+	status = group_status(manifest->component);
+	if (status) {
+		fprintf(out, "status:\t%s\n", status);
+	}
 	while (includes) {
 		struct manifest *sub = includes->data;
 		includes = g_list_next(includes);
 		fprintf(out, "includes:\t%s\n", sub->component);
 	}
+
+	actions = manifest->actions;
+	while (actions) {
+		char *action = actions->data;
+		fprintf(out, "actions:\t%s\n", action);
+		actions = g_list_next(actions);
+	}
+
 	fprintf(out, "\n");
 
 	list = g_list_first(manifest->files);
@@ -776,24 +783,61 @@ static int write_manifest_plain(struct manifest *manifest)
 	}
 
 	list = g_list_first(manifest->manifests);
+	string_or_die(&manifest_tempdir, "%s/XXXXXX", state_dir);
+	if (!mkdtemp(manifest_tempdir)) {
+		LOG(NULL, "Failed to create temporary directory %s for untarring Manifests", manifest_tempdir);
+		assert(0);
+	}
+
 	while (list) {
 		file = list->data;
 		list = g_list_next(list);
 
 		string_or_die(&submanifest_filename, "%s/%i/Manifest.%s", conf, file->last_change, file->filename);
 		populate_file_struct(file, submanifest_filename);
-		ret = compute_hash(file, submanifest_filename);
+
+		if (file->is_deleted) {
+			goto write_entry;
+		}
+
+		char *param1, *param2;
+
+		/* Untar the Manifest.BUNDLE.tar and calculate the hash on that,
+		 * otherwise we may get incorrect hashes due to owner permissions on
+		 * the files. */
+		string_or_die(&param1, "%s", manifest_tempdir);
+		string_or_die(&param2, "%s.tar", submanifest_filename);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param1, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", param2, NULL };
+		ret = system_argv(tarcmd);
+		free(param1);
+		free(param2);
+
+		if (ret != 0) {
+			assert(0);
+		}
+
+		string_or_die(&tempmanifest, "%s/Manifest.%s", manifest_tempdir, file->filename);
+		populate_file_struct(file, tempmanifest);
+		ret = compute_hash(file, tempmanifest);
 		if (ret != 0) {
 			printf("Hash computation failed\n");
 			assert(0);
 		}
-
+		unlink(tempmanifest);
+		free(tempmanifest);
+	write_entry:
 		fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename);
 		free(submanifest_filename);
 	}
 
 	ret = 0;
 exit:
+	if (manifest_tempdir) {
+		if (rmdir(manifest_tempdir) != 0) {
+			LOG(NULL, "rmdir failed for %s: %s", manifest_tempdir, strerror(errno));
+		}
+		free(manifest_tempdir);
+	}
 	if (out) {
 		fclose(out);
 	}
@@ -807,34 +851,28 @@ exit:
 static int write_manifest_tar(struct manifest *manifest)
 {
 	char *conf = config_output_dir();
-	char *tarcmd = NULL;
-	int ret = -1;
+	char *directory, *manifesttar, *manifestcomp;
+	int ret = 0;
 
 	if (conf == NULL) {
 		assert(0);
 	}
 
+	string_or_die(&directory, "--directory=%s/%i", conf, manifest->version);
+	string_or_die(&manifesttar, "%s/%i/Manifest.%s.tar", conf, manifest->version, manifest->component);
+	string_or_die(&manifestcomp, "Manifest.%s", manifest->component);
+
 	/* now, tar the thing up for efficient full file download */
-	/* and put the signature of the plain manifest into the archive, too */
-	if (enable_signing) {
-		string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
-						   "%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed",
-			      conf, manifest->version, conf, manifest->version, manifest->component,
-			      manifest->component, manifest->component);
-	} else {
-		string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
-						   "%s/%i/Manifest.%s.tar Manifest.%s",
-			      conf, manifest->version, conf, manifest->version, manifest->component,
-			      manifest->component);
+	char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
+				 manifesttar, manifestcomp, NULL };
+	ret = system_argv(tarcmd);
+	if (ret) {
+		fprintf(stderr, "Creation of Manifest.tar failed\n");
 	}
 
-	if (system(tarcmd) != 0) {
-		fprintf(stderr, "Creation of Manifest.tar failed\n");
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(tarcmd);
+	free(directory);
+	free(manifesttar);
+	free(manifestcomp);
 	free(conf);
 	return ret;
 }
@@ -864,9 +902,7 @@ bool compute_hash_with_xattrs(const char *filename)
 int write_manifest(struct manifest *manifest)
 {
 	if (write_manifest_plain(manifest) == 0 &&
-	    write_manifest_signature(manifest, "") == 0 &&
-	    write_manifest_tar(manifest) == 0 &&
-	    write_manifest_signature(manifest, ".tar") == 0) {
+	    write_manifest_tar(manifest) == 0) {
 		return 0;
 	}
 	return -1;
@@ -926,13 +962,12 @@ bool changed_includes(struct manifest *old, struct manifest *new)
 	return false;
 }
 
-/* For a format bump, it's convenient to remove deleted files from manifests
- * that last changed prior to the format bump, since they are no longer
- * considered for deletion as part of an update.
- *
+/* Remove files deprecated in this version as identified by the compfunc
+ * function. This function can be used to remove old deleted files over a format
+ * bump and remove deprecated ghosted files.
  * Note: this function should be called after match_manifests().
  */
-int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
+int remove_deprecated_files(struct manifest *m1, struct manifest *m2, bool (*compfunc)(struct file *file1, struct file *file2))
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
@@ -948,11 +983,6 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 		return -1;
 	}
 
-	/* This is the common case, so bail early, reporting no deletions */
-	if (m1->format == m2->format) {
-		return 0;
-	}
-
 	/* At this point, the manifest formats mismatch, and it's assumed that
 	 * m1 is the old manifest, and m2 is the new.
 	 */
@@ -970,7 +1000,9 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->is_deleted && file2->is_deleted) {
+			/* use the comparison function passed in to determine if this file
+			 * should be removed */
+			if (compfunc(file1, file2)) {
 				GList *to_delete = list2;
 				list1 = g_list_next(list1);
 				list2 = g_list_next(list2);
@@ -991,6 +1023,36 @@ int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
 	return count;
 }
 
+/* Performs a final link of renames in the manifest to catch any renames from
+ * old versions and then removes any orphaned renames */
+void clean_renames(struct manifest *manifest)
+{
+	GList *list;
+	struct file *file;
+
+	/* make sure all renames are linked, this is necessary for renames
+	 * from old manifests that carry over to the current one */
+	final_link(manifest->files);
+
+	list = g_list_first(manifest->files);
+	while (list) {
+		file = list->data;
+		/* if a file is marked as a rename but has lost its rename_peer
+		 * it needs to be cleaned up */
+		if (file->is_rename && !file->rename_peer) {
+			/* no longer a rename */
+			file->is_rename = 0;
+			/* if the file is marked as deleted and renamed it is a
+			 * renamed-from file. Mark these as deleted now */
+			if (file->is_deleted) {
+				hash_set_zeros(file->hash);
+			}
+		}
+
+		list = g_list_next(list);
+	}
+}
+
 /* Conditionally remove some things from a manifest.
  * Returns > 0 when the pruned manifest has new files.
  * Returns 0 when the pruned manifest no longer has new files.
@@ -1015,8 +1077,16 @@ int prune_manifest(struct manifest *manifest)
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		} else if (file->is_boot && file->is_deleted) {
-			// only expose the current best boot files, a client side entity can manage /boot's actual contents
-			// LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component);
+			/* mark boot files that are going away as ghosted, these will be
+			 * cleaned up with the next update */
+			file->is_deleted = 0;
+			file->is_ghosted = 1;
+		} else if (config_ban_debuginfo() && file_is_debuginfo(file->filename)) {
+			/* The configuration option to ban debuginfo from the manifests was
+			 * set in server.ini via the [Debuginfo][banned] option. Although
+			 * debuginfo additions are banned via analyze_fs, prune it here
+			 * to insure mistakenly included debuginfo from old versions is
+			 * removed from the manifests. */
 			manifest->files = g_list_delete_link(manifest->files, list);
 			manifest->count--;
 		}
@@ -1068,9 +1138,6 @@ void create_manifest_delta(int oldversion, int newversion, char *module)
 			}
 			LOG(NULL, "Failed to rename", "");
 		}
-		if (!signature_sign(outfile)) {
-			fprintf(stderr, "Creating signature for '%s' failed\n", outfile);
-		}
 	} else {
 		sleep(1); /* we raced. whatever. sleep for a bit to get the other guy to make progress */
 	}

src/pack.c

@@ -40,31 +40,29 @@
 
 static void empty_pack_stage(int full, int from_version, int to_version, char *module)
 {
-	char *cmd;
+	char *param;
 	char *path;
-	int ret;
 
 	// clean any stale data (eg: re-run after a failure)
-	string_or_die(&cmd, "rm -rf %s/%s/%i_to_%i/", packstage_dir, module,
-		      from_version, to_version);
-	ret = system(cmd);
-	if (ret) {
+	string_or_die(&param, "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version);
+	char *const rmcmd[] = { "rm", "-fr", param, NULL };
+	if (system_argv(rmcmd) != 0) {
 		fprintf(stderr, "Failed to clean %s/%s/%i_to_%i\n",
 			packstage_dir, module, from_version, to_version);
-		free(cmd);
+		free(param);
 		exit(EXIT_FAILURE);
 	}
-	free(cmd);
+	free(param);
 
 	if (!full) {
 		// (re)create module/version/{delta,staged}
 		string_or_die(&path, "%s/%s/%i_to_%i/delta", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 		string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 			      from_version, to_version);
-		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+		g_mkdir_with_parents(path, S_IRWXU);
 		free(path);
 	}
 }
@@ -81,7 +79,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 
 	string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
 		      from_version, to_version);
-	g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
+	g_mkdir_with_parents(path, S_IRWXU);
 	dir = opendir(path);
 	if (!dir) {
 		fprintf(stderr, "There are problems accessing %s, exiting\n", path);
@@ -89,7 +87,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 	}
 	free(path);
 	while (1) {
-		char *path, *tar;
+		char *path, *param;
 		int ret;
 
 		entry = readdir(dir);
@@ -117,13 +115,12 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 		 * the resulting pack is slightly smaller, and in addition, we're saving CPU
 		 * time on the client...
 		 */
-		string_or_die(&tar, TAR_COMMAND " --directory=%s/%s/%i_to_%i/staged " TAR_WARN_ARGS " " TAR_PERM_ATTR_ARGS " -xf %s",
-			      packstage_dir, module, from_version, to_version, path);
-		ret = system(tar);
-		if (!ret) {
+		string_or_die(&param, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
+		if (system_argv(tarcmd) == 0) {
 			unlink(path);
 		}
-		free(tar);
+		free(param);
 		free(path);
 	}
 	closedir(dir);
@@ -141,13 +138,14 @@ static void prepare_pack(struct packdata *pack)
 		return;
 	}
 
+	/* read in manifest from file */
 	pack->end_manifest = manifest_from_file(pack->to, pack->module);
-
+	/* wipe any old packs (failed) and re-create pack directory structure */
 	empty_pack_stage(0, pack->from, pack->to, pack->module);
-
+	/* match up old and new manifests */
 	match_manifests(manifest, pack->end_manifest);
-
-	link_renames(pack->end_manifest->files, manifest);
+	/* link renames together */
+	link_renames(pack->end_manifest->files, pack->to);
 }
 
 static void make_pack_full_files(struct packdata *pack)
@@ -164,7 +162,11 @@ static void make_pack_full_files(struct packdata *pack)
 	while (item) {
 		file = item->data;
 		item = g_list_next(item);
-		if ((!file->peer || file->peer->is_deleted) && !file->is_deleted && !file->rename_peer) {
+		/* only create full files if renames or deltas are not appropriate */
+		if ((!file->peer || file->peer->is_deleted || file->peer->is_ghosted) &&
+		    !file->is_deleted &&  /* no full-files for deletes */
+		    !file->is_ghosted &&  /* no full-files for ghosts */
+		    !file->rename_peer) { /* no full-files for renames */
 			char *from, *to;
 			char *fullfrom, *fullto;
 
@@ -251,16 +253,21 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 		file = item->data;
 		item = g_list_next(item);
 
+		/* skip old files, files without a peer, and files that are not
+		 * files, directories, or links */
 		if ((file->last_change <= pack->from) ||
 		    (!file->peer) ||
 		    (!file->is_file && !file->is_dir && !file->is_link)) {
 			continue;
 		}
 
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
 
+		/* check for existence */
 		ret = stat(from, &stat_delta);
+		/* only add if delta does not already exist and the file is not
+		 * in files */
 		if (ret && !find_file_in_list(files, file)) {
 			files = g_list_prepend(files, file);
 		}
@@ -277,7 +284,7 @@ static void create_delta(gpointer data, __unused__ gpointer user_data)
 
 	/* if the file was not found in the from version, skip delta creation */
 	if (file->peer) {
-		__create_delta(file, file->peer->last_change);
+		__create_delta(file, file->peer->last_change, file->peer->hash);
 	}
 }
 
@@ -288,10 +295,11 @@ static void make_pack_deltas(GList *files)
 	struct file *file;
 	int ret;
 	GError *err = NULL;
+	int numthreads = num_threads(1.0);
 
-	LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
+	LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_delta, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
+				       numthreads, FALSE, NULL);
 
 	item = g_list_first(files);
 	while (item) {
@@ -310,31 +318,13 @@ static void make_pack_deltas(GList *files)
 	g_thread_pool_free(threadpool, FALSE, TRUE);
 }
 
-/* Returns 0 == success, -1 == failure */
-static int write_pack_signature(struct packdata *pack)
-{
-	char *filename = NULL;
-	int ret = -1;
-
-	string_or_die(&filename, "%s/%i/pack-%s-from-%i.tar",
-		      staging_dir, pack->to, pack->module, pack->from);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	return ret;
-}
-
 /* Returns 0 == success, other == failure */
 static int make_final_pack(struct packdata *pack)
 {
 	GList *item;
 	struct file *file;
 	int ret;
-	char *tar;
+	char *param1, *param2;
 	double penalty;
 
 	LOG(NULL, "make_final_pack", "%s: %i to %i", pack->module, pack->from, pack->to);
@@ -356,11 +346,11 @@ static int make_final_pack(struct packdata *pack)
 
 		/* for each file changed since <X> */
 		/* locate delta, check if the diff it's from is >= <X> */
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
-		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s", packstage_dir,
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
+		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s-%s", packstage_dir,
 			      pack->module, pack->from, pack->to, file->peer->last_change,
-			      file->last_change, file->hash);
+			      file->last_change, file->peer->hash, file->hash);
 		string_or_die(&tarfrom, "%s/%i/files/%s.tar", staging_dir,
 			      file->last_change, file->hash);
 		string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
@@ -438,6 +428,8 @@ static int make_final_pack(struct packdata *pack)
 		explode_pack_stage(pack->from, pack->to, pack->module);
 	}
 
+	char *bundle_delta = NULL;
+
 	/* now... link in the Manifest pack */
 	if (pack->from != 0) {
 		char *from, *to;
@@ -457,14 +449,18 @@ static int make_final_pack(struct packdata *pack)
 
 		ret = link(from, to);
 		if (ret) {
-			LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from);
+			LOG(NULL, "Failed to link", "Manifest-%s-delta-from-%i (%s)", pack->module, pack->from, strerror(errno));
+		} else {
+			string_or_die(&bundle_delta, "Manifest-%s-delta-from-%i", pack->module, pack->from);
 		}
 
 		free(from);
 		free(to);
 	}
 
-	/* now... link in the MoM Manifest into the base pack */
+	char *mom_delta = NULL;
+
+	/* now... link in the MoM Manifest into the os-core pack */
 	if ((pack->from != 0) && (strcmp(pack->module, "os-core") == 0)) {
 		char *from, *to;
 		struct stat st;
@@ -482,7 +478,9 @@ static int make_final_pack(struct packdata *pack)
 
 		ret = link(from, to);
 		if (ret) {
-			LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from);
+			LOG(NULL, "Failed to link", "Manifest-MoM-delta-from-%i (%s)", pack->from, strerror(errno));
+		} else {
+			string_or_die(&mom_delta, "Manifest-MoM-delta-from-%i", pack->from);
 		}
 
 		free(from);
@@ -491,24 +489,17 @@ static int make_final_pack(struct packdata *pack)
 
 	/* tar the staging directory up */
 	LOG(NULL, "starting tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to);
-	string_or_die(&tar, TAR_COMMAND " " TAR_PERM_ATTR_ARGS " --directory=%s/%s/%i_to_%i/ "
-					"--numeric-owner -Jcf %s/%i/pack-%s-from-%i.tar delta staged",
-		      packstage_dir, pack->module, pack->from, pack->to, staging_dir, pack->to,
-		      pack->module, pack->from);
-	ret = system(tar);
-	free(tar);
+	string_or_die(&param1, "%s/%s/%i_to_%i/", packstage_dir, pack->module, pack->from, pack->to);
+	string_or_die(&param2, "%s/%i/pack-%s-from-%i.tar", staging_dir, pack->to, pack->module, pack->from);
+	char *const tarcmd[] = { TAR_COMMAND, "-C", param1, TAR_PERM_ATTR_ARGS_STRLIST, "--numeric-owner", "-Jcf", param2, "delta", "staged", bundle_delta, mom_delta, NULL };
+	ret = system_argv(tarcmd);
+	free(param1);
+	free(param2);
 	LOG(NULL, "finished tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to);
 	/* FIXME: this is a hack workaround, needs diagnosed and removed */
-	if ((ret != 0) && (ret != 256)) {
+	if ((ret != 0) && (ret != 1)) {
 		fprintf(stderr, "Unexpected return value (%d) creating tar of pack %s from %i to %i\n",
 			ret, pack->module, pack->from, pack->to);
-	} else {
-		/* Write the signature file */
-		ret = write_pack_signature(pack);
-		if (ret != 0) {
-			fprintf(stderr, "Failure creating signature of pack %s from %i to %i\n",
-				pack->module, pack->from, pack->to);
-		}
 	}
 
 	/* and clean up */

src/rename.c

@@ -21,6 +21,9 @@
  *
  */
 
+/* Rename detection and support.
+ */
+
 #define _GNU_SOURCE
 #include <assert.h>
 #include <ctype.h>
@@ -37,8 +40,55 @@
 
 #include <magic.h>
 
-static magic_t mcookie;
+static bool samefiletype(char *t1, char *t2)
+{
+	if (t1 && (t1 == t2) && *t1) {
+		return true;
+	}
+	return false;
+}
+/* For an elf binary, we get the BuildID in it, which
+ * is essentially a hash of the original loaded sections
+ * and hence is unique. This hardly makes for a 'type'
+ */
+static char *getmagic(char *filename)
+{
+	static magic_t mcookie;
+	static GStringChunk *typestore;
+	char *c2;
+	char *c1;
+	if (mcookie == NULL) {
+		mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS);
+		magic_load(mcookie, NULL);
+		typestore = g_string_chunk_new(200);
+	}
 
+	c1 = (char *)magic_file(mcookie, filename);
+	if (!c1) {
+		LOG(NULL, "Cannot find file type", "%s", filename);
+		c1 = "";
+	}
+	c1 = strdup(c1);
+	c2 = strstr(c1, ", BuildID[");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = strstr(c1, "not stripped");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = strstr(c1, "stripped");
+	if (c2) {
+		*c2 = 0;
+	}
+	c2 = g_string_chunk_insert_const(typestore, c1);
+	free(c1);
+	return c2;
+}
+
+/* Assign a score roughly in the range -100 to 1000 to express how similar
+ * two files are.
+ */
 double rename_score(struct file *old, struct file *new)
 {
 	double score = 0.0;
@@ -50,6 +100,16 @@ double rename_score(struct file *old, struct file *new)
 		score += 400;
 	}
 
+	/* If the files are smaller than about 200 bytes then even a
+	 * single byte change using bsdiff is going to work out as
+	 * bigger than just shipping the new file, so stop if they are
+	 * not the same. No point in running up bsdiff just for the
+	 * sake of it.
+	 */
+	if (new->stat.st_size < BSDIFFSIZE) {
+		return -99.0;
+	}
+
 	/* points for being in the same directory */
 	if (strcmp(old->dirname, new->dirname) == 0) {
 		score += 10;
@@ -86,12 +146,14 @@ double rename_score(struct file *old, struct file *new)
 		in--;
 	}
 
+#if 0
 	/* if both start with /boot/vmlinuz give it a boost; this is a local hack due to vmlinuz being very short */
 	if (strncmp(old->filename, "/boot/vmlinuz", 13) == 0 && strncmp(new->filename, "/boot/vmlinuz", 13) == 0) {
 		score += 80;
 	}
 
 	/* if ELF, points for sharing the same soname to the first dot */
+#endif
 
 	/* negative points for not being within 25%+/-1Kb of the same file size */
 	if (old->stat.st_size > ((new->stat.st_size * 1.25) + 1024)) {
@@ -115,66 +177,40 @@ double rename_score(struct file *old, struct file *new)
 	}
 	/* negative points for not having the same 'file' type */
 
-	if (old->filetype && new->filetype && strcmp(old->filetype, new->filetype) != 0) {
+	if (!samefiletype(old->filetype, new->filetype)) {
 		score -= 60;
 	}
 
 	return score;
 }
 
-static void precompute_file_data(struct manifest *manifest, struct file *file, int old_rename, GList *last_versions_list)
+static void precompute_file_data(int version, const char *component, struct file *file)
 {
-	GList *item;
 	char *c1, *c2;
 	char *filename = NULL;
-	int last_change;
-	struct stat buf;
 
+	assert(file);
 	/* fill in the filename-minus-the-numbers field */
 	file->alpha_only_filename = calloc(strlen(file->filename) + 1, sizeof(char));
 
 	c1 = file->filename;
 	c2 = file->alpha_only_filename;
-	while (*c1) {
-		while (*c1 && !isalpha(*c1))
-			c1++;
-		if (!*c1) {
-			break;
-		}
-		if (c2 == NULL) {
-			break;
-		}
-		*c2 = *c1;
-		c1++;
-		c2++;
-	}
-
-	if (manifest) {
-		string_or_die(&filename, "%s/%i/%s/%s", image_dir, manifest->version, manifest->component, file->filename);
-	} else if (old_rename) {
-		item = g_list_first(last_versions_list);
-		while (item) {
-			last_change = GPOINTER_TO_INT(item->data);
-			item = g_list_next(item);
-
-			free(filename);
-			string_or_die(&filename, "%s/%i/full/%s", image_dir, last_change, file->filename);
-			if (!lstat(filename, &buf)) {
-				break;
+	if (c2) {
+		for (; *c1; c1++) {
+			if (isalpha(*c1)) { /* Only copy letters */
+				*c2++ = *c1;
 			}
 		}
-	} else {
-		string_or_die(&filename, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+		/* alpha_only_filename is NUL terminated by calloc */
 	}
+	string_or_die(&filename, "%s/%i/%s/%s", image_dir, version, component, file->filename);
 
 	/* make sure file->stat.st_size is valid */
 	if (file->stat.st_size == 0) {
 		int ret;
+		struct stat buf;
 
-		if (filename == NULL) {
-			printf("filename is null...impossible to stat\n");
-			assert(0);
-		}
+		assert(filename);
 		ret = lstat(filename, &buf);
 		if (!ret) {
 			file->stat.st_size = buf.st_size;
@@ -182,27 +218,19 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i
 			printf("Stat failure on %s\n", filename);
 		}
 	}
-
-	c1 = (char *)magic_file(mcookie, filename);
-	if (c1) {
-		char *c2;
-		file->filetype = strdup(c1);
-		c2 = strstr(file->filetype, "not stripped");
-		if (c2) {
-			*c2 = 0;
-		}
-		c2 = strstr(file->filetype, "stripped");
-		if (c2) {
-			*c2 = 0;
-		}
+	if (file->stat.st_size < BSDIFFSIZE) {
+		/* thing is too small, always will be regenerated
+		 * so no point in trying to figure out what kind
+		 * of file it is
+		 */
 	} else {
-		LOG(file, "Cannot find file type", "%s", filename);
+		file->filetype = getmagic(filename);
 	}
 
 	free(filename);
 
-	file->basename = strdup(basename(file->filename));
-	file->dirname = strdup(dirname(file->filename));
+	file->basename = g_path_get_basename(file->filename);
+	file->dirname = g_path_get_dirname(file->filename);
 }
 
 int file_sort_score(gconstpointer a, gconstpointer b)
@@ -222,6 +250,10 @@ int file_sort_score(gconstpointer a, gconstpointer b)
 	return 0;
 }
 
+/* compare file to each deleted file.
+ * set file->rename_peer to best matched deleted file
+ * set file->rename_score to the score
+ */
 static void score_file(GList *deleted_files, struct file *file)
 {
 	GList *list2;
@@ -246,61 +278,107 @@ static void score_file(GList *deleted_files, struct file *file)
 	}
 }
 
-void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list)
+/* delete the first element of the list and return the new head */
+static GList *del_first(GList *list)
 {
-	GList *new_files = NULL;
-	GList *deleted_files = NULL;
+	/* The first list is the pointer to the list, the second is
+	 * the pointer to what to delete */
+	return g_list_delete_link(list, list);
+}
+
+/* Take a list, return a new list where the filter function returns true */
+static GList *new_filtered_list(GList *list, int version, int (*f)(struct file *file, int version))
+{
+	/* make a list of new files, no peer */
+	GList *newlist = NULL;
+	list = g_list_first(list);
+	for (; list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		if (f(file, version)) {
+			newlist = g_list_prepend(newlist, file);
+		}
+	}
+	return newlist;
+}
+
+static int renamed_file_p(struct file *file, int unused __attribute__((unused)))
+{
+	return file->is_rename;
+}
+/* Return a new list of renamed files */
+static GList *new_list_renamed_files(GList *infiles)
+{
+	return new_filtered_list(infiles, 0, renamed_file_p);
+}
+
+/* Predicate that returns true if this is a new file in the stated version */
+static int new_file_p(struct file *file, int version)
+{
+	if ((file->last_change != version) ||
+	    (file->is_deleted) ||
+	    (!file->is_file) ||
+	    (file->peer)) {
+		return 0;
+	}
+	return 1;
+}
+
+/* return a new list of the new files */
+static GList *list_new_files(struct manifest *manifest)
+{
+	GList *list = new_filtered_list(manifest->files, manifest->version, new_file_p);
+	/* call precompute_file_data for each file on list, return the list */
+	GList *ret = list;
+	for (list = g_list_first(list); list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		precompute_file_data(manifest->version, manifest->component, file);
+	}
+	return ret;
+}
+
+static int deleted_p(struct file *file, int version)
+{
+	if ((!file->is_deleted) ||
+	    (!file->peer) ||
+	    (file->last_change != version) ||
+	    (file->peer->is_dir || file->peer->is_link)) {
+		return 0;
+	}
+	return 1;
+}
+
+static GList *list_deleted_files(struct manifest *manifest)
+{
+	GList *list = new_filtered_list(manifest->files, manifest->version, deleted_p);
+	GList *ret = list;
+	/* call precompute_file_data for each peer of file on list */
+	for (list = g_list_first(list); list; list = g_list_next(list)) {
+		struct file *file = list->data;
+		struct file *peer = file->peer;
+		/* Need to get things from the /full/ as we do not know
+		 * which  component may be coming from? */
+		precompute_file_data(peer->last_change, "full", peer);
+	}
+	return ret;
+}
+
+void rename_detection(struct manifest *manifest)
+{
+	GList *new_files;
+	GList *deleted_files;
 
 	GList *list;
 	struct file *file;
-	int old_rename = 0;
 
-	if (last_change != manifest->version) {
-		old_rename = 1;
-	}
+	new_files = list_new_files(manifest);
 
-	if (mcookie == NULL) {
-		mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS);
-		magic_load(mcookie, NULL);
-	}
-
-	/* make a list of new files, no peer */
-	list = g_list_first(manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
-		if ((file->last_change != manifest->version) ||
-		    (file->is_deleted) ||
-		    (!file->is_file) ||
-		    (file->peer)) {
-			continue;
-		}
-
-		new_files = g_list_prepend(new_files, file);
-		precompute_file_data(manifest, file, old_rename, last_versions_list);
-	}
-
-	/* if there are no new files, we're not having any renames -- early exit */
+	/* no new files --> no renames -- early exit */
 	if (!new_files) {
 		LOG(NULL, "No new files, no rename detection", "%s", manifest->component);
 		return;
 	}
-	/* make a list of newly deleted files that have a peer */
 
-	list = g_list_first(manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
-		if ((!file->is_deleted) ||
-		    (!file->peer) ||
-		    (file->last_change != last_change) ||
-		    (file->peer->is_dir || file->peer->is_link)) {
-			continue;
-		}
-
-		deleted_files = g_list_prepend(deleted_files, file);
-		precompute_file_data(NULL, file->peer, old_rename, last_versions_list);
-	}
+	deleted_files = list_deleted_files(manifest);
 
 	/* nothing got deleted --> no renames --> early exit */
 	if (!deleted_files) {
@@ -309,7 +387,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		return;
 	}
 
-	/* for each new file, find the deleted file with the highest score, and store the score */
+	/* for each new file, find the deleted file with the highest score,
+	 * store it in file->rename_peer and store the score */
 	list = g_list_first(new_files);
 	while (list) {
 		file = list->data;
@@ -317,20 +396,24 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		score_file(deleted_files, file);
 	}
 
+redo:
 	/* sort all new files by score */
-
+	/* walk the sorted score list.
+	 * pick the top score,
+	 * check if the score is still valid,
+	 * if not, recompute the score and resort
+	 * This is probably an O(n^3).
+	 */
 	new_files = g_list_sort(new_files, file_sort_score);
 
-	/* pick the top score, check if the score is still valid, if not, recompute the score and resort */
-
-	while (new_files) {
+	for (; new_files; new_files = del_first(new_files)) {
 		file = new_files->data;
+		if (file->rename_peer == NULL) {
+			continue;
+		}
 
-		if (file->rename_score < 15.0 || file->rename_peer == NULL) {
-			new_files = g_list_delete_link(new_files, new_files);
-			if (file->rename_peer) {
-				LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
-			}
+		if (file->rename_score < 15.0) {
+			LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
 			continue;
 		}
 
@@ -338,10 +421,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 			/* the candidate peer got already taken by another file! */
 			LOG(NULL, "Rename not done due to target already taken", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
 			file->rename_peer = NULL;
-			file->rename_score = -100;
 			score_file(deleted_files, file);
-			new_files = g_list_sort(new_files, file_sort_score);
-			continue;
+			goto redo;
 		}
 
 		/* if valid and score is high enough, make the link by setting the flag and storing the hash */
@@ -349,13 +430,14 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 		LOG(NULL, "Rename detected!", "%s -> %s   score %4.1f", file->rename_peer->filename, file->filename, file->rename_score);
 
 		file->rename_peer->rename_peer = file;
-		/* must delete the file from the deleted list */
+		/* must remove the file from the deleted list */
 		deleted_files = g_list_remove(deleted_files, file->rename_peer);
 		hash_assign(file->hash, file->rename_peer->hash);
 		file->is_rename = 1;
 		file->rename_peer->is_rename = 1;
-
-		new_files = g_list_delete_link(new_files, new_files);
+		if (!deleted_files) {
+			break; /* No more deleted files to rename */
+		}
 
 	} /* lather, rinse, repeat until all files have a target */
 
@@ -364,65 +446,89 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve
 	g_list_free(deleted_files);
 }
 
-static int file_found_in_older_manifest(struct manifest *from_manifest, struct file *searched_file)
+void final_link(GList *files)
 {
-	GList *list;
-	struct file *file;
+	GList *list1, *list2;
+	struct file *file1, *file2;
 
-	list = g_list_first(from_manifest->files);
-	while (list) {
-		file = list->data;
-		list = g_list_next(list);
+	list1 = new_list_renamed_files(files);
+	list1 = list2 = g_list_sort(list1, file_sort_version);
 
-		if (file->is_deleted) {
+	for (; list1; list1 = g_list_next(list1)) {
+		file1 = list1->data;
+		/* do not re-link already-linked renames */
+		if (file1->rename_peer != NULL) {
 			continue;
 		}
-		if (!strcmp(file->filename, searched_file->filename)) {
-			return 1;
+
+		/* check the rest of the list */
+		list2 = g_list_next(list1);
+		for (; list2; list2 = g_list_next(list2)) {
+			file2 = list2->data;
+			/* must be the same version to link */
+			if (file1->last_change != file2->last_change) {
+				continue;
+			}
+
+			/* need one and only one to be deleted to link as this
+			 * indicates a renamed-from/renamed-to relationship
+			 * (.d.r -> F..r) */
+			if (file1->is_deleted == file2->is_deleted) {
+				continue;
+			}
+
+			/* must have same hash to link */
+			if (!hash_compare(file2->hash, file1->hash)) {
+				continue;
+			}
+
+			file1->rename_peer = file2;
+			file2->rename_peer = file1;
 		}
 	}
-
-	return 0;
 }
 
-void link_renames(GList *newfiles, struct manifest *from_manifest)
+/* What do we need this for?
+ *
+ * rename_detection has already set up the links in the manifest it
+ * was given.
+ *
+ */
+void link_renames(GList *newfiles, int to_version)
 {
 	GList *list1, *list2;
 	GList *targets;
 	struct file *file1, *file2;
 
-	targets = newfiles = g_list_sort(newfiles, file_sort_version);
+	targets = new_list_renamed_files(newfiles);
+	/* TODO: Check that g_list_sort is reasonable speed */
+	targets = newfiles = g_list_sort(targets, file_sort_version);
 
-	list1 = g_list_first(newfiles);
-
-	/* todo: sort newfiles and targets by hash */
-
-	while (list1) {
+	for (list1 = newfiles; list1; list1 = g_list_next(list1)) {
 		file1 = list1->data;
-		list1 = g_list_next(list1);
 
-		if ((file1->peer || !file1->is_rename) ||
-		    (file1->is_deleted)) {
+		if (file1->peer || file1->is_deleted) {
 			continue;
 		}
-		/* now, file1 is the new file that got renamed. time to search the rename targets */
+		/* now, file1 is the new file that got renamed.
+		 * time to search the rename targets */
 		list2 = g_list_first(targets);
-		while (list2) {
+		for (; list2; list2 = g_list_next(list2)) {
 			file2 = list2->data;
-			list2 = g_list_next(list2);
-
-			if ((!file2->peer || !file2->is_rename) ||
+			/* This is like deleted_p but not quite */
+			/* deleted_p returns false for directories and links */
+			if (!file2->peer ||
 			    (!file2->is_deleted) ||
-			    (!file_found_in_older_manifest(from_manifest, file2))) {
+			    (file2->last_change != to_version)) {
 				continue;
 			}
 			if (hash_compare(file2->hash, file1->hash)) {
 				file1->rename_peer = file2->peer;
 				file1->peer = file2->peer;
 				file2->peer->rename_peer = file1;
-				list2 = NULL;
+				break;
 			}
 		}
 	}
-	free(from_manifest);
+	g_list_free(targets);
 }

src/signature.c

@@ -1,171 +0,0 @@
-/*
- *   Software Updater - server side
- *
- *      Copyright © 2012-2016 Intel Corporation.
- *
- *   This program is free software: you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, version 2 or later of the License.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *   Authors:
- *         Tom Keel <thomas.keel@intel.com>
- *
- */
-
-#define _GNU_SOURCE
-#include <err.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "swupd.h"
-
-static char *make_filename(const char *, const char *, const char *);
-
-static const char *CMD_FMT = "openssl smime -sign -in %s -binary "
-			     "-out %s.signed -outform PEM -md sha256 -inkey %s -signer %s "
-			     "-certfile %s -passin file:%s";
-
-static char *leaf_key = NULL;
-static char *leaf_cert = NULL;
-static char *ca_chain_cert = NULL;
-static char *passphrase = NULL;
-
-static bool initialized = false;
-
-/*
- * Initialize this module.
- * @return true <=> success
- */
-bool signature_initialize(void)
-{
-	if (!enable_signing) {
-		return true;
-	}
-
-	char *cdir;
-	char *pphr;
-	struct stat s;
-
-	if (initialized) {
-		return true;
-	}
-	cdir = getenv("SWUPD_CERTS_DIR");
-	if (cdir == NULL || cdir[0] == '\0') {
-		printf("No certificates directory specified\n");
-		goto err;
-	}
-	if (stat(cdir, &s)) {
-		printf("Can't stat certificates directory '%s' (%s)\n", cdir,
-		       strerror(errno));
-		goto err;
-	}
-	leaf_key = make_filename(cdir, "LEAF_KEY", "leaf key");
-	if (leaf_key == NULL) {
-		goto err;
-	}
-	leaf_cert = make_filename(cdir, "LEAF_CERT", "leaf certificate");
-	if (leaf_cert == NULL) {
-		goto err;
-	}
-	ca_chain_cert = make_filename(cdir, "CA_CHAIN_CERT", "CA chain certificate");
-	if (ca_chain_cert == NULL) {
-		goto err;
-	}
-	pphr = getenv("PASSPHRASE");
-	if (pphr == NULL || (passphrase = strdup(pphr)) == NULL) {
-		goto err;
-	}
-	if (stat(passphrase, &s)) {
-		printf("Can't stat '%s' (%s)\n", passphrase,
-		       strerror(errno));
-		goto err;
-	}
-	initialized = true;
-	return true;
-err:
-	signature_terminate();
-	return false;
-}
-
-/* Make filename from dir name and env variable containing basename */
-static char *make_filename(const char *dir, const char *env, const char *desc)
-{
-	char *fn = getenv(env);
-	char *result = NULL;
-	struct stat s;
-
-	if (fn == NULL || fn[0] == '\0') {
-		printf("No %s file specified\n", desc);
-		return NULL;
-	}
-	string_or_die(&result, "%s/%s", dir, fn);
-	if (stat(result, &s)) {
-		printf("Can't stat %s '%s' (%s)\n", desc, result, strerror(errno));
-		free(result);
-		return NULL;
-	}
-	return result;
-}
-
-/*
- * Terminate this module, free resources.
- */
-void signature_terminate(void)
-{
-	if (!enable_signing) {
-		return;
-	}
-
-	free(leaf_key);
-	free(leaf_cert);
-	free(ca_chain_cert);
-	free(passphrase);
-
-	leaf_key = NULL;
-	leaf_cert = NULL;
-	ca_chain_cert = NULL;
-	passphrase = NULL;
-
-	initialized = false;
-}
-
-/*
- * Write the signature file corresponding to the given data file.
- * The name of the signature file is the name of the data file with suffix
- * ".signed" appended.
- */
-bool signature_sign(const char *filename)
-{
-	char *cmd = NULL;
-	int status;
-
-	if (!enable_signing) {
-		return true;
-	}
-
-	if (!initialized) {
-		return false;
-	}
-	string_or_die(&cmd, CMD_FMT, filename, filename, leaf_key, leaf_cert,
-		      ca_chain_cert, passphrase);
-	status = system(cmd);
-	if (status) {
-		printf("Bad status %d from signing command:%s\n", status, cmd);
-	}
-	free(cmd);
-	return status == 0;
-}

src/versions.c

@@ -27,6 +27,7 @@
 #include <ctype.h>
 #include <dirent.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -126,35 +127,63 @@ void ensure_version_image_exists(int version)
 void write_cookiecrumbs_to_download_area(int version)
 {
 	char *conf;
-	char *cmd;
+	char *filename, *strformat;
+	FILE *versionfd, *formatfd;
 
 	conf = config_output_dir();
 	if (conf == NULL) {
 		assert(0);
 	}
 
-	string_or_die(&cmd, "echo %s > %s/%i/swupd-server-src-version", VERSION, conf, version);
-	if (system(cmd) != 0) {
+	string_or_die(&filename, "%s/%i/swupd-server-src-version", conf, version);
+	versionfd = fopen(filename, "w");
+	if (versionfd == NULL) {
 		assert(0);
 	}
-	free(cmd);
-
-	string_or_die(&cmd, "echo %llu > %s/%i/format", format, conf, version);
-	if (system(cmd) != 0) {
+	if (fwrite(VERSION, 1, strlen(VERSION), versionfd) != strlen(VERSION)) {
 		assert(0);
 	}
-	free(cmd);
+	fclose(versionfd);
+	free(filename);
 
-/* written from pack_maker.sh */
+	string_or_die(&filename, "%s/%i/format", conf, version);
+	string_or_die(&strformat, "%llu", format);
+	formatfd = fopen(filename, "w");
+	if (formatfd == NULL) {
+		assert(0);
+	}
+	if (fwrite(strformat, 1, strlen(strformat), formatfd) != strlen(strformat)) {
+		assert(0);
+	}
+	fclose(formatfd);
+	free(filename);
+	free(strformat);
+
+/* Updating the WEBDIR/version/formatN/latest file is an operation very closely tied to a DevOps
+ * workflow and not something swupd should make a decision about.
+ *
+ * The supported format values (replacing the "N" in the path above) are:
+ *
+ * - "staging": intended for testing only
+ * - [1-+inf] (positive integer): the number that a corresponding version of swupd-client
+ *   understands to run software updates. For example, if a given swupd-client release understands
+ *   format 1, then WEBDIR/version/format1/latest contains the value of most recent release that
+ *   that swupd-client can update to.
+ *
+ * Adapt this step appropriately for your DevOps flow.
+ */
 #if 0
 	FILE *file;
 	char *fullfile = NULL;
+	char *param;
 
-	string_or_die(&cmd, "mkdir -p %s/version/formatstaging/", conf);
-	if (system(cmd) != 0) {
+	string_or_die($param, "%s/version/formatstaging/", conf);
+	char *const mkdircmd[] = { "mkdir", "-p", param, NULL };
+
+	if (system_argv(mkdircmd) != 0) {
 		assert(0);
 	}
-	free(cmd);
+	free(param);
 
 	string_or_die(&fullfile, "%s/version/formatstaging/latest", conf);
 	file = fopen(fullfile, "w");
@@ -206,8 +235,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 	DIR *dir;
 	GList *list = NULL;
 	GList *cur_item, *next_item;
-	struct dirent entry;
-	struct dirent *result;
+	struct dirent *entry;
 	struct stat stat;
 	char *filename = NULL;
 	int idx, build_num, build_type, jump_point;
@@ -220,13 +248,13 @@ GList *get_last_versions_list(int next_version, int max_versions)
 		return NULL;
 	}
 
-	while (readdir_r(dir, &entry, &result) == 0 && result != NULL) {
-		if (strspn(entry.d_name, "0123456789") != strlen(entry.d_name)) {
+	while ((entry = readdir(dir))) {
+		if (strspn(entry->d_name, "0123456789") != strlen(entry->d_name)) {
 			continue;
 		}
 
 		free(filename);
-		string_or_die(&filename, "%s/%s", staging_dir, entry.d_name);
+		string_or_die(&filename, "%s/%s", staging_dir, entry->d_name);
 
 		if (lstat(filename, &stat)) {
 			LOG(NULL, "lstat failed", "path= %s, strerror= %s",
@@ -238,11 +266,11 @@ GList *get_last_versions_list(int next_version, int max_versions)
 			continue;
 		}
 
-		if (atoi(entry.d_name) >= next_version) {
+		if (atoi(entry->d_name) >= next_version) {
 			continue;
 		}
 
-		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry.d_name)));
+		list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry->d_name)));
 	}
 	free(filename);
 	closedir(dir);

src/xattrs.c

@@ -60,8 +60,7 @@ static int xattr_get_value(const char *path, const char *name, char **blob,
 
 	/* realloc needed len + 1 in case we need to add final zero
 	 * to ensure consistent blob */
-	value = realloc(*blob, *blob_len + len +
-				   (action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
+	value = realloc(*blob, *blob_len + len + (action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
 	assert(value);
 
 	*blob = value;

test/functional/contentsize-across-versions-includes/test.bats

@@ -0,0 +1,64 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+  gen_file_plain 10 test-bundle1 foo
+  gen_file_plain 10 test-bundle1 foobar
+  gen_file_plain 10 test-bundle2 foo2
+  gen_includes_file test-bundle2 10 test-bundle1
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+  gen_file_plain 20 test-bundle1 foo
+  gen_file_plain 20 test-bundle1 foobar
+  gen_file_plain 20 test-bundle1 foobarbaz
+  gen_file_plain 20 test-bundle2 foo2
+  gen_file_plain 20 test-bundle2 foo2bar
+  gen_includes_file test-bundle2 20 test-bundle1
+}
+
+@test "correct contentsize" {
+  # create a couple updates to both check that contentsize does not add included
+  # bundles and to verify that files changed in previous updates are counted.
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # contentsize for test-bundle2 should not include test-bundle1's contentsize
+  [[ 1 -eq $(grep '^contentsize:	11$' $DIR/www/10/Manifest.test-bundle1 | wc -l) ]]
+  [[ 1 -eq $(grep '^contentsize:	5$'  $DIR/www/10/Manifest.test-bundle2 | wc -l) ]]
+  # os-core is large because it includes /usr/*
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # 5134 + 11 + 5 = 5150
+  [[ 1 -eq $(grep '^contentsize:	5150$' $DIR/www/10/Manifest.full | wc -l) ]]
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  set_latest_ver 20
+
+  # one new file: foobarbaz (10 bytes)
+  [[ 1 -eq $(grep '^contentsize:	21$' $DIR/www/20/Manifest.test-bundle1 | wc -l) ]]
+  # one new file: foo2bar (8 bytes)
+  [[ 1 -eq $(grep '^contentsize:	13$'  $DIR/www/20/Manifest.test-bundle2 | wc -l) ]]
+  # os-core should not change size
+  [[ 1 -eq $(grep '^contentsize:	5134$'  $DIR/www/10/Manifest.os-core | wc -l) ]]
+  # contentsize for full should be all files, including ones not changed in this release
+  # two new files: foo2bar (8 bytes) and foobarbaz (10 bytes)
+  # 5150 + 10 + 8  = 5168
+  # 5134 + 21 + 13 = 5168
+  [[ 1 -eq $(grep '^contentsize:	5168$' $DIR/www/20/Manifest.full | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/file-name-blacklisted/test.bats

@@ -0,0 +1,40 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  gen_file_plain 10 os-core "semicolon;"
+}
+
+assert_illegal_char() {
+  local output="$1"
+  local str=$(echo "Filename $2 includes illegal character(s)")
+  [[ "$output" =~ "$str" ]]
+}
+
+assert_valid_char() {
+  local output="$1"
+  local str=$(echo "Filename $2 includes illegal character(s)")
+  [[ ! "$output" =~ "$str" ]]
+}
+
+@test "file names with blacklisted characters" {
+  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
+
+  echo "$output"
+  assert_illegal_char "$output" "/semicolon;"
+  assert_valid_char "$output" "/usr"
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/file-name-debuginfo/test.bats

@@ -0,0 +1,56 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  # unban debuginfo
+  sed -i "s|banned=true|banned=false|" $DIR/server.ini
+  set_latest_ver 0
+  init_groups_ini os-core
+  init_groups_ini test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 test-bundle
+
+  gen_file_plain 10 test-bundle "/usr/lib/debug/foo"
+  gen_file_plain 10 test-bundle "/usr/src/debug/bar"
+  gen_file_plain 10 test-bundle "/usr/bin/foobar"
+
+  track_bundle 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 test-bundle
+
+  gen_file_plain 20 test-bundle "/usr/foo"
+}
+
+@test "debuginfo files pruned" {
+  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
+  echo "$output"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/bin/foobar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/src/debug$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  # These should not be pruned right now, but should be pruned with the next
+  # update after being banned in server.ini again
+  [[ 1 -eq $(grep '/usr/src/debug/bar$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+
+  # grab the server.ini that bans debuginfo again
+  init_server_ini
+  set_os_release 20 os-core
+  run sudo sh -c "$CREATE_UPDATE --osversion 20 --statedir $DIR --format 3"
+  # This should not be pruned
+  [[ 1 -eq $(grep '/usr/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  # These should be pruned
+  [[ 0 -eq $(grep '/usr/src/debug/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/usr/lib/debug/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/format-bump/test.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+}
+
+@test "full run update creation with delta packs over format bump" {
+  # build the first version
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+
+  set_latest_ver 10
+
+  # then the second version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+
+  set_latest_ver 20
+
+  # then the third version...
+  echo $CREATE_UPDATE
+  echo $DIR
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 4
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  sudo $MAKE_PACK --statedir $DIR 0 30 os-core
+
+  # zero packs should exist (non-zero size) for all versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/30/pack-os-core-from-0.tar ]
+
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/10/Manifest.MoM | wc -l) ]]
+  [[ 0 -eq $(grep '^actions:	update$' $DIR/www/20/Manifest.MoM | wc -l) ]]
+  [[ 1 -eq $(grep '^actions:	update$' $DIR/www/30/Manifest.MoM | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/format-no-decrement/test.bats

@@ -0,0 +1,35 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  gen_file_plain 10 os-core foo
+  gen_file_plain 10 os-core bar
+  gen_file_plain 20 os-core foo
+  gen_file_plain 20 os-core baz
+}
+
+@test "ensure format numbers cannot be decremented" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+
+  # Now, decrement the format number. Should result in an EXIT_FAILURE.
+  run sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 2
+  echo "$output"
+  [ $status -eq 1 ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run-delta/test.bats

@@ -21,7 +21,7 @@ setup() {
   track_bundle 20 included-two
   track_bundle 20 included-nested
 
-  gen_file_to_delta 10 4096 20 4 test-bundle
+  gen_file_to_delta 10 4096 20 4 test-bundle randomfile
 
   gen_file_plain 10 test-bundle foo
   gen_file_plain 10 test-bundle foobarbaz
@@ -79,6 +79,11 @@ setup() {
   [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/20/Manifest.included-nested | wc -l) ]]
   [[ 6 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/20/pack-test-bundle-from-0.tar | wc -l) ]]
+
+  # delta packs should contain appropriate delta manifests
+  [[ $(tar -tf $DIR/www/20/pack-os-core-from-10.tar | grep '^Manifest-os-core-delta-from-10') ]]
+  [[ $(tar -tf $DIR/www/20/pack-os-core-from-10.tar | grep '^Manifest-MoM-delta-from-10') ]]
+  [[ $(tar -tf $DIR/www/20/pack-test-bundle-from-10.tar | grep '^Manifest-test-bundle-delta-from-10') ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run/test.bats

@@ -39,7 +39,13 @@ setup() {
   [[ 1 -eq $(grep '/usr/share/clear$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
-  [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+  [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+
+  # extract test-bundle pack to make sure staged and delta are created with the
+  # correct permissions
+  sudo tar -xf $DIR/www/10/pack-test-bundle-from-0.tar --directory $DIR/www/10/
+  [[ $(stat -c %a $DIR/www/10/staged) -eq 700 ]]
+  [[ $(stat -c %a $DIR/www/10/delta) -eq 700 ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/fullfiles/test.bats

@@ -14,6 +14,7 @@ load "../swupdlib"
 }
 
 @test "make_fullfiles root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $MAKE_FULLFILES foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]

test/functional/ghosting/test.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  gen_file_plain_with_content 10 test-bundle usr/lib/kernel/bar testfile_contents
+  gen_file_plain_with_content 20 test-bundle usr/lib/kernel/baz new_testfile_contents
+  # make sure /usr/lib/kernel stays around
+  mkdir $DIR/image/30/test-bundle/usr/lib/kernel
+}
+
+@test "create updates while ghosting boot files before removing them" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  # version 10: add boot file to 10
+  [ 1 -eq $(grep 'F\.b\.	.*	10	/usr/lib/kernel/bar' $DIR/www/10/Manifest.full | wc -l) ]
+  # version 20: ghost boot file instead of deleting it when it doesn't exist
+  [ 1 -eq $(grep '\.gb\.	.*	20	/usr/lib/kernel/bar' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 20: add a new boot file
+  [ 1 -eq $(grep 'F\.b\.	.*	20	/usr/lib/kernel/baz' $DIR/www/20/Manifest.full | wc -l) ]
+  # version 30: old ghosted file /usr/lib/kernel/bar cleaned up
+  [ 0 -eq $(grep '/usr/lib/kernel/bar' $DIR/www/30/Manifest.full | wc -l) ]
+  # version 30: boot file added in version 20 ghosted
+  [ 1 -eq $(grep '\.gb\.	.*	30	/usr/lib/kernel/baz' $DIR/www/30/Manifest.full | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/includes-deduplicate/test.bats

@@ -0,0 +1,42 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+
+  gen_file_plain 10 os-core test0
+  gen_file_plain 10 test-bundle1 test1
+  gen_file_plain 10 test-bundle2 test2
+
+  gen_includes_file test-bundle2 10 test-bundle1 test-bundle1
+  gen_includes_file test-bundle2 20 test-bundle1 test-bundle1
+}
+
+@test "deduplicate bundle includes" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+
+  # includes list should be deduplicated in both the old and new manifests
+  [[ 1 -eq $(grep '^includes:	test-bundle1$' $DIR/www/10/Manifest.test-bundle2 | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	test-bundle1$' $DIR/www/20/Manifest.test-bundle2 | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/no-delta/test.bats

@@ -0,0 +1,73 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+
+  # symlink -> regular file type change (L -> F)
+  gen_file_to_delta 10 4096 20 4 os-core testfile1
+  gen_symlink_to_file 10 os-core testsym1 testfile1
+  copy_file 10 os-core testfile1 20 os-core testsym1
+
+  # regular file -> symlink type change (F -> L)
+  gen_file_to_delta 10 4096 20 4 os-core testfile2
+  copy_file 10 os-core testfile2 10 os-core testsym2
+  gen_symlink_to_file 20 os-core testsym2 testfile2
+
+  # symlink change + symlink target change; delta should be created for
+  # testfile3, but not for the dereferenced testsym3
+  gen_file_to_delta 10 4096 20 4 os-core testfile3
+  copy_file 20 os-core testfile3 20 os-core testfile4
+  gen_symlink_to_file 10 os-core testsym3 testfile3
+  gen_symlink_to_file 20 os-core testsym3 testfile4
+}
+
+@test "no deltas created for type changes or dereferenced symlinks" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+
+  # attempt to create some deltas
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+
+  # F -> F deltas should exist
+  hash1=$(hash_for 10 os-core "/testfile1")
+  hash2=$(hash_for 20 os-core "/testfile1")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testfile2")
+  hash2=$(hash_for 20 os-core "/testfile2")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testfile3")
+  hash2=$(hash_for 20 os-core "/testfile3")
+  [ -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  # deltas for symlink type changes should not be created
+  hash1=$(hash_for 10 os-core "/testsym1")
+  hash2=$(hash_for 20 os-core "/testsym1")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testsym2")
+  hash2=$(hash_for 20 os-core "/testsym2")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+
+  hash1=$(hash_for 10 os-core "/testsym3")
+  hash2=$(hash_for 20 os-core "/testsym3")
+  [ ! -f $DIR/www/20/delta/10-20-$hash1-$hash2 ]
+}
+
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/orphaned-renames/test.bats

@@ -0,0 +1,105 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  set_os_release 30 os-core
+  set_os_release 30 test-bundle
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  set_os_release 40 os-core
+  set_os_release 40 test-bundle
+  track_bundle 40 os-core
+  track_bundle 40 test-bundle
+
+  # /usr/lib/bar and /one will be renamed to /usr/lib/baz and /two
+  gen_file_plain_with_content 10 test-bundle /usr/lib/bar "$(seq 100)"
+  gen_file_plain_with_content 10 test-bundle /one "$(printf 'a%.0s' {1..200})"
+  gen_file_plain_with_content 10 test-bundle /usr/lib/kernel/a "$(printf 'a%.0s' {1..210})"
+
+  gen_file_plain_with_content 20 test-bundle /usr/lib/baz "$(seq 100)"
+  gen_file_plain_with_content 20 test-bundle /two "$(printf 'a%.0s' {1..200})"
+  gen_file_plain_with_content 20 test-bundle /usr/lib/kernel/ab "$(printf 'a%.0s' {1..210})"
+
+  # different content just to make sure this works with delta renames as well as
+  # direct renames
+  gen_file_plain_with_content 30 test-bundle /usr/lib/foo "$(seq 100) new"
+  gen_file_plain_with_content 30 test-bundle /usr/lib/kernel/abc "$(printf 'a%.0s' {1..210})"
+
+  gen_file_plain_with_content 40 test-bundle /usr/lib/foo "$(seq 100) new"
+  gen_file_plain_with_content 40 test-bundle /usr/lib/kernel/abc "$(printf 'a%.0s' {1..210})"
+  # new file to force manifest generation
+  gen_file_plain_with_content 40 test-bundle /a "testfile"
+}
+
+@test "create updates with renamed-to file getting deleted" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  set_latest_ver 20
+
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+
+  set_latest_ver 30
+
+  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 40
+  # version 10: add files to 10
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/usr/lib/bar' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.\.\t.*\t10\t/one' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.b\.\t.*\t10\t/usr/lib/kernel/a' $DIR/www/10/Manifest.test-bundle | wc -l) ]
+  # version 20: rename bar to baz and one to two
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/one' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/two' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t20\t/usr/lib/bar' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t20\t/usr/lib/baz' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.gbr\t.*\t20\t/usr/lib/kernel/a' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.br\t.*\t20\t/usr/lib/kernel/ab' $DIR/www/20/Manifest.test-bundle | wc -l) ]
+  # version 30: original renamed-from file (bar) is now orphaned and therefore
+  # deleted, baz is now a renamed-from file and foo is a renamed-to file.
+  # /two was deleted in this version, so both /one and /two should be marked as
+  # deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.gbr\t.*\t30\t/usr/lib/kernel/ab' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.br\t.*\t30\t/usr/lib/kernel/abc' $DIR/www/30/Manifest.test-bundle | wc -l) ]
+
+  # version 40: the existing rename from baz -> foo must persist while all
+  # others remain deleted
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/one' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t30\t/two' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.\.\t0\{64\}\t20\t/usr/lib/bar' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'\.d\.r\t.*\t30\t/usr/lib/baz' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.\.r\t.*\t30\t/usr/lib/foo' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+  [ 1 -eq $(grep $'F\.b\.\t.*\t30\t/usr/lib/kernel/abc' $DIR/www/40/Manifest.test-bundle | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/pack/test.bats

@@ -20,6 +20,7 @@ load "../swupdlib"
 }
 
 @test "make_pack root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $MAKE_PACK foo bar foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]

test/functional/renames/job

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export BATS_TEST_DIRNAME=$PWD
+export TIMEFORMAT
+TIMEFORMAT="time %R %P%% (%U+%S)"
+source ../swupdlib.bash
+exec 3< rundata
+
+while read -r -u 3 
+do
+    read -i "$REPLY" -e CMD
+    eval "set -x ; time ( $CMD ) ; set +x "
+done

test/functional/renames/rundata

@@ -0,0 +1,22 @@
+sudo rm -r logs/ web-dir/
+tar xf ~/data/test.tar 
+init_test_dir
+init_server_ini
+set_latest-ver 0
+init_groups_ini os-core test-bundle
+set_os_release 10 os-core
+track_bundle 10 os-core
+track_bundle 10 test-bundle
+set_os_release 20 os-core
+track_bundle 20 os-core
+track_bundle 20 test-bundle
+sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 10
+sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+time  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+set_latest-ver 10
+sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 20
+sudo $MAKE_PACK --statedir $DIR  10 20  os-core
+sudo $MAKE_PACK --statedir $DIR  10 20  test-bundle

test/functional/renames/test.bats

@@ -0,0 +1,246 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  maybeskip
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+}
+
+# Generate data files of 3 types in test-bundle
+# All files are big enough that they get rename detection
+gendataA() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 100)"
+}
+gendataB() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 50) $(seq 52 101)"
+}
+gendataC() {
+  # cache string
+  [ -z "$dataC" ] || dataC="$(seq 1000 | gzip | uuencode wombat)"
+  gen_file_plain_with_content "$1" test-bundle "$2" "$dataC"
+}
+# Generate small data files
+gendataAs() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 50)"
+}
+gendataBs() {
+  gen_file_plain_with_content "$1" test-bundle "$2" "$(seq 24) $(seq 26 49)"
+}
+gendataCs() {
+  # cache string
+  [ -z "$dataCs" ] || dataC="$(seq 50 | gzip | uuencode wombat)"
+  gen_file_plain_with_content "$1" test-bundle "$2" "$dataCs"
+}
+
+checkrenamed(){
+    local flags sha1 ver name fromsha1="bad" tosha1
+    # Check that $1 is renamed to $2
+    exec 9< $DIR/www/20/Manifest.test-bundle
+    # skip the header
+    while read -u9
+    do
+	[ -z "$REPLY" ] && break
+    done
+    while read -r -u9 flags sha1 ver name
+    do
+	case "$flags" in
+	    (?"d.r") [ "$name" = "$1" ] && fromsha1=$sha1 ;;
+	    (?"..r") [ "$name" = "$2" ] && tosha1=$sha1 ;;
+	esac
+    done
+    if [ "$fromsha1" = "$tosha1" ] ; then return 0 ; else return 1 ; fi
+}
+
+# Guts of doing an update
+do_an_update() {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+}
+
+
+@test "basic rename detection support" {
+  gendataA 10 foo
+  gendataA 20 bar
+  do_an_update
+  checkrenamed /foo /bar
+}
+
+@test "ignore rename detection for small files" {
+  gendataAs 10 foo
+  gendataAs 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "ignore rename detection for large to small files" {
+  gendataA 10 foo
+  gendataAs 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "ignore rename detection for small to large files" {
+  gendataAs 10 foo
+  gendataA 20 bar
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 0 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+@test "rename one file to two" {
+  gendataA 10 foo
+  gendataA 20 bar
+  gendataA 20 baz
+  do_an_update  
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two file to one" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 baz
+  do_an_update  
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/baz$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.\..*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two files to two" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataA 20 baz  
+  do_an_update
+  checkrenamed /foo /bar
+  checkrenamed /foz /baz
+}
+
+@test "rename two files to two, one slightly different" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataB 20 baz  
+  do_an_update
+  # we don't actually know how the client we do this rename, but don't care
+  checkrenamed /foo /bar
+  checkrenamed /foz /baz
+}
+
+@test "rename two files to two, each pair slightly different" {
+  gendataA 10 foo
+  gendataB 10 foz
+  gendataA 20 bar
+  gendataB 20 baz
+  do_an_update
+  checkrenamed /foo /bar
+  checkrenamed /foz /baz
+}
+
+@test "rename two files to two, one very different" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataC 20 baz  
+  do_an_update
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/fo[oz]$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+@test "rename two files to two, one small" {
+  gendataA 10 foo
+  gendataA 10 foz
+  gendataA 20 bar
+  gendataCs 20 baz  
+  do_an_update
+  run checkrenamed /foo /bar
+  if [ $status -eq 1 ] ; then
+    checkrenamed /foz /bar
+  fi
+}
+
+@test "directory name changes" {
+  gendataA 10 dir1/foo
+  gendataA 20 dir2/foo
+  do_an_update
+  checkrenamed /dir1/foo /dir2/foo
+}
+
+@test "directory name changes small files" {
+  gendataAs 10 dir1/foo
+  gendataAs 20 dir2/foo
+  do_an_update
+  if ! checkrenamed /dir1/foo /dir2/foo ; then false ; fi
+}
+
+@test "directory name and small data changes" {
+  gendataA 10 dir1/foo
+  gendataB 20 dir2/foz
+  do_an_update
+  checkrenamed /dir1/foo /dir2/foz
+}
+
+@test "directory name and small data changes, choose same name" {
+  gendataA 10 dir1/foo
+  gendataA 10 dir1/foz
+  gendataB 20 dir2/foz
+  do_an_update
+  checkrenamed /dir1/foz /dir2/foz
+}
+
+@test "same basename test" {
+    gendataA 10 dir1/foo.so.1
+    gendataA 10 dir1/foz.so.1
+    gendataB 20 dir2/foo.so.2
+    do_an_update
+    checkrenamed /dir1/foo.so.1 /dir2/foo.so.2
+}
+
+@test "rename file to dir/file" {
+    gendataA 10 foo
+    gendataA 20 foo/bar
+    do_an_update
+    checkrenamed /foo /foo/bar
+}
+# @test "rename foo/foo to foo" {
+#     gendataA 10 foo/foo
+#     gendataA 20 foo
+#     do_an_update
+#     checkrenamed /foo/foo /foo
+# }
+
+# Emacs and vi support
+# Local variables:
+# sh-indentation: 2
+# End:
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/renames2/job

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export BATS_TEST_DIRNAME=$PWD
+export TIMEFORMAT
+TIMEFORMAT="time %R %P%% (%U+%S)"
+source ../swupdlib.bash
+exec 3< rundata
+
+while read -r -u 3 
+do
+    read -i "$REPLY" -e CMD
+    eval "set -x ; time ( $CMD ) ; set +x "
+done

test/functional/renames2/rundata

@@ -0,0 +1,22 @@
+sudo rm -r logs/ web-dir/
+tar xf ~/data/test.tar 
+init_test_dir
+init_server_ini
+set_latest_ver 0
+init_groups_ini os-core test-bundle
+set_os_release 10 os-core
+track_bundle 10 os-core
+track_bundle 10 test-bundle
+set_os_release 20 os-core
+track_bundle 20 os-core
+track_bundle 20 test-bundle
+sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 10
+sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+time  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+set_latest_ver 10
+sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+sudo $MAKE_FULLFILES --statedir $DIR 20
+sudo $MAKE_PACK --statedir $DIR  10 20  os-core
+sudo $MAKE_PACK --statedir $DIR  10 20  test-bundle

test/functional/renames2/test.bats

@@ -0,0 +1,144 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+  tar x -C $DIR/.. -f ~/data/test.tar 
+  mv $DIR/image/20/test-bundle/usr/share/bash-completion/completions{,.old}
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+
+  # Files have different names ("foo" vs "bar"), but have the same content
+  gen_file_plain_with_content 10 test-bundle "foo" "data"
+  gen_file_plain_with_content 20 test-bundle "bar" "data"
+}
+
+@test "rename detection support" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+  # A renamed file comprises a new file and a deleted file
+  [[ 1 -eq $(grep '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+}
+
+# @test "rename detection support 1 to 2" {
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "baz" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/ba[rz]$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^F\.\.\..*/ba[rz]$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^\.d\.r.*/foo$' $DIR/www/20/Manifest.test-bundle) ]]
+# }
+
+# @test "rename detection support 2 to 2" {
+#   gen_file_plain_with_content 10 test-bundle "fee" "data"
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo~" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/bar$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 1 -eq $(grep -c '^F\.\.r.*/foo~$' $DIR/www/20/Manifest.test-bundle) ]]
+#   [[ 2 -eq $(grep -c '^\.d\.r.*/f..$' $DIR/www/20/Manifest.test-bundle) ]]
+# }
+
+# @test "rename detection support binary files" {
+#   sudo mkdir -p  $DIR/image/10/test-bundle/bin/
+#   sudo cp /bin/gcc-ar $DIR/image/10/test-bundle/bin/gcc-ar.1
+#   sudo cp /bin/gcc-ar $DIR/image/20/test-bundle/gcc-ar.1
+#   strip $DIR/image/20/test-bundle/gcc-ar.1
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo~" "And now not the same"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+# }
+
+# @test "rename detection support binary files 2" {
+#   sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 10
+#   sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+#   set_latest_ver 10
+
+#   gen_file_plain_with_content 20 test-bundle "foo" "And now not the same"
+#   gen_file_plain_with_content 20 test-bundle "foo.1" "data"
+#   sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+#   sudo $MAKE_FULLFILES --statedir $DIR 20
+#   sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+#   sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+#   sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+#   # A renamed file comprises a new file and a deleted file.
+#   # Should be one file that is not renamed
+#   # Need to figure out what happens to hard links
+# }
+
+# # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/state-file/test.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  gen_file_plain 10 os-core "/var/lib/test"
+}
+
+@test "state file marked in manifest" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+
+  # a file and dir installed in /var should be marked state
+  grep '^D\.s\..*/var/lib$' $DIR/www/10/Manifest.os-core
+  grep '^F\.s\..*/var/lib/test$' $DIR/www/10/Manifest.os-core
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/subtract-delete/test.bats

@@ -0,0 +1,56 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  # start with both bundles containing a file "foo"
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  gen_file_plain 10 os-core foo
+  gen_file_plain 10 test-bundle foo
+
+  # delete "foo" from os-core (the included bundle)
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  gen_file_plain 20 test-bundle foo
+
+  # delete "foo" from test-bundle
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+
+  # make modification (add new file) to test-bundle
+  set_os_release 40 os-core
+  track_bundle 40 os-core
+  track_bundle 40 test-bundle
+  gen_file_plain 40 test-bundle foobar
+}
+
+@test "no subtraction for two deleted files" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  set_latest_ver 10
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  set_latest_ver 20
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  set_latest_ver 30
+  sudo $CREATE_UPDATE --osversion 40 --statedir $DIR --format 3
+  set_latest_ver 40
+
+  # If the file is absent in test-bundle, this means it was subtracted. If
+  # present, subtraction was not performed.
+  hash1=$(hash_for 40 test-bundle "/foo")
+  [ -n "$hash1" ]
+}
+
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/swupdlib.bash

@@ -27,7 +27,7 @@ init_server_ini() {
 }
 
 set_latest_ver() {
-  echo "$1" > $DIR/image/latest.version
+  echo "$1" > $DIR/image/LAST_VER
 }
 
 init_groups_ini() {
@@ -35,10 +35,16 @@ init_groups_ini() {
     cat >> $DIR/groups.ini << EOF
 [$bundle]
 group=$bundle
+status=ACTIVE
 EOF
   done
 }
 
+# If the variable RUN_JUST_ONE is set then only run that test
+maybeskip() {
+    [ -z "$RUN_JUST_ONE" ] || [ "$RUN_JUST_ONE" -eq "$BATS_TEST_NUMBER" ] || skip
+}
+
 set_os_release() {
   local ver=$1
   local bundle=$2
@@ -57,9 +63,9 @@ gen_includes_file() {
   local bundle=$1
   local ver=$2
   local includes="${@:3}"
-  mkdir -p $DIR/www/$ver/noship
+  mkdir -p $DIR/image/$ver/noship
   for b in $includes; do
-    cat >> $DIR/www/$ver/noship/"$bundle"-includes << EOF
+    cat >> $DIR/image/$ver/noship/"$bundle"-includes << EOF
 $b
 EOF
   done
@@ -71,37 +77,87 @@ gen_file_to_delta() {
   local newver=$3
   local newbytes=$4
   local bundle=$5
+  local name=$6
 
   # create some random data for the original version
   mkdir -p $DIR/image/$origver/$bundle
-  dd if=/dev/urandom of=$DIR/image/$origver/$bundle/randomfile bs=1 count=$origsize
+  dd if=/dev/urandom of=$DIR/image/$origver/$bundle/$name bs=1 count=$origsize
 
   # append more random data to the end of the file in the new version
   TMP=$(mktemp foo.XXXXXX)
   mkdir -p $DIR/image/$newver/$bundle
   dd if=/dev/urandom of=$TMP bs=1 count=$newbytes
-  cat $DIR/image/$origver/$bundle/randomfile $TMP > $DIR/image/$newver/$bundle/randomfile
+  cat $DIR/image/$origver/$bundle/$name $TMP > $DIR/image/$newver/$bundle/$name
   rm $TMP
 }
 
 gen_file_plain() {
   local ver=$1
   local bundle=$2
-  local name=$3
+  local name="$3"
 
   # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
-  echo $name > $DIR/image/$ver/$bundle/$name
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
+  echo "$name" > $DIR/image/$ver/$bundle/"$name"
 }
 
 gen_file_plain_change() {
   local ver=$1
   local bundle=$2
-  local name=$3
+  local name="$3"
 
   # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
-  echo $ver $name > $DIR/image/$ver/$bundle/$name
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
+  echo "$ver $name" > $DIR/image/$ver/$bundle/"$name"
+}
+
+gen_symlink_to_file() {
+  local ver=$1
+  local bundle=$2
+  local symname="$3"
+  local filename="$4"
+
+  mkdir -p $DIR/image/$ver/$bundle/$(dirname "$symname")
+  ln -s "$filename" $DIR/image/$ver/$bundle/"$symname"
+}
+
+copy_file() {
+  local origver=$1
+  local origbundle=$2
+  local origname="$3"
+  local newver=$4
+  local newbundle=$5
+  local newname="$6"
+
+  mkdir -p $DIR/image/$newver/$newbundle/$(dirname "$newname")
+  cp -a $DIR/image/$origver/$origbundle/"$origname" $DIR/image/$newver/$newbundle/"$newname"
+}
+
+# Gets the hash for file NAME in BUNDLE manifest for VER
+hash_for() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+
+  awk -F'\t' -v NAME="$name" 'NF == 4 && $4 == NAME { print $2 }' $DIR/www/$ver/Manifest.$bundle
+}
+
+gen_file_plain_with_content() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+  local content="$4"
+  case "$name" in
+      (*"/"*)	mkdir -p "$DIR/image/$ver/$bundle/${name%/*}" ;;
+      (*)	mkdir -p $DIR/image/$ver/$bundle ;;
+  esac
+  echo "$content" > $DIR/image/$ver/$bundle/"$name"
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/update/test.bats

@@ -15,6 +15,7 @@ load "../swupdlib"
 }
 
 @test "create_update root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $CREATE_UPDATE -F 3 -o 10
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]