Release v3.2.5

This release includes a fix to replace system calls by local system_argv
functions, cleanup patches for error output and code style, adding delta
manifests to delta packs, consistency changes for tar and string allocation,
and removes creation of delta manifests out of create_update.
This is a re-roll of release 3.2.4 to fix a time skew issue that caused some
build failures.

Signed-off-by: Tudor Marcu <tudor.marcu@intel.com>

.clang-format

@@ -0,0 +1,67 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: true
+AlignConsecutiveAssignments: false
+AlignEscapedNewlinesLeft: false
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: false
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Linux
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     0
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IndentCaseLabels: false
+IndentWidth:     8
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Always
+...
+

.gitignore

@@ -19,8 +19,17 @@ stamp-h1
 DEADJOE
 swupd_create_update
 swupd_make_pack
-swupd-update-server.log*
 swupd_make_fullfiles
 swupd-server-*.tar.gz
 cscope.*
 *.kdev4
+coverage/
+src/*.gcno
+src/*.gcda
+tap-driver.sh
+test/functional/*/*.log
+test/functional/*/*.trs
+test/functional/*/logs/
+test/functional/*/web-dir/
+swupd-*.log
+test-suite.log

Makefile.am

@@ -15,6 +15,7 @@ swupd_create_update_SOURCES = \
 	src/analyze_fs.c \
 	src/chroot.c \
 	src/config.c \
+	src/create_update.c \
 	src/delta.c \
 	src/fullfiles.c \
 	src/globals.c \
@@ -22,7 +23,6 @@ swupd_create_update_SOURCES = \
 	src/helpers.c \
 	src/heuristics.c \
 	src/log.c \
-	src/main.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
@@ -96,6 +96,41 @@ noinst_HEADERS = \
 	include/swupd.h \
 	include/xattrs.h
 
+TEST_EXTENSIONS = .bats
+
+if ENABLE_TESTS
+tap_driver = env AM_TAP_AWK='$(AWK)' $(SHELL) \
+	$(top_srcdir)/tap-driver.sh
+
+LOG_DRIVER = $(tap_driver)
+BATS_LOG_DRIVER = $(tap_driver)
+
+TESTS = $(dist_check_SCRIPTS)
+
+dist_check_SCRIPTS = \
+	test/functional/basic/test.bats \
+	test/functional/delete-no-version-bump/test.bats \
+	test/functional/include-version-bump/test.bats \
+	test/functional/update/test.bats \
+	test/functional/fullfiles/test.bats \
+	test/functional/pack/test.bats \
+	test/functional/full-run/test.bats \
+	test/functional/full-run-delta/test.bats \
+	test/functional/file-name-blacklisted/test.bats
+endif
+
+if COVERAGE
+AM_CFLAGS += --coverage
+
+coverage: coverage-clean
+	mkdir -p coverage
+	lcov --compat-libtool --directory . --capture --output-file coverage/report
+	genhtml -o coverage/ coverage/report
+
+coverage-clean:
+	rm -rf coverage
+endif
+
 distclean-local:
 	rm -rf aclocal.m4 autogen.sh autom4te.cache config.h.in config.h.in~ configure depcomp install-sh Makefile.in missing compile
 

basic_creator.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
 VER=$1
 XZ_DEFAULTS="--threads 0"
-PREVREL=`cat /var/lib/update/image/latest.version`
 
-SWUPDREPO=/usr/src/clear-projects/swupd-server
+SWUPDREPO=${SWUPDREPO:-"/usr/src/clear-projects/swupd-server"}
-BUNDLEREPO=/usr/src/clear-projects/clr-bundles
+BUNDLEREPO=${BUNDLEREPO:-"/usr/src/clear-projects/clr-bundles"}
-UPDATEDIR=/var/lib/update
+UPDATEDIR=${UPDATEDIR:-"/var/lib/update"}
 
 error() {
 	echo "${1:-"Unknown Error"}"
@@ -16,16 +15,16 @@ error() {
 # eg: all of openstack, pnp, bat, cloud stuff, non-basic scripting language bundles
 ${SWUPDREPO}/mk_groups_ini.sh
 
-export SWUPD_CERTS_DIR="/root/swupd-certs"
+PREVREL=`cat ${UPDATEDIR}/image/latest.version`
+
+export SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR:-"/root/swupd-certs"}
 export LEAF_KEY="leaf.key.pem"
 export LEAF_CERT="leaf.cert.pem"
 export CA_CHAIN_CERT="ca-chain.cert.pem"
 export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase"
 
-${SWUPDREPO}/swupd_create_update --osversion ${VER}
+${SWUPDREPO}/swupd_create_update --osversion ${VER} --statedir ${UPDATEDIR} --format 3
-${SWUPDREPO}/swupd_make_fullfiles ${VER}
+${SWUPDREPO}/swupd_make_fullfiles --statedir ${UPDATEDIR} ${VER}
-
-pushd ${SWUPDREPO}
 
 # create zero packs
 MOM=${UPDATEDIR}/www/${VER}/Manifest.MoM
@@ -35,7 +34,7 @@ fi
 BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }')
 for BUNDLE in $BUNDLE_LIST; do
 	#background them for parallelization
-	${SWUPDREPO}/swupd_make_pack 0 ${VER} ${BUNDLE} &
+	${SWUPDREPO}/swupd_make_pack --statedir ${UPDATEDIR} 0 ${VER} ${BUNDLE} &
 done
 #now await all completing
 for job in $(jobs -p); do
@@ -46,20 +45,23 @@ for job in $(jobs -p); do
 	fi
 done
 
-# create delta packs for 2 versions back
-NUM_PACKS=2
-${SWUPDREPO}/pack_maker.sh ${VER} ${NUM_PACKS}
-
-popd ${SWUPDREPO}
-
 # expose the new build to staging / testing
 echo ${VER} > ${UPDATEDIR}/image/latest.version
-STAGING=$(cat ${UPDATEDIR}/www//version/formatstaging/latest)
+
-if [ "${STAGING}" -lt "${VER}" ]; then
+STAGING_FILE="${UPDATEDIR}/www/version/formatstaging/latest"
-	echo ${VER} > ${UPDATEDIR}/www//version/formatstaging/latest
+if [ ! -f "${STAGING_FILE}" ]; then
+	mkdir -p "${UPDATEDIR}/www/version/formatstaging"
+	STAGING=0
+	echo ${STAGING} > ${UPDATEDIR}/www/version/formatstaging/latest
+else
+	STAGING=$(cat ${STAGING_FILE})
 fi
 
-#valgrind /usr/src/clear-projects/swupd-server/swupd_create_update ${VER}
+if [ "${STAGING}" -lt "${VER}" ]; then
-time hardlink /var/lib/update/image/${VER}/*
+	echo ${VER} > ${UPDATEDIR}/www/version/formatstaging/latest
-time hardlink /var/lib/update/image/$PREVREL /var/lib/update/image/${VER}
+fi
-time hardlink /var/lib/update/www/$PREVREL /var/lib/update/www/${VER}
+
+#valgrind ${SWUPDREPO}/swupd_create_update --osversion ${VER} --statedir ${UPDATEDIR}
+time hardlink ${UPDATEDIR}/image/${VER}/*
+time hardlink ${UPDATEDIR}/image/${PREVREL} ${UPDATEDIR}/image/${VER}
+time hardlink ${UPDATEDIR}/www/${PREVREL} ${UPDATEDIR}/www/${VER}

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.0.0, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.2.5, timothy.c.pepper@linux.intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC
@@ -28,6 +28,47 @@ AC_ARG_ENABLE(
 		AC_DEFINE(SWUPD_WITH_STATELESS,1,[OS is stateless])
 )
 
+AC_ARG_ENABLE(
+	[bsdtar],
+	AS_HELP_STRING([--enable-bsdtar],[Use alternative bsdtar command (uses tar by default)]),
+	AS_IF([test "x$enable_bsdtar" = "xyes"],
+		AC_DEFINE([SWUPD_WITH_BSDTAR], 1, [Use alternative bsdtar]),
+		AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])),
+	AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])
+)
+
+AC_ARG_ENABLE(
+  [tests],
+  [AS_HELP_STRING([--disable-tests], [Do not enable functional test framework (enabled by default)])]
+)
+
+have_coverage=no
+AC_ARG_ENABLE(coverage, AS_HELP_STRING([--enable-coverage], [enable test coverage]))
+if test "$enable_coverage" = "yes" ; then
+        AC_CHECK_PROG(lcov_found, [lcov], [yes], [no])
+        if test "$lcov_found" = "no" ; then
+                AC_MSG_ERROR([*** lcov support requested but the program was not found])
+        else
+                lcov_version_major="`lcov --version | cut -d ' ' -f 4 | cut -d '.' -f 1`"
+                lcov_version_minor="`lcov --version | cut -d ' ' -f 4 | cut -d '.' -f 2`"
+                if test "$lcov_version_major" -eq 1 -a "$lcov_version_minor" -lt 10; then
+                        AC_MSG_ERROR([*** lcov version is too old. 1.10 required])
+                else
+			have_coverage=yes
+			AC_DEFINE([COVERAGE], [1], [Coverage enabled])
+                fi
+        fi
+fi
+AM_CONDITIONAL([COVERAGE], [test "$have_coverage" = "yes"])
+
+AS_IF([test "$enable_tests" != "no"], [
+  AC_PATH_PROG([have_bats], [bats])
+  AS_IF([test -z "${have_bats}"], [
+    AC_MSG_ERROR([Must have the Bash Automated Testing System (bats) installed to run functional tests])
+  ])
+])
+AM_CONDITIONAL([ENABLE_TESTS], [test "$enable_tests" != "no"])
+
 AS_IF([test "$enable_bzip2" != "no"], [
   AC_CHECK_LIB([bz2], [BZ2_bzBuffToBuffCompress], [], [AC_MSG_ERROR([the libbz2 library is missing])])
   AC_CHECK_PROGS(BZIP2, bzip2)
@@ -42,4 +83,5 @@ AS_IF([test "$enable_lzma" != "no"], [
 AM_CONDITIONAL([ENABLE_LZMA], [test "$enable_lzma" != "no"])
 
 AC_CONFIG_FILES([Makefile])
+AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_OUTPUT

include/swupd.h

@@ -8,19 +8,32 @@
 
 #include "config.h"
 
-#define __unused__  __attribute__ ((__unused__))
+#define __unused__ __attribute__((__unused__))
 
 // SWUPD_NUM_PACKS is also "PREV_CHECK" in releas tool swupd_bb.py (change both)
-#define SWUPD_NUM_PACKS			4
+#define SWUPD_NUM_PACKS 4
-#define SWUPD_NUM_MANIFEST_DELTAS		25
+#define SWUPD_NUM_MANIFEST_DELTAS 25
-#define SWUPD_DEFAULT_FORMAT	3
 
 #define SWUPD_SERVER_STATE_DIR "/var/lib/update"
 
-#if SWUPD_WITH_SELINUX
+#if SWUPD_WITH_BSDTAR
-#define TAR_PERM_ATTR_ARGS "--preserve-permissions --xattrs --xattrs-include='*' --selinux"
+#define TAR_COMMAND "bsdtar"
+#define TAR_XATTR_ARGS ""
+#define TAR_XATTR_ARGS_STRLIST
+#define TAR_WARN_ARGS ""
 #else
-#define TAR_PERM_ATTR_ARGS "--preserve-permissions --xattrs --xattrs-include='*'"
+#define TAR_COMMAND "tar"
+#define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'"
+#define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'",
+#define TAR_WARN_ARGS "--warning=no-timestamp"
+#endif
+
+#if SWUPD_WITH_SELINUX
+#define TAR_PERM_ATTR_ARGS "--preserve-permissions --selinux " TAR_XATTR_ARGS
+#define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions", "--selinux"
+#else
+#define TAR_PERM_ATTR_ARGS "--preserve-permissions " TAR_XATTR_ARGS
+#define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions"
 #endif
 
 #if SWUPD_WITH_STATELESS
@@ -31,8 +44,8 @@
 
 /* Build types */
 #define REGULAR_BUILD 0
-#define FIX_BUILD     1
+#define FIX_BUILD 1
-#define DEV_BUILD     2
+#define DEV_BUILD 2
 
 /* We don't have BZIP2 support in Android.  This define translates to "ifdef 0"
  * and demarcates codes which otherwise would enable BZIP2.  If this define
@@ -51,72 +64,75 @@
 #endif
 
 struct manifest {
+	unsigned long long int format;
 	int version;
 	int prevversion;
 	char *component;
 	int count;
 	uint64_t contentsize;
 	GList *files;
-	GList *manifests;    /* as struct file */
+	GList *manifests; /* as struct file */
 
 	GList *submanifests; /* as struct manifest */
+
+	GList *includes; /* struct manifests for all bundles included into this one */
 };
 
 struct file;
 struct header;
 
 struct update_stat {
-	uint64_t	st_mode;
+	uint64_t st_mode;
-	uint64_t	st_uid;
+	uint64_t st_uid;
-	uint64_t	st_gid;
+	uint64_t st_gid;
-	uint64_t	st_rdev;
+	uint64_t st_rdev;
-	uint64_t	st_size;
+	uint64_t st_size;
 };
 
 #define DIGEST_LEN_SHA256 64
 /* +1 for null termination */
-#define SWUPD_HASH_LEN (DIGEST_LEN_SHA256+1)
+#define SWUPD_HASH_LEN (DIGEST_LEN_SHA256 + 1)
 
 struct file {
 	char *filename;
 	char hash[SWUPD_HASH_LEN];
 	bool use_xattrs;
-	int  last_change; 
+	int last_change;
 
 	struct update_stat stat;
 
 	/* the following are exclusive */
-	unsigned int is_dir		: 1;
+	unsigned int is_dir : 1;
-	unsigned int is_file		: 1;
+	unsigned int is_file : 1;
-	unsigned int is_link		: 1;
+	unsigned int is_link : 1;
-	unsigned int is_deleted		: 1;
+	unsigned int is_deleted : 1;
-	unsigned int is_manifest	: 1;
+	unsigned int is_manifest : 1;
-	
+
 	/* and these are modifiers */
-	unsigned int is_config		: 1;
+	unsigned int is_config : 1;
-	unsigned int is_state		: 1;
+	unsigned int is_state : 1;
-	unsigned int is_boot		: 1;
+	unsigned int is_boot : 1;
-	unsigned int is_rename		: 1;
+	unsigned int is_rename : 1;
 
-	struct file *peer;  /* same file in another manifest */
+	struct file *peer; /* same file in another manifest */
-	
-        /* data fields to help rename detection */
-        double		rename_score;
-        struct file 	*rename_peer;
-        char		*alpha_only_filename;  /* filename minus all numerics/etc */
-        char		*filetype;
-	char		*basename;
-	char		*dirname;        
-        /* end of rename detection fields */
 
-	unsigned int multithread	: 1;  /* if set to 1, current file is computed in a
+	/* data fields to help rename detection */
+	double rename_score;
+	struct file *rename_peer;
+	char *alpha_only_filename; /* filename minus all numerics/etc */
+	char *filetype;
+	char *basename;
+	char *dirname;
+	/* end of rename detection fields */
+
+	unsigned int multithread : 1; /* if set to 1, current file is computed in a
 						multithreaded process for fullfile creation */
 };
 
 struct packdata {
 	char *module;
-	int  from;
+	int from;
-	int  to;
+	int to;
 	int fullcount;
 	struct manifest *end_manifest;
 };
@@ -124,7 +140,7 @@ struct packdata {
 extern int current_version;
 extern int newversion;
 extern int minversion;
-extern char *format_string;
+extern unsigned long long int format;
 extern bool enable_signing;
 
 extern char *state_dir;
@@ -134,7 +150,7 @@ extern char *staging_dir;
 
 extern bool init_globals(void);
 extern void free_globals(void);
-extern bool set_format_string(char *);
+extern bool set_format(char *);
 extern void check_root(void);
 extern bool set_state_dir(char *);
 extern bool init_state_globals(void);
@@ -146,9 +162,9 @@ extern int file_sort_version(gconstpointer a, gconstpointer b);
 
 extern bool read_configuration_file(char *filename);
 extern void release_configuration_data(void);
-extern char * config_image_base(void);
+extern char *config_image_base(void);
-extern char * config_output_dir(void);
+extern char *config_output_dir(void);
-extern char * config_empty_dir(void);
+extern char *config_empty_dir(void);
 extern int config_initial_version(void);
 
 extern void read_current_version(char *filename);
@@ -163,10 +179,13 @@ extern void free_manifest(struct manifest *manifest);
 extern struct manifest *alloc_manifest(int version, char *module);
 extern int match_manifests(struct manifest *m1, struct manifest *m2);
 extern void sort_manifest_by_version(struct manifest *manifest);
+extern bool manifest_includes(struct manifest *manifest, char *component);
+extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
+extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
-extern void subtract_manifests(struct manifest *m1, struct manifest *m2);
+extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);
 extern void nest_manifest(struct manifest *parent, struct manifest *sub);
 extern void nest_manifest_file(struct manifest *parent, struct file *file);
 extern int manifest_subversion(struct manifest *parent, char *group);
@@ -212,7 +231,7 @@ extern void populate_file_struct(struct file *file, char *filename);
 extern void download_exta_base_content(void);
 
 extern char *get_elapsed_time(struct timeval *t1, struct timeval *t2);
-extern void init_log(int version);
+extern void init_log(const char *prefix, const char *bundle, int start, int end);
 extern void init_log_stdout(void);
 extern void close_log(int version, int exit_status);
 extern void __log_message(struct file *file, char *msg, char *filename, int linenr, const char *fmt, ...);
@@ -228,11 +247,14 @@ extern void __create_delta(struct file *file, int from_version);
 extern void account_delta_hit(void);
 extern void account_delta_miss(void);
 
-extern FILE * fopen_exclusive(const char *filename); /* no mode, opens for write only */
+extern FILE *fopen_exclusive(const char *filename); /* no mode, opens for write only */
 extern void dump_file_info(struct file *file);
 extern void string_or_die(char **strp, const char *fmt, ...);
-extern void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time);
+extern void print_elapsed_time(const char *step, struct timeval *previous_time, struct timeval *current_time);
 extern int system_argv(char *const argv[]);
+extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
+extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+			    char *const argvp2[], int stdoutp2, int stderrp2);
 
 extern bool signature_initialize(void);
 extern void signature_terminate(void);

mk_groups_ini.sh

@@ -1,24 +1,25 @@
 #! /bin/bash
 
 COMMON=${COMMON:-"/root/common"}
-BUNDLEREPO=${BUNDLEREPO:-"/root/clr-bundles"}
+BUNDLEREPO=${BUNDLEREPO:-"/usr/src/clear-projects/clr-bundles"}
 
-SWUPD_SERVER_DIR=${SWUPD_SERVER_DIR:-"/var/lib/update"}
+UPDATEDIR=${UPDATEDIR:-"/var/lib/update"}
-SWUPD_GROUPS_INI=${SWUPD_GROUPS_INI:-"$SWUPD_SERVER_DIR/groups.ini"}
+SWUPD_GROUPS_INI=${SWUPD_GROUPS_INI:-"$UPDATEDIR/groups.ini"}
-SWUPD_SERVER_INI=${SWUPD_SERVER_INI:-"$SWUPD_SERVER_DIR/server.ini"}
+SWUPD_SERVER_INI=${SWUPD_SERVER_INI:-"$UPDATEDIR/server.ini"}
 
-if [ ! -d "$SWUPD_SERVER_DIR" ]; then
+if [ ! -d "$UPDATEDIR" ]; then
-	mkdir -p $SWUPD_SERVER_DIR/{image,www}
+	mkdir -p $UPDATEDIR/{image,www}
 fi
 
 if [ ! -f "$SWUPD_SERVER_INI" ]; then
 	template=$(echo $BASH_SOURCE)
 	template=$(dirname $template)
 	cp -p $template/server.ini $SWUPD_SERVER_INI
+	sed -i "s|/var/lib/update|$UPDATEDIR|" $SWUPD_SERVER_INI
 fi
 
-if [ ! -f "$SWUPD_SERVER_DIR/image/latest.version" ]; then
+if [ ! -f "$UPDATEDIR/image/latest.version" ]; then
-	echo "0" > $SWUPD_SERVER_DIR/image/latest.version
+	echo "0" > $UPDATEDIR/image/latest.version
 fi
 
 echo "rebuilding $SWUPD_GROUPS_INI based on $BUNDLEREPO"

pack_maker.sh

@@ -1,111 +0,0 @@
-#!/bin/bash -e
-# usage:
-#	pack_maker.sh <to version> <back_count>
-#	pack_maker.sh 2820 2
-#	pack_maker.sh 2730 3
-#
-# For each bundle changed as of the Manifest.MoM at the <to version>,
-# create version-pair packs going back <back_count>
-# versions of that bundle.  For example if bundle "editors" changed in 2650
-# and the script is run as "pack_maker.sh 2650 2", it will find that
-# "editors" last two change points were 2550 and 2260 and thus create a
-# 2550-to-2650 and a 2260-to-2650 version pair pack for the "editors"
-# bundle.
-
-#NOTE: fullfiles must be created before calling this script
-#NOTE: zero packs need similarly created
-
-#NOTE: build numbers are sparse, search back for the next oldest existing build
-#NOTE: in the future isn't simply -10, and when that change happens we must also
-#      consider which version-pair packs are meaningful for intermediate builds
-#      and revert builds.
-
-SWUPDREPO=${SWUPDREPO:-"/root/swupd-server"}
-BUNDLEREPO=${BUNDLEREPO:-"/root/clr-bundles"}
-SWUPDWEBDIR="/var/lib/update/www"
-SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR=:-"/root/swupd-certs"}
-
-export XZ_DEFAULTS="--threads 0"
-
-export SWUPD_CERTS_DIR
-export LEAF_KEY="leaf.key.pem"
-export LEAF_CERT="leaf.cert.pem"
-export CA_CHAIN_CERT="ca-chain.cert.pem"
-export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase"
-
-VER=$1
-BACK_COUNT=$2
-
-if [ "${BACK_COUNT}" == "" ]; then
-	echo "missing to pack count"
-	exit 1
-fi
-
-if [ "${VER}" == "" ]; then
-	echo "missing to version"
-	exit 1
-fi
-
-MOM=${SWUPDWEBDIR}/${VER}/Manifest.MoM
-if [ ! -e ${MOM} ]; then
-	echo "invalid version (no ${MOM})"
-	exit 1
-fi
-
-#time ${SWUPDREPO}/swupd_make_fullfiles ${VER}
-
-BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }')
-
-# build packs for all bundles changed in $VER
-for BUNDLE in $BUNDLE_LIST; do
-	#${SWUPDREPO}/swupd_make_pack 0 ${VER} ${BUNDLE} &
-
-	BUNDLE_VER_LIST=""
-	MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$VER/Manifest.MoM | grep "^previous:" | cut -f 2)
-	BUNDLE_VER=""
-	COUNT=0
-	while [ $COUNT -lt $BACK_COUNT ] && [ $MANIFEST_VER -gt 0 ]; do
-		MOM=${SWUPDWEBDIR}/$MANIFEST_VER/Manifest.MoM
-		if [ -e ${MOM} ]; then
-			BUNDLE_VER=$(cat ${MOM} | grep "	${BUNDLE}$" | cut -f 3)
-			if [ "$BUNDLE_VER" != "" ]; then
-				BUNDLE_VER_LIST="$BUNDLE_VER $BUNDLE_VER_LIST"
-				MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$BUNDLE_VER/Manifest.MoM | grep "^previous:" | cut -f 2)
-				let COUNT=$((COUNT+1))
-			else
-				# back in history to where bundle didn't exist yet
-				MANIFEST_VER=0
-			fi
-		else
-			# seek backwards until a manifest if found
-			MANIFEST_VER=$((MANIFEST_VER-10))
-		fi
-	done
-
-	for v in $BUNDLE_VER_LIST; do
-		if [ -e /var/lib/update/www/${VER}/pack-${BUNDLE}-from-$v.tar ]; then
-			echo "${VER}/pack-${BUNDLE}-from-$v.tar already exists, skipping."
-		else
-			${SWUPDREPO}/swupd_make_pack $v ${VER} ${BUNDLE}
-		fi
-	done
-done
-
-for job in $(jobs -p)
-do
-	wait ${job}
-	RET=$?
-	if [ "$RET" != "0" ]; then
-		echo "pack creation failure!"
-		exit $RET
-	fi
-done
-
-# NOTE: your devops will want to expose the completed swupd server build to
-# trial usage at some point.  This would be done via code like:
-#
-#	echo ${VER} > ${SWUPDWEBDIR}/../image/latest.version
-#	STAGING=$(cat ${SWUPDWEBDIR}/version/formatstaging/latest)
-#	if [ "${STAGING}" -lt "${VER}" ]; then
-#		echo ${VER} > ${SWUPDWEBDIR}/version/formatstaging/latest
-#	fi

publish.sh

@@ -13,8 +13,9 @@ if [[ $# < 1 ]]; then
     exit 1
 fi
 FORMAT=$1
+UPDATEDIR=${UPDATEDIR:-"/var/lib/update"}
 
-ver1=`cat /var/lib/update/www/version/formatstaging/latest`
+ver1=`cat ${UPDATEDIR}/www/version/formatstaging/latest`
-ver2=`cat /var/lib/update/www/version/format$FORMAT/latest`
+ver2=`cat ${UPDATEDIR}/www/version/format$FORMAT/latest`
-cp /var/lib/update/www/version/formatstaging/latest /var/lib/update/www/version/format$FORMAT/latest
+cp ${UPDATEDIR}/www/version/formatstaging/latest ${UPDATEDIR}/www/version/format$FORMAT/latest
 echo "Updated from $ver2 to $ver1 in format $FORMAT"

src/analyze_fs.c

@@ -23,19 +23,19 @@
  */
 
 #define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
 #include <assert.h>
+#include <dirent.h>
 #include <errno.h>
 #include <glib.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <openssl/hmac.h>
 #include <linux/limits.h>
+#include <openssl/hmac.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 #include "xattrs.h"
@@ -44,13 +44,13 @@ static GThreadPool *threadpool;
 
 void hash_assign(char *src, char *dst)
 {
-	memcpy(dst, src, SWUPD_HASH_LEN-1);
+	memcpy(dst, src, SWUPD_HASH_LEN - 1);
-	dst[SWUPD_HASH_LEN-1] = '\0';
+	dst[SWUPD_HASH_LEN - 1] = '\0';
 }
 
 bool hash_compare(char *hash1, char *hash2)
 {
-	if (bcmp(hash1, hash2, SWUPD_HASH_LEN-1) == 0) {
+	if (bcmp(hash1, hash2, SWUPD_HASH_LEN - 1) == 0) {
 		return true;
 	} else {
 		return false;
@@ -100,8 +100,8 @@ static void hmac_sha256_for_data(char *hash,
 }
 
 static void hmac_sha256_for_string(char *hash,
-				    const unsigned char *key, size_t key_len,
+				   const unsigned char *key, size_t key_len,
-				    const char *str)
+				   const char *str)
 {
 	if (str == NULL) {
 		hash_set_zeros(hash);
@@ -111,7 +111,6 @@ static void hmac_sha256_for_string(char *hash,
 	hmac_sha256_for_data(hash, key, key_len, (const unsigned char *)str, strlen(str));
 }
 
-
 static void hmac_compute_key(const char *filename,
 			     const struct update_stat *updt_stat,
 			     char *key, size_t *key_len, bool use_xattrs)
@@ -124,14 +123,14 @@ static void hmac_compute_key(const char *filename,
 	}
 
 	hmac_sha256_for_data(key, (const unsigned char *)updt_stat,
-				    sizeof(struct update_stat),
+			     sizeof(struct update_stat),
-				    (const unsigned char *)xattrs_blob,
+			     (const unsigned char *)xattrs_blob,
-				    xattrs_blob_len);
+			     xattrs_blob_len);
 
 	if (hash_is_zeros(key)) {
 		*key_len = 0;
 	} else {
-		*key_len = SWUPD_HASH_LEN-1;
+		*key_len = SWUPD_HASH_LEN - 1;
 	}
 
 	if (xattrs_blob_len != 0) {
@@ -203,14 +202,14 @@ int compute_hash(struct file *file, char *filename)
 		char link[PATH_MAX];
 		memset(link, 0, PATH_MAX);
 
-		ret = readlink(filename, link, PATH_MAX-1);
+		ret = readlink(filename, link, PATH_MAX - 1);
 
 		if (ret >= 0) {
 			hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs);
 			hmac_sha256_for_string(file->hash,
-					(const unsigned char *)key,
+					       (const unsigned char *)key,
-					key_len,
+					       key_len,
-					link);
+					       link);
 			return 0;
 		} else {
 			LOG(NULL, "readlink error ", "%i - %i / %s", ret, errno, strerror(errno));
@@ -221,9 +220,9 @@ int compute_hash(struct file *file, char *filename)
 	if (file->is_dir) {
 		hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs);
 		hmac_sha256_for_string(file->hash,
-					(const unsigned char *)key,
+				       (const unsigned char *)key,
-					key_len,
+				       key_len,
-					file->filename);	//file->filename not filename
+				       file->filename); //file->filename not filename
 		return 0;
 	}
 
@@ -238,10 +237,10 @@ int compute_hash(struct file *file, char *filename)
 
 	hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs);
 	hmac_sha256_for_data(file->hash,
-				(const unsigned char *)key,
+			     (const unsigned char *)key,
-				key_len,
+			     key_len,
-				blob,
+			     blob,
-				file->stat.st_size);
+			     file->stat.st_size);
 	munmap(blob, file->stat.st_size);
 	fclose(fl);
 	return 0;
@@ -281,24 +280,21 @@ static bool illegal_characters(char *filename)
 	char c;
 	int i;
 #define BAD_CHAR_COUNT 11
-	char bad_chars[BAD_CHAR_COUNT] = {';', '&', '|', '*', '`', '/',
+	char bad_chars[BAD_CHAR_COUNT] = { ';', '&', '|', '*', '`', '/',
-					  '<', '>', '\\', '\"', '\''};
+					   '<', '>', '\\', '\"', '\'' };
 
 	// these breaks the tar transform sed-like expression,
 	// hopefully can remove this check after moving to libtar
 	if (strncmp(filename, "+", 1) == 0) {
-		printf("WARNING: Filename %s begins with '+'! ...skipping.\n", filename);
 		return true;
 	}
 	if (strstr(filename, "+package+") != NULL) {
-		printf("WARNING: Filename %s contains \"+package+\"! ...skipping.\n", filename);
 		return true;
 	}
 
-	for(i=0; i<BAD_CHAR_COUNT; i++) {
+	for (i = 0; i < BAD_CHAR_COUNT; i++) {
-		c=bad_chars[i];
+		c = bad_chars[i];
 		if (strchr(filename, c) != NULL) {
-			printf("WARNING: Filename %s includes illegal character '%c'! ...skipping.\n", filename, c);
 			return true;
 		}
 	}
@@ -324,6 +320,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 
 	while (dir) {
 		struct file *file;
+		char *sub_filename;
 		char *fullname;
 
 		entry = readdir(dir);
@@ -332,8 +329,15 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		if ((strcmp(entry->d_name, ".") == 0) ||
-		    (strcmp(entry->d_name, "..") == 0) ||
+		    (strcmp(entry->d_name, "..") == 0)) {
-		    (illegal_characters(entry->d_name))) {
+			continue;
+		}
+
+		string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
+
+		if (illegal_characters(entry->d_name)) {
+			printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+			free(sub_filename);
 			continue;
 		}
 
@@ -343,7 +347,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		file->last_change = manifest->version;
-		string_or_die(&file->filename, "%s/%s", subpath, entry->d_name);
+		file->filename = sub_filename;
 
 		string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
 		populate_file_struct(file, fullname);
@@ -365,7 +369,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		if (do_hash) {
 			/* compute the hash from a thread */
 			ret = g_thread_pool_push(threadpool, file, &err);
-			if (ret == FALSE){
+			if (ret == FALSE) {
 				printf("GThread hash computation push error\n");
 				printf("%s\n", err->message);
 				closedir(dir);
@@ -379,7 +383,6 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 	free(fullpath);
 }
 
-
 struct manifest *full_manifest_from_directory(int version)
 {
 	struct manifest *manifest;
@@ -404,6 +407,58 @@ struct manifest *full_manifest_from_directory(int version)
 	return manifest;
 }
 
+/* Read includes from $manifest-includes file */
+GList *get_sub_manifest_includes(char *component, int version)
+{
+	FILE *infile;
+	GList *includes = NULL;
+	char *c;
+	char *conf;
+	char *filename;
+	char *included;
+	char line[8192];
+
+	conf = config_output_dir();
+	if (conf == NULL) {
+		assert(0);
+	}
+
+	string_or_die(&filename, "%s/%i/noship/%s-includes", conf, version, component);
+	free(conf);
+
+	LOG(NULL, "Reading includes", "%s", filename);
+	infile = fopen(filename, "rb");
+
+	if (infile == NULL) {
+		if (errno != ENOENT) {
+			LOG(NULL, "Cannot read includes", "%s (%s)\n", filename, strerror(errno));
+		}
+		free(filename);
+		return NULL;
+	}
+
+	line[0] = 0;
+	while (strcmp(line, "\n") != 0) {
+		line[0] = 0;
+		if (fgets(line, 8191, infile) == NULL) {
+			break;
+		}
+		c = strchr(line, '\n');
+		if (c) {
+			*c = 0;
+		}
+		if (strlen(line) == 0) {
+			break;
+		}
+		included = strdup(line);
+		includes = g_list_prepend(includes, included);
+	}
+	fclose(infile);
+	includes = g_list_sort(includes, (GCompareFunc)strcmp);
+
+	return includes;
+}
+
 struct manifest *sub_manifest_from_directory(char *component, int version)
 {
 	struct manifest *manifest;
@@ -421,6 +476,8 @@ struct manifest *sub_manifest_from_directory(char *component, int version)
 
 	manifest->files = g_list_sort(manifest->files, file_sort_filename);
 
+	manifest->includes = get_sub_manifest_includes(component, version);
+
 	return manifest;
 }
 

src/chroot.c

@@ -21,37 +21,35 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-
+#include <sys/types.h>
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
 void chroot_create_full(int newversion)
 {
-	int ret;
 	char *group;
-	char * command;
+	char *param;
 	char *full_dir;
 
 	string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
 
 	g_mkdir_with_parents(full_dir, S_IRWXU);
 
-
 	/* start with base */
 	LOG(NULL, "Copying chroot os-core to full", "");
-	string_or_die(&command, "rsync -aAX %s/%i/os-core/ %s",
+	string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
-	              image_dir, newversion, full_dir);
+	char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
-	ret = system(command);
+	if (system_argv(rsynccmd) != 0) {
-	assert(ret==0);
+		assert(0);
-	free(command);
+	}
+	free(param);
 
 	/* overlay any new files from each group */
 	while (1) {
@@ -61,31 +59,12 @@ void chroot_create_full(int newversion)
 		}
 
 		LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
-		string_or_die(&command, "rsync -aAX --ignore-existing %s/%i/%s/ %s",
+		string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
-		             image_dir, newversion, group, full_dir);
+		char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
-		ret = system(command);
+		if (system_argv(rsynccmd) != 0) {
-		assert(ret==0);
+			assert(0);
-		free(command);
-	}
-
-	/* and since some dumb files have things like install timestamps in
-	 * them, we pull files from full back to bundles so each instance
-	 * of the "same" file is truly identical across bundles.  It may
-	 * make sense at some point to add more logic here for conflict
-	 * detection and resolution, including work to insure renames are
-	 * consistent across groups. */
-	while (1) {
-		group = next_group();
-		if (!group) {
-			break;
 		}
-
+		free(param);
-		LOG(NULL, "Recopy bundle chroot out of full", "%s", group);
-		string_or_die(&command, "rsync -aAX --existing %s %s/%i/%s",
-		             full_dir, image_dir, newversion, group);
-		ret = system(command);
-		assert(ret==0);
-		free(command);
 	}
 	free(full_dir);
 }

src/config.c

@@ -20,17 +20,16 @@
  *
  */
 
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <assert.h>
-#include <glib.h>
 
 #include "swupd.h"
 
 static GKeyFile *keyfile = NULL;
 
-
 char *config_image_base(void)
 {
 	assert(keyfile != NULL);
@@ -68,7 +67,6 @@ int config_initial_version(void)
 	return version;
 }
 
-
 bool read_configuration_file(char *filename)
 {
 	GError *error = NULL;

src/create_update.c

@@ -24,19 +24,18 @@
  *
  */
 
-
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <getopt.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <glib.h>
-#include <errno.h>
 #include <sys/stat.h>
-#include <sys/wait.h>
 #include <sys/time.h>
-#include <getopt.h>
+#include <sys/wait.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
@@ -48,18 +47,19 @@ static void banner(void)
 }
 
 static const struct option prog_opts[] = {
-	{"help", no_argument, 0, 'h'},
+	{ "help", no_argument, 0, 'h' },
-	{"version", no_argument, 0, 'v'},
+	{ "version", no_argument, 0, 'v' },
-	{"osversion", required_argument, 0, 'o'},
+	{ "osversion", required_argument, 0, 'o' },
-	{"minversion", required_argument, 0, 'm'},
+	{ "minversion", required_argument, 0, 'm' },
-	{"format", required_argument, 0, 'F'},
+	{ "format", required_argument, 0, 'F' },
-	{"getformat", no_argument, 0, 'g'},
+	{ "getformat", no_argument, 0, 'g' },
-	{"statedir", required_argument, 0, 'S'},
+	{ "statedir", required_argument, 0, 'S' },
-	{"signcontent", no_argument, 0, 's'},
+	{ "signcontent", no_argument, 0, 's' },
-	{0, 0, 0, 0}
+	{ 0, 0, 0, 0 }
 };
 
-static void print_help(const char *name) {
+static void print_help(const char *name)
+{
 	printf("Usage:\n");
 	printf("   %s [OPTION...]\n\n", name);
 	printf("Help Options:\n");
@@ -69,7 +69,7 @@ static void print_help(const char *name) {
 	printf("Application Options:\n");
 	printf("   -o, --osversion         The OS version for which to create an update\n");
 	printf("   -m, --minversion        Optional minimum file version to write into manifests per file\n");
-	printf("   -F, --format            Optional format string [ default:=%d ]\n", SWUPD_DEFAULT_FORMAT);
+	printf("   -F, --format            Format number for the update\n");
 	printf("   -g, --getformat         Print current format string and exit\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
 	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
@@ -100,12 +100,12 @@ static bool parse_options(int argc, char **argv)
 		case 'm':
 			minversion = strtoull(optarg, NULL, 10);
 			if (minversion < 0) {
-				printf("-m | --minversion == %d, require value >= 0 \n", newversion);
+				printf("-m | --minversion == %d, require value >= 0 \n", minversion);
 				return false;
 			}
 			break;
 		case 'F':
-			if (!optarg || !set_format_string(optarg)) {
+			if (!optarg || !set_format(optarg)) {
 				printf("Invalid --format argument\n\n");
 				return false;
 			}
@@ -117,10 +117,10 @@ static bool parse_options(int argc, char **argv)
 			}
 			break;
 		case 'g':
-			if (format_string == NULL) {
+			if (format == 0) {
-				printf("%d\n", SWUPD_DEFAULT_FORMAT);
+				printf("No format specified\n");
 			} else {
-				printf("%s\n", format_string);
+				printf("%llu\n", format);
 				free_globals();
 			}
 			exit(0);
@@ -223,8 +223,11 @@ int main(int argc, char **argv)
 	struct manifest *old_full = NULL;
 	struct manifest *new_full = NULL;
 
+	GHashTable *new_manifests = g_hash_table_new(g_str_hash, g_str_equal);
+	GHashTable *old_manifests = g_hash_table_new(g_str_hash, g_str_equal);
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
+	int old_deleted = 0;
 
 	struct timeval current_time;
 	struct timeval previous_time;
@@ -256,7 +259,6 @@ int main(int argc, char **argv)
 		goto exit;
 	}
 
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	if (!read_configuration_file(file_path)) {
 		printf("Failed to read %s configuration file!\n", state_dir);
@@ -275,7 +277,7 @@ int main(int argc, char **argv)
 	populate_dirs(newversion);
 
 	printf("Next version is %i \n", newversion);
-	init_log(newversion);
+	init_log("swupd-create-update", "", current_version, newversion);
 
 	gettimeofday(&previous_time, NULL);
 
@@ -286,7 +288,7 @@ int main(int argc, char **argv)
 	printf("Syncing bundle chroots with full chroot\n");
 	chroot_create_full(newversion);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("full chroot creation", &previous_time, &current_time);
 
 	printf("Calculating full manifest (this is expensive/slow)\n");
 	old_full = manifest_from_file(current_version, "full");
@@ -294,18 +296,25 @@ int main(int argc, char **argv)
 	apply_heuristics(old_full);
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
+
+	old_deleted = remove_old_deleted_files(old_full, new_full);
+	if (old_deleted > 0) {
+		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
+		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
+	}
+
 	apply_heuristics(new_full);
 #warning disabled rename detection for some simplicity
-/*	rename_detection(new_full); */
+	// rename_detection(new_full);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("full manifest creation", &previous_time, &current_time);
 
 	printf("Entering phase 2: Doing the os-core bundle\n");
 	/* Phase 2 : the os-core set */
 
 	/* Step 2: Make a manifest for the os-core set */
-	old_MoM =  manifest_from_file(current_version, "MoM");
+	old_MoM = manifest_from_file(current_version, "MoM");
-	new_MoM =  alloc_manifest(newversion, "MoM");
+	new_MoM = alloc_manifest(newversion, "MoM");
 	old_core = manifest_from_file(manifest_subversion(old_MoM, "os-core"), "os-core");
 	new_core = sub_manifest_from_directory("os-core", newversion);
 	add_component_hashes_to_manifest(new_core, new_full);
@@ -329,10 +338,11 @@ int main(int argc, char **argv)
 		/* Step 3c: ... else save the manifest */
 		type_change_detection(new_core);
 
-		/* Detect renamed files specifically for each pack */
 #warning disabled rename detection for some simplicity
-/*		rename_detection(...); */
+		/* Detect renamed files specifically for each pack */
+		// rename_detection(...);
 
+		old_deleted = remove_old_deleted_files(old_core, new_core);
 		sort_manifest_by_version(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
@@ -340,20 +350,56 @@ int main(int argc, char **argv)
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d), writing out new manifest", newfiles);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
-		printf("Core component has changes (%d), writing out new manifest\n", newfiles);
+		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
 			goto exit;
 		}
-		create_manifest_deltas(new_core, manifests_last_versions_list);
 	}
 
 	nest_manifest(new_MoM, new_core);
 
 	/* Phase 3: the functional bundles */
 	printf("Entering phase 3: The bundles\n");
+	while (1) {
+		char *group = next_group();
+
+		if (!group) {
+			break;
+		}
+
+		(void)g_hash_table_insert(new_manifests, group, sub_manifest_from_directory(group, newversion));
+		(void)g_hash_table_insert(old_manifests, group, manifest_from_file(manifest_subversion(old_MoM, group), group));
+	}
+	while (1) {
+		GList *manifest_includes = NULL;
+		GList *name_includes;
+		char *group = next_group();
+		struct manifest *manifest;
+
+		if (!group) {
+			break;
+		}
+		manifest = g_hash_table_lookup(new_manifests, group);
+		name_includes = manifest->includes;
+		while (name_includes) {
+			char *name = name_includes->data;
+			name_includes = g_list_next(name_includes);
+			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(new_manifests, name));
+		}
+		manifest->includes = manifest_includes;
+		manifest_includes = NULL;
+		manifest = g_hash_table_lookup(old_manifests, group);
+		name_includes = manifest->includes;
+		while (name_includes) {
+			char *name = name_includes->data;
+			name_includes = g_list_next(name_includes);
+			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(old_manifests, name));
+		}
+		manifest->includes = manifest_includes;
+	}
 	while (1) {
 		char *group = next_group();
 		struct manifest *oldm;
@@ -370,19 +416,27 @@ int main(int argc, char **argv)
 		printf("Processing bundle %s\n", group);
 
 		/* Step 4: Make a manifest for this functonal group */
-		oldm = manifest_from_file(manifest_subversion(old_MoM, group), group);
+		oldm = g_hash_table_lookup(old_manifests, group);
-		newm = sub_manifest_from_directory(group, newversion);
+		newm = g_hash_table_lookup(new_manifests, group);
 		add_component_hashes_to_manifest(newm, new_full);
 		apply_heuristics(oldm);
 		apply_heuristics(newm);
 		newm->prevversion = oldm->version;
 
+		/* add os-core as an included manifest */
+		if (!manifest_includes(oldm, "os-core")) {
+			oldm->includes = g_list_prepend(oldm->includes, old_core);
+		}
+		if (!manifest_includes(newm, "os-core")) {
+			newm->includes = g_list_prepend(newm->includes, new_core);
+		}
+
 		/* Step 5: Subtract the core files from the manifest */
-		subtract_manifests(oldm, old_core);
+		subtract_manifests_frontend(oldm, oldm);
-		subtract_manifests(newm, new_core);
+		subtract_manifests_frontend(newm, newm);
 
 		/* Step 6: Compare manifest to the previous version... */
-		if (match_manifests(oldm, newm) == 0) {
+		if (match_manifests(oldm, newm) == 0 && !changed_includes(oldm, newm)) {
 			LOG(NULL, "", "%s components have not changed, no new manifest", group);
 			printf("%s components have not changed, no new manifest\n", group);
 			/* Step 6a: if nothing changed, stay at the old version */
@@ -391,18 +445,18 @@ int main(int argc, char **argv)
 			apply_heuristics(newm);
 #warning missing rename_detection here
 			/* Step 6b: otherwise, write out the manifest */
+			old_deleted = remove_old_deleted_files(oldm, newm);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0) {
+			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
-				LOG(NULL, "", "%s component has changes (%d), writing out new manifest", group, newfiles);
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
-				printf("%s component has changes (%d), writing out new manifest\n", group, newfiles);
+				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);
 					goto exit;
 				}
-				create_manifest_deltas(newm, manifests_last_versions_list);
 			} else {
 				LOG(NULL, "", "%s component has not changed (after pruning), no new manifest", group);
 				printf("%s component has not changed (after pruning), no new manifest\n", group);
@@ -413,7 +467,7 @@ int main(int argc, char **argv)
 		nest_manifest(new_MoM, newm);
 	}
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("bundle manifest creation", &previous_time, &current_time);
 
 	printf("Entering phase 4: completion manifests \n");
 	/* Phase 4 : manifest completion */
@@ -424,15 +478,13 @@ int main(int argc, char **argv)
 		printf("Failed to write new MoM\n");
 		goto exit;
 	}
-	create_manifest_deltas(new_MoM, manifests_last_versions_list);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("MoM creation", &previous_time, &current_time);
 
 	printf("Entering phase 5: creating download content\n");
 	/* Phase 5: wrapping up */
 
-
+	//TODO: should be fixup_versions() and preceed all write_manifest() calls
-//TODO: should be fixup_versions() and preceed all write_manifest() calls
 	maximize_to_full(new_MoM, new_full);
 
 	sort_manifest_by_version(new_full);
@@ -444,7 +496,7 @@ int main(int argc, char **argv)
 	/* Step 8: Prepare delta directory */
 	prepare_delta_dir(new_full);
 
-	print_elapsed_time(&previous_time, &current_time);
+	print_elapsed_time("finalize update", &previous_time, &current_time);
 
 	exit_status = EXIT_SUCCESS;
 exit:

src/delta.c

@@ -21,26 +21,24 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <bsdiff.h>
+#include <errno.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <errno.h>
+#include <sys/types.h>
-
+#include <unistd.h>
-#include <bsdiff.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 #include "xattrs.h"
 
 void __create_delta(struct file *file, int from_version)
 {
-	char *original, *newfile, *outfile, *dotfile, *testnewfile, *sanitycheck;
+	char *original, *newfile, *outfile, *dotfile, *testnewfile;
-	char *conf;
+	char *conf, *param1, *param2;
 	int ret;
 
 	if (file->is_link) {
@@ -67,7 +65,6 @@ void __create_delta(struct file *file, int from_version)
 	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
 	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
 	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&sanitycheck, "cmp -s \"%s\" \"%s\"", newfile, testnewfile);
 
 	LOG(file, "Making delta", "%s->%s", original, newfile);
 
@@ -112,13 +109,18 @@ void __create_delta(struct file *file, int from_version)
 		goto out;
 	}
 
-	ret = system(sanitycheck);
+	string_or_die(&param1, "%s", newfile);
-	if (ret == -1 || !WIFEXITED(ret) || WEXITSTATUS(ret) == 2) {
+	string_or_die(&param2, "%s", testnewfile);
+	char *const sanitycheck[] = { "cmp", "-s", param1, param2, NULL };
+	ret = system_argv(sanitycheck);
+	free(param1);
+	free(param2);
+	if (ret == -1 || ret == 2) {
 		printf("Sanity check system command failed %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
 		assert(0);
 		goto out;
-	} else if (WEXITSTATUS(ret) == 1) {
+	} else if (ret == 1) {
 		printf("Delta application resulted in file mismatch %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);
 		LOG(file, "Delta mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile);
@@ -138,7 +140,7 @@ void __create_delta(struct file *file, int from_version)
 		}
 		LOG(NULL, "Failed to rename", "");
 	}
-out:	free(sanitycheck);
+out:
 	free(testnewfile);
 	free(conf);
 	free(newfile);
@@ -156,7 +158,6 @@ void prepare_delta_dir(struct manifest *manifest)
 
 	conf = config_output_dir();
 
-
 	string_or_die(&path, "%s/%i", conf, manifest->version);
 	if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
 		if (errno != EEXIST) {
@@ -176,4 +177,3 @@ void prepare_delta_dir(struct manifest *manifest)
 	free(path);
 	free(conf);
 }
-

src/fullfiles.c

@@ -22,18 +22,18 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <glib.h>
+#include <libgen.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <stdint.h>
+#include <sys/types.h>
-#include <errno.h>
+#include <unistd.h>
-#include <libgen.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 
@@ -42,15 +42,15 @@
 static void create_fullfile(struct file *file)
 {
 	char *origin;
-	char *tarcommand=NULL;
+	char *tarname = NULL;
-	char *tarname=NULL;
 	char *rename_source = NULL;
 	char *rename_target = NULL;
 	char *rename_tmpdir = NULL;
 	int ret;
 	struct stat sbuf;
 	char *empty, *indir, *outdir;
-
+	char *param1, *param2;
+	int stderrfd;
 
 	if (file->is_deleted) {
 		return; /* file got deleted -> by definition we cannot tar it up */
@@ -88,19 +88,27 @@ static void create_fullfile(struct file *file)
 		dir = dirname(tmp2);
 
 		string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir);
-		if(!mkdtemp(rename_tmpdir)) {
+		if (!mkdtemp(rename_tmpdir)) {
 			LOG(NULL, "Failed to create temporary directory for %s move", origin);
 			assert(0);
 		}
 
-		string_or_die(&tarcommand, "tar -C %s " TAR_PERM_ATTR_ARGS " -cf - '%s' --exclude='%s'/* 2> /dev/null | "
+		string_or_die(&param1, "--exclude=%s/?*", base);
-			"tar -C %s " TAR_PERM_ATTR_ARGS " -xf - 2> /dev/null",
+		string_or_die(&param2, "./%s", base);
-			 dir, base, base, rename_tmpdir);
+		char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
-		if (system(tarcommand) != 0) {
+		char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+
+		stderrfd = open("/dev/null", O_WRONLY);
+		if (stderrfd == -1) {
+			LOG(NULL, "Failed to open /dev/null", "");
 			assert(0);
 		}
-		free(tarcommand);
+		if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
+			assert(0);
+		}
+		free(param1);
+		free(param2);
+		close(stderrfd);
 
 		string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
 		string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
@@ -111,13 +119,13 @@ static void create_fullfile(struct file *file)
 		free(rename_source);
 
 		/* for a directory file, tar up simply with gzip */
-		string_or_die(&tarcommand, "tar -C %s %s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar",
+		string_or_die(&param1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
-			      rename_tmpdir, file->hash, outdir, file->last_change, file->hash);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL };
-		if (system(tarcommand) != 0) {
+
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+		if (system_argv(tarcmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
 
 		if (rmdir(rename_target)) {
 			LOG(NULL, "rmdir failed for %s", rename_target);
@@ -126,6 +134,7 @@ static void create_fullfile(struct file *file)
 		if (rmdir(rename_tmpdir)) {
 			LOG(NULL, "rmdir failed for %s", rename_tmpdir);
 		}
+		free(rename_tmpdir);
 
 		free(tmp1);
 		free(tmp2);
@@ -138,7 +147,7 @@ static void create_fullfile(struct file *file)
 		string_or_die(&tempfile, "%s/%s", empty, file->hash);
 		if (link(origin, tempfile) < 0) {
 			LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile);
-			char *const argv[] = {"cp", "-a", origin, tempfile, NULL};
+			char *const argv[] = { "cp", "-a", origin, tempfile, NULL };
 			if (system_argv(argv) != 0) {
 				assert(0);
 			}
@@ -146,29 +155,38 @@ static void create_fullfile(struct file *file)
 
 		/* step 2a: tar it with each compression type  */
 		// lzma
-		string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -Jcf %s/%i/files/%s.tar.xz %s",
+		string_or_die(&param1, "--directory=%s", empty);
-			     empty, outdir, file->last_change, file->hash, file->hash);
+		string_or_die(&param2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
-		if (system(tarcommand) != 0) {
+		char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL };
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+
+		if (system_argv(tarlzmacmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 		// gzip
-		string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar.gz %s",
+		string_or_die(&param1, "--directory=%s", empty);
-			     empty, outdir, file->last_change, file->hash, file->hash);
+		string_or_die(&param2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
-		if (system(tarcommand) != 0) {
+		char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL };
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+
+		if (system_argv(targzipcmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 #ifdef SWUPD_WITH_BZIP2
-		string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -jcf %s/%i/files/%s.tar.bz2 %s",
+		string_or_die(&param1, "--directory=%s", empty);
-			     empty, outdir, file->last_change, file->hash, file->hash);
+		string_or_die(&param2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
-		if (system(tarcommand) != 0) {
+		char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL };
-			LOG(NULL, "Failed to run command:", "%s", tarcommand);
+
+		if (system_argv(tarbzip2cmd) != 0) {
 			assert(0);
 		}
-		free(tarcommand);
+		free(param1);
+		free(param2);
+
 #endif
 
 		/* step 2b: pick the smallest of the three compression formats */
@@ -271,13 +289,13 @@ static void submit_fullfile_tasks(GList *files)
 	GList *item;
 	struct file *file;
 	int ret;
-	int count=0;
+	int count = 0;
 	GError *err = NULL;
 
 	LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
 	threadpool = g_thread_pool_new(create_fullfile_task, NULL,
-				sysconf(_SC_NPROCESSORS_ONLN) * 3,
+				       sysconf(_SC_NPROCESSORS_ONLN) * 3,
-				TRUE, NULL);
+				       TRUE, NULL);
 
 	printf("Starting downloadable fullfiles data creation\n");
 
@@ -287,7 +305,7 @@ static void submit_fullfile_tasks(GList *files)
 		item = g_list_next(item);
 
 		ret = g_thread_pool_push(threadpool, file, &err);
-		if (ret == FALSE){
+		if (ret == FALSE) {
 			printf("GThread create_fullfile_task push error\n");
 			printf("%s\n", err->message);
 			assert(0);
@@ -304,27 +322,24 @@ void create_fullfiles(struct manifest *manifest)
 {
 	GList *deduped_file_list;
 	char *path;
-	char *cmd;
 	char *conf, *empty;
 
 	empty = config_empty_dir();
 
-	string_or_die(&cmd, "rm -rf %s", empty);
+	char *const rmcmd[] = { "rm", "-rf", empty, NULL };
-	if (system(cmd) != 0) {
+	if (system_argv(rmcmd) != 0) {
-		LOG(NULL, "Failed to run command:", "%s", cmd);
 		assert(0);
 	}
-	free(cmd);
 	g_mkdir_with_parents(empty, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
 	free(empty);
 
 	conf = config_output_dir();
 
 	string_or_die(&path, "%s/%i", conf, manifest->version);
-	if(mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
+	if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
 		if (errno != EEXIST) {
 			LOG(NULL, "Failed to create directory ", "%s", path);
-				return;
+			return;
 		}
 	}
 	free(path);
@@ -332,7 +347,7 @@ void create_fullfiles(struct manifest *manifest)
 	if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
 		if (errno != EEXIST) {
 			LOG(NULL, "Failed to create directory ", "%s", path);
-				return;
+			return;
 		}
 	}
 	free(path);

src/globals.c

@@ -21,19 +21,19 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
+#include <unistd.h>
-#include <errno.h>
 
 #include "swupd.h"
 
 int newversion = -1;
 int minversion = 0;
-char *format_string = NULL;
+unsigned long long int format = 0;
 bool enable_signing = false;
 
 char *state_dir = NULL;
@@ -41,20 +41,17 @@ char *packstage_dir = NULL;
 char *image_dir = NULL;
 char *staging_dir = NULL;
 
-bool set_format_string(char *userinput)
+bool set_format(char *userinput)
 {
-	int version;
+	unsigned long long int user_format;
 
-	// expect a positive integer
+	// format string shall be a positive integer
 	errno = 0;
-	version = strtoull(userinput, NULL, 10);
+	user_format = strtoull(userinput, NULL, 10);
-	if ((errno < 0) || (version <= 0)) {
+	if ((errno < 0) || (user_format == 0)) {
 		return false;
 	}
-	if (format_string) {
+	format = user_format;
-		free(format_string);
-	}
-	string_or_die(&format_string, "%d", version);
 
 	return true;
 }
@@ -81,8 +78,9 @@ bool set_state_dir(char *dir)
 
 bool init_globals(void)
 {
-	if (format_string == NULL) {
+	if (format == 0) {
-		string_or_die(&format_string, "%d", SWUPD_DEFAULT_FORMAT);
+		printf("Error: Missing format parameter. Please specify a format with -F.\n");
+		return false;
 	}
 
 	if (!init_state_globals()) {
@@ -101,7 +99,6 @@ bool init_globals(void)
 
 void free_globals(void)
 {
-	free(format_string);
 	free(state_dir);
 	free(packstage_dir);
 	free(image_dir);

src/groups.c

@@ -23,11 +23,11 @@
  *
  */
 
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <assert.h>
-#include <glib.h>
 
 #include "swupd.h"
 
@@ -51,7 +51,6 @@ char *group_groups(char *group)
 	return g_key_file_get_value(groupfile, group, "groups", NULL);
 }
 
-
 void read_group_file(char *filename)
 {
 	GError *error = NULL;
@@ -76,7 +75,6 @@ void release_group_file(void)
 	}
 }
 
-
 char *next_group(void)
 {
 	if (groupcursor >= groupcount) {

src/helpers.c

@@ -21,33 +21,31 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/time.h>
+#include <sys/types.h>
 #include <sys/wait.h>
-#include <fcntl.h>
+#include <unistd.h>
-#include <errno.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 
-
+FILE *fopen_exclusive(const char *filename) /* no mode, opens for write only */
-FILE * fopen_exclusive(const char *filename) /* no mode, opens for write only */
 {
-        int fd;
+	int fd;
 
-        fd = open(filename,O_CREAT | O_EXCL | O_RDWR , 00644);
+	fd = open(filename, O_CREAT | O_EXCL | O_RDWR, 00644);
-        if (fd < 0) {
+	if (fd < 0) {
 		printf("exclusive open failed, filename=\"%s\",err=\"%s\"\n",
-			filename, strerror(errno));
+		       filename, strerror(errno));
 		LOG(NULL, "exclusive open failed", "\\*filename=\"%s\",err=\"%s\"*\\",
-			filename, strerror(errno));
+		    filename, strerror(errno));
 		return NULL;
 	}
 	return fdopen(fd, "w");
@@ -103,22 +101,27 @@ void string_or_die(char **strp, const char *fmt, ...)
 	va_list ap;
 
 	va_start(ap, fmt);
-	if (vasprintf(strp, fmt, ap) <= 0) {
+	if (vasprintf(strp, fmt, ap) < 0) {
 		assert(0);
 	}
 	va_end(ap);
 }
 
-void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time)
+/* "step" should be a descriptive string for the process that completed before
+ * this function is called */
+void print_elapsed_time(const char *step, struct timeval *previous_time,
+			struct timeval *current_time)
 {
 	char *elapsed;
 
+	assert(strlen(step) > 0);
+
 	gettimeofday(current_time, NULL);
 	elapsed = get_elapsed_time(previous_time, current_time);
 	if (strcmp(elapsed, " ") == 0) {
-		printf("\t~0.0 elapsed for Manifest.full creation\n");
+		printf("\t~0.0 elapsed for '%s' step\n", step);
 	} else {
-		printf("\t%s elapsed for full chroot creation\n", elapsed);
+		printf("\t%s elapsed for '%s' step\n", elapsed, step);
 	}
 	previous_time->tv_sec = current_time->tv_sec;
 	previous_time->tv_usec = current_time->tv_usec;
@@ -150,7 +153,7 @@ int system_argv(char *const argv[])
 {
 	int child_exit_status;
 	pid_t pid;
-	int status;
+	int status = -1;
 
 	pid = fork();
 
@@ -177,15 +180,99 @@ int system_argv(char *const argv[])
 		}
 
 		if (status != 0) {
-			char* cmdline = NULL;
+			char *cmdline = NULL;
 
 			concat_str_array(&cmdline, argv);
 			LOG(NULL, "Failed to run command:", "%s", cmdline);
 			free(cmdline);
 		}
-
-		return status;
 	}
+
+	return status;
+}
+
+int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr)
+{
+	int child_exit_status;
+	pid_t pid;
+	int status = -1;
+
+	pid = fork();
+
+	if (pid == 0) { /* child */
+		if (newstdin >= 0) {
+			if (dup2(newstdin, STDIN_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stdin", "");
+				assert(0);
+			}
+			close(newstdin);
+		}
+		if (newstdout >= 0) {
+			if (dup2(newstdout, STDOUT_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stdout", "");
+				assert(0);
+			}
+			close(newstdout);
+		}
+		if (newstderr >= 0) {
+			if (dup2(newstderr, STDERR_FILENO) == -1) {
+				LOG(NULL, "Could not redirect stderr", "");
+				assert(0);
+			}
+			close(newstderr);
+		}
+
+		execvp(*argv, argv);
+		LOG(NULL, "This line must not be reached", "");
+		assert(0);
+	} else if (pid < 0) {
+		LOG(NULL, "Failed to fork a child process", "");
+		assert(0);
+	} else {
+		pid_t ws = waitpid(pid, &child_exit_status, 0);
+
+		if (ws == -1) {
+			LOG(NULL, "Failed to wait for child process", "");
+			assert(0);
+		}
+
+		if (WIFEXITED(child_exit_status)) {
+			status = WEXITSTATUS(child_exit_status);
+		} else {
+			LOG(NULL, "Child process didn't exit", "");
+			assert(0);
+		}
+
+		if (status != 0) {
+			char *cmdline = NULL;
+
+			concat_str_array(&cmdline, argv);
+			LOG(NULL, "Failed to run command:", "%s", cmdline);
+			free(cmdline);
+		}
+	}
+
+	return status;
+}
+
+int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+		     char *const argvp2[], int stdoutp2, int stderrp2)
+{
+	int statusp2;
+	int pipefd[2];
+
+	if (pipe(pipefd)) {
+		LOG(NULL, "Failed to create a pipe", "");
+		return -1;
+	}
+	system_argv_fd(argvp1, stdinp1, pipefd[1], stderrp1);
+	close(pipefd[1]);
+	statusp2 = system_argv_fd(argvp2, pipefd[0], stdoutp2, stderrp2);
+	close(pipefd[0]);
+
+	/* Returns the status of the failed process if any
+       If both processes failed returns the status of first one */
+	return statusp2;
 }
 
 void check_root(void)

src/heuristics.c

@@ -21,18 +21,15 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
+#include <unistd.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 
-
-
 /* TODO: this needs to come from a file that has a list of regexps */
 
 static void config_file_heuristics(struct file *file)

src/log.c

@@ -22,15 +22,14 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <stdarg.h>
 #include <sys/time.h>
-
+#include <unistd.h>
-#include <glib.h>
 
 #include "swupd.h"
 
@@ -38,11 +37,12 @@ static FILE *logfile;
 
 static struct timeval start_time;
 
-void init_log(int version)
+void init_log(const char *prefix, const char *bundle, int start, int end)
 {
-	char filename[4096];
+	char *filename;
-	sprintf(filename, "swupd-server.log.%i", version);
+	string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end);
 	logfile = fopen(filename, "w");
+	free(filename);
 	gettimeofday(&start_time, NULL);
 }
 void init_log_stdout(void)
@@ -93,7 +93,7 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 	char filebuf2[4096];
 
 	if (!logfile) {
-		return;\
+		return;
 	}
 
 	gettimeofday(&current_time, NULL);
@@ -119,13 +119,12 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 		strcat(filebuf2, " ");
 	}
 
-	fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", 
+	fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
-		(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);	
+		(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
 
 	free(logstring);
 	free(buf);
 	fflush(logfile);
-
 }
 
 void close_log(int version, int exit_status)
@@ -133,7 +132,7 @@ void close_log(int version, int exit_status)
 	struct timeval current_time;
 	int t_sec;
 	int t_msec;
-	
+
 	if (!logfile) {
 		return;
 	}

src/make_fullfiles.c

@@ -21,20 +21,19 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <getopt.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
+#include <unistd.h>
-
-#include <getopt.h>
 
 #include "swupd.h"
 
 static const struct option prog_opts[] = {
-	{"help", no_argument, 0, 'h'},
+	{ "help", no_argument, 0, 'h' },
-	{"statedir", required_argument, 0, 'S'},
+	{ "statedir", required_argument, 0, 'S' },
-	{0, 0, 0, 0}
+	{ 0, 0, 0, 0 }
 };
 
 static void usage(const char *name)
@@ -93,12 +92,6 @@ int main(int argc, char **argv)
 		free_state_globals();
 		return EXIT_FAILURE;
 	}
-	banner();
-	check_root();
-
-	string_or_die(&file_path, "%s/server.ini", state_dir);
-	read_configuration_file(file_path);
-	free(file_path);
 
 	if (argc - optind != 1) {
 		usage(argv[0]);
@@ -106,12 +99,21 @@ int main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	}
 
+	banner();
+	check_root();
+
+	string_or_die(&file_path, "%s/server.ini", state_dir);
+	read_configuration_file(file_path);
+	free(file_path);
+
 	version = strtoull(argv[optind++], NULL, 10);
 	if (version < 0) {
 		printf("Usage:\n\tswupd_make_fullfiles <version>\n\n");
 		exit(EXIT_FAILURE);
 	}
 
+	init_log("swupd-make-fullfiles", "", 0, version);
+
 	manifest = manifest_from_file(version, "full");
 	create_fullfiles(manifest);
 

src/make_packs.c

@@ -21,20 +21,18 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <getopt.h>
+#include <getopt.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <dirent.h>
+#include <sys/types.h>
-#include <getopt.h>
+#include <unistd.h>
-
-#include <glib.h>
-
-#include <getopt.h>
 
 #include "swupd.h"
 
@@ -46,10 +44,10 @@ static void banner(void)
 }
 
 static const struct option prog_opts[] = {
-	{"help", no_argument, 0, 'h'},
+	{ "help", no_argument, 0, 'h' },
-	{"statedir", required_argument, 0, 'S'},
+	{ "statedir", required_argument, 0, 'S' },
-	{"signcontent", no_argument, 0, 's'},
+	{ "signcontent", no_argument, 0, 's' },
-	{0, 0, 0, 0}
+	{ 0, 0, 0, 0 }
 };
 
 static void usage(const char *name)
@@ -107,17 +105,14 @@ int main(int argc, char **argv)
 		free_state_globals();
 		return EXIT_FAILURE;
 	}
-	banner();
-	check_root();
 
 	if (argc - optind != 3) {
 		usage(argv[0]);
 		exit(EXIT_FAILURE);
 	}
 
-	/* FIXME: should use "end_version" not "0" and a unique filename
+	banner();
-	init_log(0);
+	check_root();
-	*/
 
 	/* Initilize the crypto signature module */
 	if (!signature_initialize()) {
@@ -140,6 +135,8 @@ int main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	}
 
+	init_log("swupd-make-pack-", module, start_version, end_version);
+
 	printf("Making pack-%s %i to %li\n", module, start_version, end_version);
 
 	pack = calloc(1, sizeof(struct packdata));
@@ -158,8 +155,8 @@ int main(int argc, char **argv)
 	signature_terminate();
 
 	printf("Pack creation %s (pack-%s %i to %li)\n",
-		exit_status == EXIT_SUCCESS ? "complete" : "failed",
+	       exit_status == EXIT_SUCCESS ? "complete" : "failed",
-		module, start_version, end_version);
+	       module, start_version, end_version);
 
 	free_state_globals();
 	return exit_status;

src/manifest.c

@@ -23,41 +23,39 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <bsdiff.h>
+#include <errno.h>
+#include <glib.h>
+#include <libgen.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <libgen.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-
+#include <unistd.h>
-#include <bsdiff.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 #include "xattrs.h"
 
-#define LINK_SIZE_HINT			1024
+#define LINK_SIZE_HINT 1024
-#define DIR_SIZE_HINT			1024
+#define DIR_SIZE_HINT 1024
 
 int file_sort_hash(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
-	A = (struct file *) a;
+	A = (struct file *)a;
-	B = (struct file *) b;
+	B = (struct file *)b;
 
 	/* this MUST be a memcmp(), not bcmp() */
-	return memcmp(A->hash, B->hash, SWUPD_HASH_LEN-1);
+	return memcmp(A->hash, B->hash, SWUPD_HASH_LEN - 1);
 }
 
 int file_sort_version(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
-	A = (struct file *) a;
+	A = (struct file *)a;
-	B = (struct file *) b;
+	B = (struct file *)b;
 
 	if (A->last_change < B->last_change) {
 		return -1;
@@ -73,8 +71,8 @@ int file_sort_filename(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
 	int ret;
-	A = (struct file *) a;
+	A = (struct file *)a;
-	B = (struct file *) b;
+	B = (struct file *)b;
 
 	ret = strcmp(A->filename, B->filename);
 	if (ret) {
@@ -101,19 +99,21 @@ struct manifest *alloc_manifest(int version, char *component)
 
 	manifest->version = version;
 	manifest->component = strdup(component);
+	manifest->format = format;
 
 	return manifest;
 }
 
-
 struct manifest *manifest_from_file(int version, char *component)
 {
 	FILE *infile;
+	GList *includes = NULL;
 	char line[8192], *c, *c2;
 	int count = 0;
 	struct manifest *manifest;
 	char *filename, *conf;
 	int previous = 0;
+	unsigned long long int format_number;
 
 	conf = config_output_dir();
 	if (conf == NULL) {
@@ -139,19 +139,21 @@ struct manifest *manifest_from_file(int version, char *component)
 		return NULL;
 	}
 
-	if (strncmp(line, "MANIFEST\t", 9)!=0) {
+	if (strncmp(line, "MANIFEST\t", 9) != 0) {
 		printf("Invalid file format: MANIFEST line\n");
 		fclose(infile);
 		return NULL;
 	}
 	c = &line[9];
-	if (strtoull(c, NULL, 10) == 0) {
+	format_number = strtoull(c, NULL, 10);
+	if ((errno < 0) || (format_number == 0)) {
+		//format string shall be a positive integer
 		printf("Unknown file format version in MANIFEST line: %s\n", c);
 		fclose(infile);
 		return NULL;
 	}
 	line[0] = 0;
-	while (strcmp(line, "\n")!=0) {
+	while (strcmp(line, "\n") != 0) {
 		/* read the header */
 		line[0] = 0;
 		if (fgets(line, 8191, infile) == NULL) {
@@ -173,16 +175,24 @@ struct manifest *manifest_from_file(int version, char *component)
 			assert(0);
 		}
 
-		if (strncmp(line,"version:", 8) == 0) {
+		if (strncmp(line, "version:", 8) == 0) {
 			version = strtoull(c, NULL, 10);
 		}
-		if (strncmp(line,"previous:", 9) == 0) {
+		if (strncmp(line, "previous:", 9) == 0) {
 			previous = strtoull(c, NULL, 10);
 		}
+		if (strncmp(line, "includes:", 9) == 0) {
+			includes = g_list_prepend(includes, strdup(c));
+			if (!includes->data) {
+				abort();
+			}
+		}
 	}
 
 	manifest = alloc_manifest(version, component);
+	manifest->format = format_number;
 	manifest->prevversion = previous;
+	manifest->includes = includes;
 
 	/* empty line */
 	while (!feof(infile)) {
@@ -286,7 +296,7 @@ struct manifest *manifest_from_file(int version, char *component)
 			manifest->files = g_list_prepend(manifest->files, file);
 		}
 		manifest->count++;
-		count ++;
+		count++;
 	}
 
 	manifest->files = g_list_sort(manifest->files, file_sort_filename);
@@ -296,7 +306,6 @@ struct manifest *manifest_from_file(int version, char *component)
 	return manifest;
 }
 
-
 void free_manifest(struct manifest *manifest)
 {
 	struct file *file;
@@ -314,7 +323,6 @@ void free_manifest(struct manifest *manifest)
 	free(manifest);
 }
 
-
 /*
 backfill the "last changed" of each file in a manifest
 by comparing the hash against the same file in the previous manifest
@@ -355,7 +363,6 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 		file1->peer = NULL;
 		file2->peer = NULL;
 
-
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
 			if (file1->is_deleted && file2->is_deleted && file1->is_rename) {
@@ -364,16 +371,16 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			}
 
 			if (hash_compare(file1->hash, file2->hash) &&
-				file1->is_dir == file2->is_dir &&
+			    file1->is_dir == file2->is_dir &&
-				file1->is_link == file2->is_link &&
+			    file1->is_link == file2->is_link &&
-				file1->is_deleted == file2->is_deleted &&
+			    file1->is_deleted == file2->is_deleted &&
-				file1->is_file == file2->is_file &&
+			    file1->is_file == file2->is_file &&
-				file1->is_config == file2->is_config &&
+			    file1->is_config == file2->is_config &&
-				file1->is_state == file2->is_state &&
+			    file1->is_state == file2->is_state &&
-				file1->is_boot == file2->is_boot &&
+			    file1->is_boot == file2->is_boot &&
-				file1->last_change >= minversion) {
+			    file1->last_change >= minversion) {
-					file2->last_change = file1->last_change;
+				file2->last_change = file1->last_change;
-					file2->is_rename = file1->is_rename;
+				file2->is_rename = file1->is_rename;
 			} else {
 				account_changed_file();
 				if (first) {
@@ -398,11 +405,13 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 		}
 		if (ret < 0) {
 			struct file *file3;
-/*
+			/*
- * if we get here, file1 got deleted... what we must do is add a file entry for it in the target list.
+			 * if we get here, file1 got deleted... what we must do
- * However, since we're currently walking the list we HAVE to prepend the entry.. and mark for sort at
+			 * is add a file entry for it in the target list.
- * the end.
+			 * However, since we're currently walking the list we
- */
+			 * HAVE to prepend the entry.. and mark for sort at the
+			 * end.
+			 */
 			file3 = calloc(1, sizeof(struct file));
 			if (file3 == NULL) {
 				assert(0);
@@ -427,7 +436,7 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			file1->peer = file3;
 
 			list1 = g_list_next(list1);
-			m2->files = g_list_prepend(m2->files, file3); 
+			m2->files = g_list_prepend(m2->files, file3);
 			m2->count++;
 			if (!file1->is_deleted) {
 				account_deleted_file();
@@ -447,7 +456,7 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 		count++;
 	}
 
-/* now deal with the tail ends */
+	/* now deal with the tail ends */
 	while (list1) {
 		file1 = list1->data;
 
@@ -458,11 +467,12 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 			first = 0;
 		}
 		count++;
-/*
+		/*
- * if we get here, file1 got deleted... what we must do is add a file entry for it in the target list.
+		 * if we get here, file1 got deleted... what we must do is add
- * However, since we're currently walking the list we HAVE to prepend the entry.. and mark for sort at
+		 * a file entry for it in the target list.  However, since
- * the end.
+		 * we're currently walking the list we HAVE to prepend the
- */
+		 * entry.. and mark for sort at the end.
+		 */
 		file3 = calloc(1, sizeof(struct file));
 		if (file3 == NULL) {
 			assert(0);
@@ -510,30 +520,57 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 	return count;
 }
 
+static GList *get_unique_includes(struct manifest *manifest)
+{
+	GHashTable *unique_includes = g_hash_table_new(g_str_hash, g_str_equal);
+	GHashTableIter iter;
+	GList *includes = NULL;
+	GList *l1, *l2;
+	gpointer k, v;
 
-/* removes all files from m2 from m1 */
+	l1 = g_list_first(manifest->includes);
+	while (l1) {
+		struct manifest *m = l1->data;
+		l1 = g_list_next(l1);
+		(void)g_hash_table_replace(unique_includes, m->component, m);
+		l2 = get_unique_includes(m);
+		while (l2) {
+			struct manifest *m = l2->data;
+			l2 = g_list_next(l2);
+			(void)g_hash_table_replace(unique_includes, m->component, m);
+		}
+		g_list_free(l2);
+	}
+	g_hash_table_iter_init(&iter, unique_includes);
+	while (g_hash_table_iter_next(&iter, &k, &v)) {
+		includes = g_list_prepend(includes, v);
+	}
+
+	return includes;
+}
+
+/*
+ * removes all files from m2 from m1
+ * This will recurse over all included manifests to also
+ * subtract those from the m1 manifest.
+ *
+ * As a special convenient semantics,
+ * subtract_manifests(M, M)
+ * will subtract all included manifests from M,
+ * but will not subtract M from M itself.
+ */
 void subtract_manifests(struct manifest *m1, struct manifest *m2)
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
 
-	if (!m1) {
-		printf("Subtracting manifests failed: No m1 manifest!\n");
-		return;
-	}
-
-	if (!m2) {
-		printf("Subtracting manifests failed: No m2 manifest!\n");
-		return;
-	}
-
 	m1->files = g_list_sort(m1->files, file_sort_filename);
 	m2->files = g_list_sort(m2->files, file_sort_filename);
 
 	list1 = g_list_first(m1->files);
 	list2 = g_list_first(m2->files);
 
-	while (list1 && list2) {
+	while (list1 && list2 && m1 != m2) {
 		int ret;
 		file1 = list1->data;
 		file2 = list2->data;
@@ -547,21 +584,40 @@ void subtract_manifests(struct manifest *m1, struct manifest *m2)
 
 			if (file1->is_deleted == file2->is_deleted && file1->is_file == file2->is_file) {
 				m1->files = g_list_delete_link(m1->files, todel);
-				m1->count --;
+				m1->count--;
-				continue;
 			}
-		}
+		} else if (ret < 0) {
-
-		if (ret < 0) {
 			list1 = g_list_next(list1);
-			continue;
+		} else {
+			list2 = g_list_next(list2);
 		}
-		list2 = g_list_next(list2);
 	}
 }
 
+void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2)
+{
+	GList *includes;
+	struct manifest *m;
 
+	if (!m1) {
+		printf("Subtracting manifests failed: No m1 manifest!\n");
+		return;
+	}
 
+	if (!m2) {
+		printf("Subtracting manifests failed: No m2 manifest!\n");
+		return;
+	}
+
+	subtract_manifests(m1, m2);
+
+	includes = get_unique_includes(m2);
+	while (includes) {
+		m = includes->data;
+		includes = g_list_next(includes);
+		subtract_manifests(m1, m);
+	}
+}
 
 char *file_type_to_string(struct file *file)
 {
@@ -650,7 +706,7 @@ static int write_manifest_signature(struct manifest *manifest, const char *suffi
 		assert(0);
 	}
 	string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version,
-			manifest->component, suffix);
+		      manifest->component, suffix);
 	if (!signature_sign(filename)) {
 		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
 		goto exit;
@@ -665,6 +721,7 @@ exit:
 /* Returns 0 == success, -1 == failure */
 static int write_manifest_plain(struct manifest *manifest)
 {
+	GList *includes;
 	GList *list;
 	struct file *file;
 	FILE *out = NULL;
@@ -672,6 +729,8 @@ static int write_manifest_plain(struct manifest *manifest)
 	char *conf = config_output_dir();
 	char *filename = NULL;
 	char *submanifest_filename = NULL;
+	char *manifest_tempdir = NULL;
+	char *tempmanifest = NULL;
 	int ret = -1;
 
 	if (conf == NULL) {
@@ -685,7 +744,7 @@ static int write_manifest_plain(struct manifest *manifest)
 	}
 	dir = dirname(base);
 
-	if (g_mkdir_with_parents(dir,  S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
+	if (g_mkdir_with_parents(dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) {
 		assert(0);
 	}
 
@@ -695,13 +754,19 @@ static int write_manifest_plain(struct manifest *manifest)
 		goto exit;
 	}
 
-	fprintf(out, "MANIFEST\t%s\n", format_string);
+	fprintf(out, "MANIFEST\t%llu\n", format);
 	fprintf(out, "version:\t%i\n", manifest->version);
 	fprintf(out, "previous:\t%i\n", manifest->prevversion);
 	fprintf(out, "filecount:\t%i\n", manifest->count);
 	fprintf(out, "timestamp:\t%i\n", (int)time(NULL));
 	compute_content_size(manifest);
 	fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
+	includes = manifest->includes;
+	while (includes) {
+		struct manifest *sub = includes->data;
+		includes = g_list_next(includes);
+		fprintf(out, "includes:\t%s\n", sub->component);
+	}
 	fprintf(out, "\n");
 
 	list = g_list_first(manifest->files);
@@ -713,24 +778,61 @@ static int write_manifest_plain(struct manifest *manifest)
 	}
 
 	list = g_list_first(manifest->manifests);
+	string_or_die(&manifest_tempdir, "%s/XXXXXX", state_dir);
+	if (!mkdtemp(manifest_tempdir)) {
+		LOG(NULL, "Failed to create temporary directory %s for untarring Manifests", manifest_tempdir);
+		assert(0);
+	}
+
 	while (list) {
 		file = list->data;
 		list = g_list_next(list);
 
 		string_or_die(&submanifest_filename, "%s/%i/Manifest.%s", conf, file->last_change, file->filename);
 		populate_file_struct(file, submanifest_filename);
-		ret = compute_hash(file, submanifest_filename);
+
+		if (file->is_deleted) {
+			goto write_entry;
+		}
+
+		char *param1, *param2;
+
+		/* Untar the Manifest.BUNDLE.tar and calculate the hash on that,
+		 * otherwise we may get incorrect hashes due to owner permissions on
+		 * the files. */
+		string_or_die(&param1, "%s", manifest_tempdir);
+		string_or_die(&param2, "%s.tar", submanifest_filename);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param1, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", param2, NULL };
+		ret = system_argv(tarcmd);
+		free(param1);
+		free(param2);
+
+		if (ret != 0) {
+			assert(0);
+		}
+
+		string_or_die(&tempmanifest, "%s/Manifest.%s", manifest_tempdir, file->filename);
+		populate_file_struct(file, tempmanifest);
+		ret = compute_hash(file, tempmanifest);
 		if (ret != 0) {
 			printf("Hash computation failed\n");
 			assert(0);
 		}
-
+		unlink(tempmanifest);
+		free(tempmanifest);
+	write_entry:
 		fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename);
 		free(submanifest_filename);
 	}
 
 	ret = 0;
 exit:
+	if (manifest_tempdir) {
+		if (rmdir(manifest_tempdir) != 0) {
+			LOG(NULL, "rmdir failed for %s: %s", manifest_tempdir, strerror(errno));
+		}
+		free(manifest_tempdir);
+	}
 	if (out) {
 		fclose(out);
 	}
@@ -744,34 +846,37 @@ exit:
 static int write_manifest_tar(struct manifest *manifest)
 {
 	char *conf = config_output_dir();
-	char *tarcmd = NULL;
+	char *directory, *manifesttar, *manifestcomp, *manifestsigned;
-	int ret = -1;
+	int ret = 0;
 
 	if (conf == NULL) {
-	    assert(0);
+		assert(0);
 	}
 
+	string_or_die(&directory, "--directory=%s/%i", conf, manifest->version);
+	string_or_die(&manifesttar, "%s/%i/Manifest.%s.tar", conf, manifest->version, manifest->component);
+	string_or_die(&manifestcomp, "Manifest.%s", manifest->component);
+	string_or_die(&manifestsigned, "Manifest.%s.signed", manifest->component);
+
 	/* now, tar the thing up for efficient full file download */
 	/* and put the signature of the plain manifest into the archive, too */
 	if (enable_signing) {
-		string_or_die(&tarcmd, "tar --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
+		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-			      "%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed",
+					 manifesttar, manifestcomp, manifestsigned, NULL };
-			      conf, manifest->version, conf, manifest->version, manifest->component,
+		ret = system_argv(tarcmd);
-			      manifest->component, manifest->component);
 	} else {
-		string_or_die(&tarcmd, "tar --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
+		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-			      "%s/%i/Manifest.%s.tar Manifest.%s",
+					 manifesttar, manifestcomp, NULL };
-			      conf, manifest->version, conf, manifest->version, manifest->component,
+		ret = system_argv(tarcmd);
-			      manifest->component);
+	}
+	if (ret) {
+		fprintf(stderr, "Creation of Manifest.tar failed\n");
 	}
 
-	if (system(tarcmd) != 0) {
+	free(directory);
-		fprintf(stderr, "Creation of Manifest.tar failed\n");
+	free(manifesttar);
-		goto exit;
+	free(manifestcomp);
-	}
+	free(manifestsigned);
-	ret = 0;
-exit:
-	free(tarcmd);
 	free(conf);
 	return ret;
 }
@@ -793,7 +898,8 @@ bool compute_hash_with_xattrs(const char *filename)
 		return true;
 	}
 
-	return false;
+	/* Currently swupd-client includes xattrs to hash sums of all files */
+	return true;
 }
 
 /* Returns 0 == success, -1 == failure */
@@ -818,6 +924,115 @@ void sort_manifest_by_version(struct manifest *manifest)
 	}
 }
 
+bool manifest_includes(struct manifest *manifest, char *component)
+{
+	GList *includes = g_list_first(manifest->includes);
+
+	while (includes) {
+		if (strcmp(((struct manifest *)includes->data)->component,
+			   component) == 0) {
+			return true;
+		}
+		includes = g_list_next(includes);
+	}
+
+	return false;
+}
+
+/* This requires the manifest to have the includes sorted.
+ * The manifests will be sorted when constructed from
+ * get_sub_manifest_includes (and then replacing the manifest
+ * strings with actual manifest structs in the same order as
+ * is done in the main.c's main()).
+ */
+bool changed_includes(struct manifest *old, struct manifest *new)
+{
+	GList *includes_old = g_list_first(old->includes);
+	GList *includes_new = g_list_first(new->includes);
+
+	if (g_list_length(includes_old) != g_list_length(includes_new)) {
+		return true;
+	}
+
+	while (includes_old && includes_new) {
+		char *bundle_old = ((struct manifest *)includes_old->data)->component;
+		char *bundle_new = ((struct manifest *)includes_new->data)->component;
+		if (strcmp(bundle_old, bundle_new) != 0) {
+			return true;
+		}
+
+		includes_old = g_list_next(includes_old);
+		includes_new = g_list_next(includes_new);
+	}
+
+	return false;
+}
+
+/* For a format bump, it's convenient to remove deleted files from manifests
+ * that last changed prior to the format bump, since they are no longer
+ * considered for deletion as part of an update.
+ *
+ * Note: this function should be called after match_manifests().
+ */
+int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
+{
+	GList *list1, *list2;
+	struct file *file1, *file2;
+	int count = 0;
+
+	if (!m1) {
+		printf("No old manifest!\n");
+		return -1;
+	}
+
+	if (!m2) {
+		printf("No new manifest!\n");
+		return -1;
+	}
+
+	/* This is the common case, so bail early, reporting no deletions */
+	if (m1->format == m2->format) {
+		return 0;
+	}
+
+	/* At this point, the manifest formats mismatch, and it's assumed that
+	 * m1 is the old manifest, and m2 is the new.
+	 */
+
+	m1->files = g_list_sort(m1->files, file_sort_filename);
+	m2->files = g_list_sort(m2->files, file_sort_filename);
+
+	list1 = g_list_first(m1->files);
+	list2 = g_list_first(m2->files);
+
+	while (list1 && list2) {
+		int ret;
+		file1 = list1->data;
+		file2 = list2->data;
+
+		ret = strcmp(file1->filename, file2->filename);
+		if (ret == 0) {
+			if (file1->is_deleted && file2->is_deleted) {
+				GList *to_delete = list2;
+				list1 = g_list_next(list1);
+				list2 = g_list_next(list2);
+				m2->files = g_list_delete_link(m2->files, to_delete);
+				m2->count--;
+				count++;
+				continue;
+			}
+			list1 = g_list_next(list1);
+			list2 = g_list_next(list2);
+		} else if (ret < 0) {
+			list1 = g_list_next(list1);
+		} else {
+			list2 = g_list_next(list2);
+		}
+	}
+
+	return count;
+}
+
 /* Conditionally remove some things from a manifest.
  * Returns > 0 when the pruned manifest has new files.
  * Returns 0 when the pruned manifest no longer has new files.
@@ -946,7 +1161,6 @@ void nest_manifest(struct manifest *parent, struct manifest *sub)
 	parent->count++;
 }
 
-
 /* adds a struct file for a submanifest and also adds it to the proper list */
 void nest_manifest_file(struct manifest *parent, struct file *file)
 {
@@ -958,11 +1172,9 @@ void nest_manifest_file(struct manifest *parent, struct file *file)
 	parent->submanifests = g_list_prepend(parent->submanifests, sub);
 	parent->count++;
 
-
 	LOG(file, "Nest manifest file", "%s", file->filename);
 }
 
-
 int manifest_subversion(struct manifest *parent, char *group)
 {
 	GList *list;
@@ -1011,7 +1223,7 @@ static void maximize_version_manifests(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->last_change > file2->last_change) {
+			if (!file1->is_deleted && file1->last_change > file2->last_change) {
 				LOG(file1, "Update", "Moving %s to version %i", file1->filename, file1->last_change);
 				file2->last_change = file1->last_change;
 			}
@@ -1028,7 +1240,6 @@ static void maximize_version_manifests(struct manifest *m1, struct manifest *m2)
 	}
 }
 
-
 /*
  * We need this in case files move modules;
  * the "full" list must be the maximum of all the versions of
@@ -1170,17 +1381,15 @@ void consolidate_submanifests(struct manifest *manifest)
 
 		/* (case 6) all others constitute errors */
 		LOG(NULL, "unhandled filename pair: file1", "%s %s (%d), file2 %s %s (%d)",
-			file1->filename, file1->hash, file1->last_change,
+		    file1->filename, file1->hash, file1->last_change,
-			file1->filename, file2->hash, file2->last_change);
+		    file1->filename, file2->hash, file2->last_change);
 		manifest->files = g_list_delete_link(manifest->files, list);
 		list = g_list_next(next);
 		manifest->files = g_list_delete_link(manifest->files, next);
 		printf("CONFLICT IN MANIFESTS\n");
-
 	}
 }
 
-
 int previous_version_manifest(struct manifest *mom, char *name)
 {
 	GList *list;
@@ -1200,4 +1409,3 @@ int previous_version_manifest(struct manifest *mom, char *name)
 	}
 	return 0;
 }
-

src/pack.c

@@ -24,43 +24,44 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <glib.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <dirent.h>
+#include <sys/types.h>
-
+#include <unistd.h>
-#include <glib.h>
 
 #include "swupd.h"
 
-static void empty_pack_stage(int full, int version, char *module)
+static void empty_pack_stage(int full, int from_version, int to_version, char *module)
 {
-	char *cmd;
+	char *param;
 	char *path;
-	int ret;
 
 	// clean any stale data (eg: re-run after a failure)
-	string_or_die(&cmd, "rm -rf %s/%s/%i/", packstage_dir, module, version);
+	string_or_die(&param, "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version);
-	ret = system(cmd);
+	char *const rmcmd[] = { "rm", "-fr", param, NULL };
-	if (ret) {
+	if (system_argv(rmcmd) != 0) {
-		fprintf(stderr, "Failed to clean %s/%s/%i\n",
+		fprintf(stderr, "Failed to clean %s/%s/%i_to_%i\n",
-			packstage_dir, module, version);
+			packstage_dir, module, from_version, to_version);
+		free(param);
 		exit(EXIT_FAILURE);
 	}
-	free(cmd);
+	free(param);
 
 	if (!full) {
 		// (re)create module/version/{delta,staged}
-		string_or_die(&path, "%s/%s/%i/delta", packstage_dir, module, version);
+		string_or_die(&path, "%s/%s/%i_to_%i/delta", packstage_dir, module,
+			      from_version, to_version);
 		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 		free(path);
-		string_or_die(&path, "%s/%s/%i/staged", packstage_dir, module, version);
+		string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
+			      from_version, to_version);
 		g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 		free(path);
 	}
@@ -69,14 +70,15 @@ static void empty_pack_stage(int full, int version, char *module)
 /*
  * for very small files (eh.. lets make that all), we untar them for pack purposes
  */
-static void explode_pack_stage(int version, char *module)
+static void explode_pack_stage(int from_version, int to_version, char *module)
 {
 	DIR *dir;
 	struct dirent *entry;
 	struct stat buf;
 	char *path;
 
-	string_or_die(&path, "%s/%s/%i/staged", packstage_dir, module, version);
+	string_or_die(&path, "%s/%s/%i_to_%i/staged", packstage_dir, module,
+		      from_version, to_version);
 	g_mkdir_with_parents(path, S_IRWXU | S_IRWXG);
 	dir = opendir(path);
 	if (!dir) {
@@ -85,7 +87,7 @@ static void explode_pack_stage(int version, char *module)
 	}
 	free(path);
 	while (1) {
-		char *path, *tar;
+		char *path, *param;
 		int ret;
 
 		entry = readdir(dir);
@@ -99,8 +101,8 @@ static void explode_pack_stage(int version, char *module)
 			continue;
 		}
 
-		string_or_die(&path, "%s/%s/%i/staged/%s",
+		string_or_die(&path, "%s/%s/%i_to_%i/staged/%s", packstage_dir, module,
-		              packstage_dir, module, version, entry->d_name);
+			      from_version, to_version, entry->d_name);
 		ret = stat(path, &buf);
 		if (ret) {
 			free(path);
@@ -113,13 +115,12 @@ static void explode_pack_stage(int version, char *module)
 		 * the resulting pack is slightly smaller, and in addition, we're saving CPU
 		 * time on the client...
 		 */
-		string_or_die(&tar, "tar --directory=%s/%s/%i/staged --warning=no-timestamp "
+		string_or_die(&param, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version);
-				TAR_PERM_ATTR_ARGS " -axf %s", packstage_dir, module, version, path);
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
-		ret = system(tar);
+		if (system_argv(tarcmd) == 0) {
-		if (!ret) {
 			unlink(path);
 		}
-		free(tar);
+		free(param);
 		free(path);
 	}
 	closedir(dir);
@@ -139,7 +140,7 @@ static void prepare_pack(struct packdata *pack)
 
 	pack->end_manifest = manifest_from_file(pack->to, pack->module);
 
-	empty_pack_stage(0, pack->from, pack->module);
+	empty_pack_stage(0, pack->from, pack->to, pack->module);
 
 	match_manifests(manifest, pack->end_manifest);
 
@@ -153,7 +154,7 @@ static void make_pack_full_files(struct packdata *pack)
 	int ret;
 
 	LOG(NULL, "starting pack full file creation", "%s: %d to %d",
-		pack->module, pack->from, pack->to);
+	    pack->module, pack->from, pack->to);
 
 	/* 	full files pack: */
 	item = g_list_first(pack->end_manifest->files);
@@ -162,25 +163,49 @@ static void make_pack_full_files(struct packdata *pack)
 		item = g_list_next(item);
 		if ((!file->peer || file->peer->is_deleted) && !file->is_deleted && !file->rename_peer) {
 			char *from, *to;
+			char *fullfrom, *fullto;
+
 			/* hardlink each file that is in <end> but not in <X> */
+			string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+			string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+				      pack->module, pack->from, pack->to, file->hash);
 			string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash);
-			string_or_die(&to, "%s/%s/%i/staged/%s.tar", packstage_dir, pack->module, pack->from, file->hash);
+			string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
-			ret = link(from, to);
+				      pack->module, pack->from, pack->to, file->hash);
-			if (ret) {
+
-				if (errno != EEXIST) {
+			ret = -1;
-					LOG(NULL, "Failure to link", "%s to %s (%s) %i", from, to, strerror(errno), errno);
+			errno = 0;
+
+			/* Prefer to hardlink uncompressed files (excluding
+			 * directories) first, and fall back to the compressed
+			 * versions if the hardlink fails.
+			 */
+			if (!file->is_dir) {
+				ret = link(fullfrom, fullto);
+				if (ret && errno != EEXIST) {
+					LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno);
 				}
-			} else {
+			}
+			if (ret) {
+				ret = link(from, to);
+				if (ret && errno != EEXIST) {
+					LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno);
+				}
+			}
+
+			if (ret == 0) {
 				pack->fullcount++;
 			}
 
 			free(from);
 			free(to);
+			free(fullfrom);
+			free(fullto);
 		}
 	}
 
 	LOG(NULL, "finished pack full file creation", "%s: %d to %d",
-		pack->module, pack->from, pack->to);
+	    pack->module, pack->from, pack->to);
 }
 
 static int find_file_in_list(GList *files, struct file *file)
@@ -195,8 +220,8 @@ static int find_file_in_list(GList *files, struct file *file)
 		item = g_list_next(item);
 
 		if (hash_compare(item_file->hash, file->hash) &&
-				(item_file->last_change == file->last_change) &&
+		    (item_file->last_change == file->last_change) &&
-				(item_file->peer->last_change == file->peer->last_change)) {
+		    (item_file->peer->last_change == file->peer->last_change)) {
 			LOG(NULL, "Found a duplicate delta", "%d %d %s %s", file->peer->last_change, file->last_change, file->hash, file->filename);
 			return TRUE;
 		}
@@ -230,7 +255,7 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 		}
 
 		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-		              file->peer->last_change, file->last_change, file->hash);
+			      file->peer->last_change, file->last_change, file->hash);
 
 		ret = stat(from, &stat_delta);
 		if (ret && !find_file_in_list(files, file)) {
@@ -263,7 +288,7 @@ static void make_pack_deltas(GList *files)
 
 	LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
 	threadpool = g_thread_pool_new(create_delta, NULL,
-				sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
+				       sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
 
 	item = g_list_first(files);
 	while (item) {
@@ -271,7 +296,7 @@ static void make_pack_deltas(GList *files)
 		item = g_list_next(item);
 
 		ret = g_thread_pool_push(threadpool, file, &err);
-		if (ret == FALSE){
+		if (ret == FALSE) {
 			// intentionally non-fatal
 			fprintf(stderr, "GThread create_delta push error\n");
 			fprintf(stderr, "%s\n", err->message);
@@ -289,7 +314,7 @@ static int write_pack_signature(struct packdata *pack)
 	int ret = -1;
 
 	string_or_die(&filename, "%s/%i/pack-%s-from-%i.tar",
-			staging_dir, pack->to, pack->module, pack->from);
+		      staging_dir, pack->to, pack->module, pack->from);
 	if (!signature_sign(filename)) {
 		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
 		goto exit;
@@ -306,7 +331,7 @@ static int make_final_pack(struct packdata *pack)
 	GList *item;
 	struct file *file;
 	int ret;
-	char *tar;
+	char *param1, *param2;
 	double penalty;
 
 	LOG(NULL, "make_final_pack", "%s: %i to %i", pack->module, pack->from, pack->to);
@@ -314,7 +339,7 @@ static int make_final_pack(struct packdata *pack)
 	item = g_list_first(pack->end_manifest->files);
 
 	while (item) {
-		char *from, *to, *tarfrom, *tarto;
+		char *from, *to, *tarfrom, *tarto, *fullfrom, *fullto;
 		struct stat stat_delta, stat_tar;
 
 		file = item->data;
@@ -329,13 +354,17 @@ static int make_final_pack(struct packdata *pack)
 		/* for each file changed since <X> */
 		/* locate delta, check if the diff it's from is >= <X> */
 		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-				file->peer->last_change, file->last_change, file->hash);
+			      file->peer->last_change, file->last_change, file->hash);
-		string_or_die(&to, "%s/%s/%i/delta/%i-%i-%s", packstage_dir, pack->module,
+		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s", packstage_dir,
-				pack->from, file->peer->last_change, file->last_change, file->hash);
+			      pack->module, pack->from, pack->to, file->peer->last_change,
+			      file->last_change, file->hash);
 		string_or_die(&tarfrom, "%s/%i/files/%s.tar", staging_dir,
-				file->last_change, file->hash);
+			      file->last_change, file->hash);
-		string_or_die(&tarto, "%s/%s/%i/staged/%s.tar", packstage_dir, pack->module,
+		string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
-				pack->from, file->hash);
+			      pack->module, pack->from, pack->to, file->hash);
+		string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+		string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+			      pack->module, pack->from, pack->to, file->hash);
 
 		ret = stat(from, &stat_delta);
 		if (ret) {
@@ -367,13 +396,28 @@ static int make_final_pack(struct packdata *pack)
 				}
 			}
 		} else {
-			/* include full file in pack */
+			ret = -1;
-			ret = link(tarfrom, tarto);
+			errno = 0;
-			if (ret) {
+
-				if (errno != EEXIST) {
+			/* Prefer to hardlink uncompressed files (excluding
-					LOG(NULL, "Failure to link", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno);
+			 * directories) first, and fall back to the compressed
+			 * versions if the hardlink fails.
+			 */
+			if (!file->is_dir) {
+				ret = link(fullfrom, fullto);
+				if (ret && errno != EEXIST) {
+					LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno);
 				}
-			} else {
+			}
+
+			if (ret) {
+				ret = link(tarfrom, tarto);
+				if (ret && errno != EEXIST) {
+					LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno);
+				}
+			}
+
+			if (ret == 0) {
 				pack->fullcount++;
 			}
 		}
@@ -382,20 +426,24 @@ static int make_final_pack(struct packdata *pack)
 		free(to);
 		free(tarfrom);
 		free(tarto);
+		free(fullfrom);
+		free(fullto);
 	}
 
 	if (pack->fullcount > 0) {
 		/* untar all files for smaller size and less cpu use on client */
-		explode_pack_stage(pack->from, pack->module);
+		explode_pack_stage(pack->from, pack->to, pack->module);
 	}
 
+	char *bundle_delta = NULL;
+
 	/* now... link in the Manifest pack */
 	if (pack->from != 0) {
 		char *from, *to;
 		struct stat st;
 
 		string_or_die(&from, "%s/%i/Manifest-%s-delta-from-%i",
-				staging_dir, pack->to, pack->module, pack->from);
+			      staging_dir, pack->to, pack->module, pack->from);
 
 		ret = stat(from, &st);
 		if (ret) {
@@ -403,56 +451,60 @@ static int make_final_pack(struct packdata *pack)
 			create_manifest_delta(pack->from, pack->to, pack->module);
 		}
 
-
+		string_or_die(&to, "%s/%s/%i_to_%i/Manifest-%s-delta-from-%i", packstage_dir,
-		string_or_die(&to, "%s/%s/%i/Manifest-%s-delta-from-%i",
+			      pack->module, pack->from, pack->to, pack->module, pack->from);
-				packstage_dir, pack->module, pack->from, pack->module, pack->from);
 
 		ret = link(from, to);
 		if (ret) {
-			LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from);
+			LOG(NULL, "Failed to link", "Manifest-%s-delta-from-%i (%s)", pack->module, pack->from, strerror(errno));
+		} else {
+			string_or_die(&bundle_delta, "Manifest-%s-delta-from-%i", pack->module, pack->from);
 		}
 
 		free(from);
 		free(to);
 	}
 
+	char *mom_delta = NULL;
 
-	/* now... link in the MoM Manifest into the base pack */
+	/* now... link in the MoM Manifest into the os-core pack */
 	if ((pack->from != 0) && (strcmp(pack->module, "os-core") == 0)) {
 		char *from, *to;
 		struct stat st;
 
 		string_or_die(&from, "%s/%i/Manifest-%s-delta-from-%i",
-				staging_dir, pack->to, "MoM", pack->from);
+			      staging_dir, pack->to, "MoM", pack->from);
 		ret = stat(from, &st);
 		if (ret) {
 			LOG(NULL, "Making extra manifest delta", "MoM: %i->%i", pack->from, pack->to);
 			create_manifest_delta(pack->from, pack->to, "MoM");
 		}
 
-		string_or_die(&to, "%s/%s/%i/Manifest-%s-delta-from-%i",
+		string_or_die(&to, "%s/%s/%i_to_%i/Manifest-%s-delta-from-%i", packstage_dir,
-				packstage_dir, pack->module, pack->from, "MoM", pack->from);
+			      pack->module, pack->from, pack->to, "MoM", pack->from);
 
 		ret = link(from, to);
 		if (ret) {
-			LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from);
+			LOG(NULL, "Failed to link", "Manifest-MoM-delta-from-%i (%s)", pack->from, strerror(errno));
+		} else {
+			string_or_die(&mom_delta, "Manifest-MoM-delta-from-%i", pack->from);
 		}
 
 		free(from);
 		free(to);
 	}
 
-
 	/* tar the staging directory up */
 	LOG(NULL, "starting tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to);
-	string_or_die(&tar, "tar " TAR_PERM_ATTR_ARGS " --directory=%s/%s/%i/ "
+	string_or_die(&param1, "%s/%s/%i_to_%i/", packstage_dir, pack->module, pack->from, pack->to);
-			"--numeric-owner -Jcf %s/%i/pack-%s-from-%i.tar delta staged",
+	string_or_die(&param2, "%s/%i/pack-%s-from-%i.tar", staging_dir, pack->to, pack->module, pack->from);
-			packstage_dir, pack->module, pack->from, staging_dir, pack->to, pack->module, pack->from);
+	char *const tarcmd[] = { TAR_COMMAND, "-C", param1, TAR_PERM_ATTR_ARGS_STRLIST, "--numeric-owner", "-Jcf", param2, "delta", "staged", bundle_delta, mom_delta, NULL };
-	ret = system(tar);
+	ret = system_argv(tarcmd);
-	free(tar);
+	free(param1);
+	free(param2);
 	LOG(NULL, "finished tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to);
 	/* FIXME: this is a hack workaround, needs diagnosed and removed */
-	if ((ret != 0) && (ret != 256)) {
+	if ((ret != 0) && (ret != 1)) {
 		fprintf(stderr, "Unexpected return value (%d) creating tar of pack %s from %i to %i\n",
 			ret, pack->module, pack->from, pack->to);
 	} else {
@@ -466,13 +518,12 @@ static int make_final_pack(struct packdata *pack)
 
 	/* and clean up */
 	free_manifest(pack->end_manifest);
-	empty_pack_stage(1, pack->from, pack->module);
+	empty_pack_stage(1, pack->from, pack->to, pack->module);
 
 	LOG(NULL, "pack complete", "%s: %i to %i", pack->module, pack->from, pack->to);
 	return ret;
 }
 
-
 /* Returns 0 == success, -1 == failure */
 int make_pack(struct packdata *pack)
 {

src/rename.c

@@ -22,32 +22,29 @@
  */
 
 #define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
 #include <assert.h>
 #include <ctype.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <glib.h>
 #include <libgen.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
-
 #include <magic.h>
 
 static magic_t mcookie;
 
-
 double rename_score(struct file *old, struct file *new)
 {
 	double score = 0.0;
 	char *c;
 	int io, in;
 
-
 	/* if we have the same hash... sounds like a winner */
 	if (hash_compare(old->hash, new->hash)) {
 		score += 400;
@@ -88,14 +85,14 @@ double rename_score(struct file *old, struct file *new)
 		io--;
 		in--;
 	}
-	
+
 	/* if both start with /boot/vmlinuz give it a boost; this is a local hack due to vmlinuz being very short */
 	if (strncmp(old->filename, "/boot/vmlinuz", 13) == 0 && strncmp(new->filename, "/boot/vmlinuz", 13) == 0) {
 		score += 80;
 	}
 
 	/* if ELF, points for sharing the same soname to the first dot */
-	
+
 	/* negative points for not being within 25%+/-1Kb of the same file size */
 	if (old->stat.st_size > ((new->stat.st_size * 1.25) + 1024)) {
 		score -= 30;
@@ -152,7 +149,7 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i
 		c2++;
 	}
 
-	if (manifest)  {
+	if (manifest) {
 		string_or_die(&filename, "%s/%i/%s/%s", image_dir, manifest->version, manifest->component, file->filename);
 	} else if (old_rename) {
 		item = g_list_first(last_versions_list);
@@ -186,7 +183,7 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i
 		}
 	}
 
-	c1 = (char *) magic_file(mcookie, filename);
+	c1 = (char *)magic_file(mcookie, filename);
 	if (c1) {
 		char *c2;
 		file->filetype = strdup(c1);
@@ -205,17 +202,15 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i
 	free(filename);
 
 	file->basename = strdup(basename(file->filename));
-	file->dirname  = strdup(dirname(file->filename));
+	file->dirname = strdup(dirname(file->filename));
-
 }
 
-
 int file_sort_score(gconstpointer a, gconstpointer b)
 {
 	struct file *A, *B;
 
-	A = (struct file *) a;
+	A = (struct file *)a;
-	B = (struct file *) b;
+	B = (struct file *)b;
 
 	if (A->rename_score > B->rename_score) {
 		return -1;
@@ -240,7 +235,6 @@ static void score_file(GList *deleted_files, struct file *file)
 		file2 = list2->data;
 		list2 = g_list_next(list2);
 
-
 		assert(file2);
 		assert(file2->peer);
 
@@ -249,7 +243,6 @@ static void score_file(GList *deleted_files, struct file *file)
 			file->rename_score = score;
 			file->rename_peer = file2;
 		}
-
 	}
 }
 

src/signature.c

@@ -21,6 +21,7 @@
  */
 
 #define _GNU_SOURCE
+#include <err.h>
 #include <errno.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -29,16 +30,11 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <err.h>
 
 #include "swupd.h"
 
 static char *make_filename(const char *, const char *, const char *);
 
-static const char *CMD_FMT = "openssl smime -sign -in %s -binary "
-	"-out %s.signed -outform PEM -md sha256 -inkey %s -signer %s "
-	"-certfile %s -passin file:%s";
-
 static char *leaf_key = NULL;
 static char *leaf_cert = NULL;
 static char *ca_chain_cert = NULL;
@@ -61,7 +57,7 @@ bool signature_initialize(void)
 	struct stat s;
 
 	if (initialized) {
-	    return true;
+		return true;
 	}
 	cdir = getenv("SWUPD_CERTS_DIR");
 	if (cdir == NULL || cdir[0] == '\0') {
@@ -70,7 +66,7 @@ bool signature_initialize(void)
 	}
 	if (stat(cdir, &s)) {
 		printf("Can't stat certificates directory '%s' (%s)\n", cdir,
-			strerror(errno));
+		       strerror(errno));
 		goto err;
 	}
 	leaf_key = make_filename(cdir, "LEAF_KEY", "leaf key");
@@ -91,7 +87,7 @@ bool signature_initialize(void)
 	}
 	if (stat(passphrase, &s)) {
 		printf("Can't stat '%s' (%s)\n", passphrase,
-			strerror(errno));
+		       strerror(errno));
 		goto err;
 	}
 	initialized = true;
@@ -150,7 +146,7 @@ void signature_terminate(void)
  */
 bool signature_sign(const char *filename)
 {
-	char *cmd = NULL;
+	char *param1, *param2;
 	int status;
 
 	if (!enable_signing) {
@@ -160,12 +156,18 @@ bool signature_sign(const char *filename)
 	if (!initialized) {
 		return false;
 	}
-	string_or_die(&cmd, CMD_FMT, filename, filename, leaf_key, leaf_cert,
+	string_or_die(&param1, "%s.signed", filename);
-			ca_chain_cert, passphrase);
+	string_or_die(&param2, "file:%s", passphrase);
-	status = system(cmd);
+	char *const opensslcmd[] = { "openssl", "smime", "-sign", "-in", (char *)filename, "-binary",
-	if (status) {
+				     "-out", param1, "-outform", " PEM", "-md", "sha256", "-inkey",
-		printf("Bad status %d from signing command:%s\n", status, cmd);
+				     leaf_key, "-signer", leaf_cert, "-certfile", ca_chain_cert,
+				     "-passin", param2, NULL };
+	status = system_argv(opensslcmd);
+	if (status != 0) {
+		printf("Bad status %d from signing command\n", status);
 	}
-	free(cmd);
+	free(param1);
+	free(param2);
+
 	return status == 0;
 }

src/stats.c

@@ -22,26 +22,21 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
+#include <unistd.h>
-
-#include <glib.h>
 
 #include "swupd.h"
 
-
-
-
 static int new_files;
 static int deleted_files;
 static int changed_files;
 static int delta_miss;
 static int delta_hit;
 
-
 void account_new_file(void)
 {
 	new_files++;
@@ -62,7 +57,6 @@ void account_delta_hit(void)
 	delta_hit++;
 }
 
-
 void account_delta_miss(void)
 {
 	delta_miss++;
@@ -84,8 +78,6 @@ int have_new_files(void)
 	return 0;
 }
 
-
-
 void print_statistics(int version1, int version2)
 {
 	printf("\n");

src/type_change.c

@@ -76,7 +76,7 @@ static bool type_has_changed(struct file *file)
 
 	LOG(file, "Type change", "%c to %c", type1, type2);
 	fprintf(stderr, "Type change from %c to %c for %s\n", type1, type2,
-			file->filename);
+		file->filename);
 
 	if (((type1 == 'F') && (type2 == 'L')) ||
 	    ((type1 == 'F') && (type2 == 'D')) ||
@@ -106,7 +106,7 @@ void type_change_detection(struct manifest *manifest)
 
 	if (n != 0) {
 		printf("Detected %d file type changes in component %s.. exiting\n",
-			n, manifest->component);
+		       n, manifest->component);
 		exit(EXIT_FAILURE);
 	}
 }

src/versions.c

@@ -23,23 +23,22 @@
 
 #define _GNU_SOURCE
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
 #include <assert.h>
-#include <sys/stat.h>
 #include <ctype.h>
 #include <dirent.h>
-
+#include <errno.h>
+#include <fcntl.h>
 #include <glib.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
 int current_version;
 
-
 void read_current_version(char *filename)
 {
 	FILE *file;
@@ -47,8 +46,7 @@ void read_current_version(char *filename)
 	char *fullfile = NULL;
 	char *conf;
 
-	conf =  config_image_base();
+	conf = config_image_base();
-
 
 	string_or_die(&fullfile, "%s/%s", conf, filename);
 
@@ -79,8 +77,7 @@ void write_new_version(char *filename, int version)
 	char *fullfile = NULL;
 	char *conf;
 
-	conf =  config_image_base();
+	conf = config_image_base();
-
 
 	string_or_die(&fullfile, "%s/%s", conf, filename);
 
@@ -130,35 +127,63 @@ void ensure_version_image_exists(int version)
 void write_cookiecrumbs_to_download_area(int version)
 {
 	char *conf;
-	char *cmd;
+	char *filename, *strformat;
+	FILE *versionfd, *formatfd;
 
-	conf =  config_output_dir();
+	conf = config_output_dir();
 	if (conf == NULL) {
 		assert(0);
 	}
 
-	string_or_die(&cmd, "echo %s > %s/%i/swupd-server-src-version", VERSION, conf, version);
+	string_or_die(&filename, "%s/%i/swupd-server-src-version", conf, version);
-	if (system(cmd) != 0) {
+	versionfd = fopen(filename, "w");
+	if (versionfd == NULL) {
 		assert(0);
 	}
-	free(cmd);
+	if (fwrite(VERSION, 1, strlen(VERSION), versionfd) != strlen(VERSION)) {
-
-	string_or_die(&cmd, "echo %s > %s/%i/format", format_string, conf, version);
-	if (system(cmd) != 0) {
 		assert(0);
 	}
-	free(cmd);
+	fclose(versionfd);
+	free(filename);
 
-/* written from pack_maker.sh */
+	string_or_die(&filename, "%s/%i/format", conf, version);
+	string_or_die(&strformat, "%llu", format);
+	formatfd = fopen(filename, "w");
+	if (formatfd == NULL) {
+		assert(0);
+	}
+	if (fwrite(strformat, 1, strlen(strformat), formatfd) != strlen(strformat)) {
+		assert(0);
+	}
+	fclose(formatfd);
+	free(filename);
+	free(strformat);
+
+/* Updating the WEBDIR/version/formatN/latest file is an operation very closely tied to a DevOps
+ * workflow and not something swupd should make a decision about.
+ *
+ * The supported format values (replacing the "N" in the path above) are:
+ *
+ * - "staging": intended for testing only
+ * - [1-+inf] (positive integer): the number that a corresponding version of swupd-client
+ *   understands to run software updates. For example, if a given swupd-client release understands
+ *   format 1, then WEBDIR/version/format1/latest contains the value of most recent release that
+ *   that swupd-client can update to.
+ *
+ * Adapt this step appropriately for your DevOps flow.
+ */
 #if 0
 	FILE *file;
 	char *fullfile = NULL;
+	char *param;
 
-	string_or_die(&cmd, "mkdir -p %s/version/formatstaging/", conf);
+	string_or_die($param, "%s/version/formatstaging/", conf);
-	if (system(cmd) != 0) {
+	char *const mkdircmd[] = { "mkdir", "-p", param, NULL };
+
+	if (system_argv(mkdircmd) != 0) {
 		assert(0);
 	}
-	free(cmd);
+	free(param);
 
 	string_or_die(&fullfile, "%s/version/formatstaging/latest", conf);
 	file = fopen(fullfile, "w");
@@ -179,7 +204,8 @@ static int compare_versions(gconstpointer a, gconstpointer b)
 	return GPOINTER_TO_INT(b) - GPOINTER_TO_INT(a);
 }
 
-static int get_build_type(int build_num) {
+static int get_build_type(int build_num)
+{
 	int build_type = build_num % 10;
 	if (build_type == 0) {
 		return REGULAR_BUILD;
@@ -190,7 +216,8 @@ static int get_build_type(int build_num) {
 	}
 }
 
-static int get_jump_point(int build_num) {
+static int get_jump_point(int build_num)
+{
 	int jump_point = (build_num / 100) * 100;
 	if (jump_point == build_num) {
 		jump_point -= 100;
@@ -218,7 +245,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 	dir = opendir(staging_dir);
 	if (dir == NULL) {
 		LOG(NULL, "Cannot open directory", "dir_path= %s, strerror= %s",
-				staging_dir, strerror(errno));
+		    staging_dir, strerror(errno));
 		return NULL;
 	}
 
@@ -232,7 +259,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 
 		if (lstat(filename, &stat)) {
 			LOG(NULL, "lstat failed", "path= %s, strerror= %s",
-					filename, strerror(errno));
+			    filename, strerror(errno));
 			continue;
 		}
 
@@ -272,7 +299,7 @@ GList *get_last_versions_list(int next_version, int max_versions)
 		build_type = get_build_type(build_num);
 		next_item = g_list_next(cur_item);
 		if ((idx >= max_versions || build_type == DEV_BUILD) &&
-				build_num != jump_point && build_num != 0) {
+		    build_num != jump_point && build_num != 0) {
 			list = g_list_delete_link(list, cur_item);
 		} else if (idx == max_versions - 1 && build_num != 0) {
 			list = g_list_delete_link(list, cur_item);

src/xattrs.c

@@ -20,15 +20,15 @@
  *
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
-#include <assert.h>
-#include <sys/xattr.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/xattr.h>
 
 #include "swupd.h"
 #include "xattrs.h"
@@ -49,8 +49,8 @@ static int xattr_get_value(const char *path, const char *name, char **blob,
 	len = lgetxattr(path, name, NULL, 0);
 	if (len < 0) {
 		LOG(NULL, "Failed to get x-attribute length",
-				"%s for file %s: %s",
+		    "%s for file %s: %s",
-				name, path, strerror(errno));
+		    name, path, strerror(errno));
 		return -1;
 	}
 
@@ -61,7 +61,7 @@ static int xattr_get_value(const char *path, const char *name, char **blob,
 	/* realloc needed len + 1 in case we need to add final zero
 	 * to ensure consistent blob */
 	value = realloc(*blob, *blob_len + len +
-			(action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
+				   (action == XATTRS_ACTION_GET_BLOB ? 1 : 0));
 	assert(value);
 
 	*blob = value;
@@ -70,8 +70,8 @@ static int xattr_get_value(const char *path, const char *name, char **blob,
 	len = lgetxattr(path, name, value, len);
 	if (len < 0) {
 		LOG(NULL, "Failed to get x-attribute",
-				"%s for file %s: %s",
+		    "%s for file %s: %s",
-				name, path, strerror(errno));
+		    name, path, strerror(errno));
 		return -1;
 	}
 
@@ -107,7 +107,7 @@ static int get_xattr_name_count(const char *names_list, ssize_t len)
 
 static int cmp_xattr_name_ptrs(const void *ptr1, const void *ptr2)
 {
-	return strcmp(*(char * const *)ptr1, *(char * const *)ptr2);
+	return strcmp(*(char *const *)ptr1, *(char *const *)ptr2);
 }
 
 static const char **get_sorted_xattr_name_table(const char *names, int n)
@@ -115,7 +115,7 @@ static const char **get_sorted_xattr_name_table(const char *names, int n)
 	const char **table;
 	int i;
 
-	table = calloc(1, n * sizeof(char*));
+	table = calloc(1, n * sizeof(char *));
 	assert(table);
 
 	for (i = 0; i < n; i++) {
@@ -123,7 +123,7 @@ static const char **get_sorted_xattr_name_table(const char *names, int n)
 		names += strlen(names) + 1;
 	}
 
-	qsort(table, n, sizeof(char*), cmp_xattr_name_ptrs);
+	qsort(table, n, sizeof(char *), cmp_xattr_name_ptrs);
 
 	return table;
 }
@@ -131,7 +131,8 @@ static const char **get_sorted_xattr_name_table(const char *names, int n)
 static void xattrs_do_action(xattrs_action_type_t action,
 			     const char *src_filename,
 			     const char *dst_filename,
-			     char **blob, size_t *blob_len) {
+			     char **blob, size_t *blob_len)
+{
 	ssize_t len;
 	char *list;
 	int ret = 0;
@@ -201,8 +202,8 @@ static void xattrs_do_action(xattrs_action_type_t action,
 			free(value);
 			if (ret < 0) {
 				LOG(NULL, "Failed to set x-attributes",
-						"%s: %s",
+				    "%s: %s",
-						dst_filename, strerror(errno));
+				    dst_filename, strerror(errno));
 				break;
 			}
 			value_len = 0;

test/functional/basic/test.bats

@@ -0,0 +1,26 @@
+#!/usr/bin/env bats
+
+load "../swupdlib"
+
+@test "create_update help output" {
+  run $CREATE_UPDATE --help
+  [ "$status" -eq 1 ]
+  run $CREATE_UPDATE -h
+  [ "$status" -eq 1 ]
+}
+
+@test "make_fullfiles help output" {
+  run $MAKE_FULLFILES --help
+  [ "$status" -eq 1 ]
+  run $MAKE_FULLFILES -h
+  [ "$status" -eq 1 ]
+}
+
+@test "make_pack help output" {
+  run $MAKE_PACK --help
+  [ "$status" -eq 1 ]
+  run $MAKE_PACK -h
+  [ "$status" -eq 1 ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/delete-no-version-bump/test.bats

@@ -0,0 +1,48 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle1
+  set_os_release 10 test-bundle2
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle1
+  set_os_release 20 test-bundle2
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+
+  gen_file_plain 10 test-bundle1 foo
+  gen_file_plain 10 test-bundle2 foo
+
+  gen_file_plain 20 test-bundle1 foo
+}
+
+@test "delete no version bump update creation" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  [ 1 -eq $(grep '10	/foo' $DIR/www/10/Manifest.full | wc -l) ]
+  [ 1 -eq $(grep '10	/foo' $DIR/www/20/Manifest.full | wc -l) ]
+  [ 0 -eq $(grep '20	/foo' $DIR/www/20/Manifest.full | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/file-name-blacklisted/test.bats

@@ -0,0 +1,40 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  gen_file_plain 10 os-core "semicolon;"
+}
+
+assert_illegal_char() {
+  local output="$1"
+  local str=$(echo "Filename $2 includes illegal character(s)")
+  [[ "$output" =~ "$str" ]]
+}
+
+assert_valid_char() {
+  local output="$1"
+  local str=$(echo "Filename $2 includes illegal character(s)")
+  [[ ! "$output" =~ "$str" ]]
+}
+
+@test "file names with blacklisted characters" {
+  run sudo sh -c "$CREATE_UPDATE --osversion 10 --statedir $DIR --format 3"
+
+  echo "$output"
+  assert_illegal_char "$output" "/semicolon;"
+  assert_valid_char "$output" "/usr"
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run-delta/test.bats

@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle included included-two included-nested
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  track_bundle 20 included
+  track_bundle 20 included-two
+  track_bundle 20 included-nested
+
+  gen_file_to_delta 10 4096 20 4 test-bundle
+
+  gen_file_plain 10 test-bundle foo
+  gen_file_plain 10 test-bundle foobarbaz
+  gen_file_plain 20 test-bundle foo
+  gen_file_plain 20 included foo
+  gen_file_plain 20 included-two foo
+  gen_file_plain 20 included-two foobar
+  gen_file_plain 20 included-nested foobarbaz
+
+  gen_includes_file test-bundle 20 included included-two
+  gen_includes_file included 20 included-nested
+}
+
+@test "full run update creation with delta packs" {
+  # build the first version
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  # then the second version...
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+
+  # and with delta packs this time
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+  # zero packs should exist (non-zero size) for both versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-0.tar ]
+
+  # and delta packs should exist (non-zero size) for the latest version
+  [ -s $DIR/www/20/pack-os-core-from-10.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-10.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/foobar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-two$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^.d..	0000000000000000000000000000000000000000000000000000000000000000	20	/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^.d..	0000000000000000000000000000000000000000000000000000000000000000	20	/foobarbaz$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/20/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '/foobar$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/20/Manifest.included-nested | wc -l) ]]
+  [[ 6 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
+  [[ 4 -eq $(tar -tf $DIR/www/20/pack-test-bundle-from-0.tar | wc -l) ]]
+
+  # delta packs should contain appropriate delta manifests
+  [[ $(tar -tf $DIR/www/20/pack-os-core-from-10.tar | grep '^Manifest-os-core-delta-from-10') ]]
+  [[ $(tar -tf $DIR/www/20/pack-os-core-from-10.tar | grep '^Manifest-MoM-delta-from-10') ]]
+  [[ $(tar -tf $DIR/www/20/pack-test-bundle-from-10.tar | grep '^Manifest-test-bundle-delta-from-10') ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run/test.bats

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  gen_file_plain 10 test-bundle foo
+}
+
+@test "full run update creation" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  # zero packs should exist (non-zero size) for the build
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles/test-bundle$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles/os-core$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
+  [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/fullfiles/test.bats

@@ -0,0 +1,22 @@
+#!/usr/bin/env bats
+
+load "../swupdlib"
+
+@test "make_fullfiles required arg" {
+  run $MAKE_FULLFILES
+  [ "$status" -eq 1 ]
+}
+
+@test "make_fullfiles too many arguments" {
+  # exactly one argument must be passed
+  run $MAKE_FULLFILES foo bar
+  [ "$status" -eq 1 ]
+}
+
+@test "make_fullfiles root priv check" {
+  run $MAKE_FULLFILES foo
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "not being run as root.. exiting" ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/include-version-bump/test.bats

@@ -0,0 +1,99 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle included included-two included-nested
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  track_bundle 20 included
+  track_bundle 20 included-two
+
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+  track_bundle 30 included
+  track_bundle 30 included-two
+  track_bundle 30 included-nested
+
+  gen_file_plain 10 test-bundle foo
+  gen_file_plain 20 test-bundle foo
+  gen_file_plain 20 included bar
+  gen_file_plain 20 included-two baz
+  gen_file_plain 30 test-bundle foo
+  gen_file_plain 30 included bar
+  gen_file_plain 30 included-two baz
+  gen_file_plain 30 included-nested foobarbaz
+
+  gen_includes_file test-bundle 20 included included-two
+  gen_includes_file test-bundle 30 included included-two
+  gen_includes_file included 30 included-nested
+}
+
+@test "full run update creation with delta packs" {
+  # build the first version
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  # then the second version...
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 0 20 included
+  sudo $MAKE_PACK --statedir $DIR 0 20 included-two
+
+  set_latest_ver 20
+
+  # then the third version...
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  sudo $MAKE_PACK --statedir $DIR 0 30 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 30 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 0 30 included
+  sudo $MAKE_PACK --statedir $DIR 0 30 included-two
+  sudo $MAKE_PACK --statedir $DIR 0 30 included-nested
+
+  # zero packs should exist (non-zero size) for both versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-included-from-0.tar ]
+  [ -s $DIR/www/20/pack-included-two-from-0.tar ]
+  [ -s $DIR/www/30/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/30/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-two-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-nested-from-0.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-two$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/bar$' $DIR/www/20/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/baz$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-nested$' $DIR/www/30/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/30/Manifest.included-nested | wc -l) ]]
+  [[ 0 -eq $(ls $DIR/www/30/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(ls $DIR/www/30/Manifest.included-two | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/pack/test.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+load "../swupdlib"
+
+@test "make_pack required arg" {
+  run $MAKE_PACK
+  [ "$status" -eq 1 ]
+}
+
+@test "make_pack too few arguments" {
+  run $MAKE_PACK foo
+  [ "$status" -eq 1 ]
+  run $MAKE_PACK foo bar
+  [ "$status" -eq 1 ]
+}
+
+@test "make_pack too many arguments" {
+  run $MAKE_PACK foo bar foo bar
+  [ "$status" -eq 1 ]
+}
+
+@test "make_pack root priv check" {
+  run $MAKE_PACK foo bar foo
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "not being run as root.. exiting" ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/swupdlib.bash

@@ -0,0 +1,107 @@
+# NOTE: source this file from a *.bats file
+
+# The location of the swupd_* binaries
+export SRCDIR="$BATS_TEST_DIRNAME/../../../"
+
+export CREATE_UPDATE="$SRCDIR/swupd_create_update"
+export MAKE_FULLFILES="$SRCDIR/swupd_make_fullfiles"
+export MAKE_PACK="$SRCDIR/swupd_make_pack"
+
+export DIR="$BATS_TEST_DIRNAME/web-dir"
+
+init_test_dir() {
+  local testdir="$BATS_TEST_DIRNAME"
+  mkdir -p "$testdir"/logs
+  mkdir -p $DIR/{image,www}
+  # run swupd_* inside the directory to dump the logs
+  cd "$testdir"/logs
+}
+
+clean_test_dir() {
+  sudo rm -rf $DIR "$BATS_TEST_DIRNAME"/logs
+}
+
+init_server_ini() {
+  cp $SRCDIR/server.ini $DIR
+  sed -i "s|/var/lib/update|$DIR|" $DIR/server.ini
+}
+
+set_latest_ver() {
+  echo "$1" > $DIR/image/latest.version
+}
+
+init_groups_ini() {
+  for bundle in "$@"; do
+    cat >> $DIR/groups.ini << EOF
+[$bundle]
+group=$bundle
+EOF
+  done
+}
+
+set_os_release() {
+  local ver=$1
+  local bundle=$2
+  mkdir -p $DIR/image/$ver/$bundle/usr/lib/
+  echo "VERSION_ID=$ver" > $DIR/image/$ver/$bundle/usr/lib/os-release
+}
+
+track_bundle() {
+  local ver=$1
+  local bundle=$2
+  mkdir -p $DIR/image/$ver/$bundle/usr/share/clear/bundles
+  touch $DIR/image/$ver/$bundle/usr/share/clear/bundles/$bundle
+}
+
+gen_includes_file() {
+  local bundle=$1
+  local ver=$2
+  local includes="${@:3}"
+  mkdir -p $DIR/www/$ver/noship
+  for b in $includes; do
+    cat >> $DIR/www/$ver/noship/"$bundle"-includes << EOF
+$b
+EOF
+  done
+}
+
+gen_file_to_delta() {
+  local origver=$1
+  local origsize=$2
+  local newver=$3
+  local newbytes=$4
+  local bundle=$5
+
+  # create some random data for the original version
+  mkdir -p $DIR/image/$origver/$bundle
+  dd if=/dev/urandom of=$DIR/image/$origver/$bundle/randomfile bs=1 count=$origsize
+
+  # append more random data to the end of the file in the new version
+  TMP=$(mktemp foo.XXXXXX)
+  mkdir -p $DIR/image/$newver/$bundle
+  dd if=/dev/urandom of=$TMP bs=1 count=$newbytes
+  cat $DIR/image/$origver/$bundle/randomfile $TMP > $DIR/image/$newver/$bundle/randomfile
+  rm $TMP
+}
+
+gen_file_plain() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+
+  # Add plain text file into a bundle
+  mkdir -p $DIR/image/$ver/$bundle
+  echo "$name" > $DIR/image/$ver/$bundle/"$name"
+}
+
+gen_file_plain_change() {
+  local ver=$1
+  local bundle=$2
+  local name="$3"
+
+  # Add plain text file into a bundle
+  mkdir -p $DIR/image/$ver/$bundle
+  echo "$ver $name" > $DIR/image/$ver/$bundle/"$name"
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/update/test.bats

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+load "../swupdlib"
+
+@test "create_update required format" {
+  run $CREATE_UPDATE
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "Missing format parameter" ]]
+}
+
+@test "create_update required version" {
+  run $CREATE_UPDATE -F 3
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "Missing version parameter:" ]]
+}
+
+@test "create_update root priv check" {
+  run $CREATE_UPDATE -F 3 -o 10
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "not being run as root.. exiting" ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80