Release v3.2.2

This release includes a trivial improvement for reporting paths that
contain blacklisted characters.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

.gitignore

@@ -19,16 +19,17 @@ stamp-h1
 DEADJOE
 swupd_create_update
 swupd_make_pack
-swupd-update-server.log*
 swupd_make_fullfiles
 swupd-server-*.tar.gz
-swupd-server.log.*
 cscope.*
 *.kdev4
 coverage/
 src/*.gcno
 src/*.gcda
 tap-driver.sh
-test/functional/*.log
-test/functional/*.trs
+test/functional/*/*.log
+test/functional/*/*.trs
+test/functional/*/logs/
+test/functional/*/web-dir/
+swupd-*.log
 test-suite.log

Makefile.am

@@ -15,6 +15,7 @@ swupd_create_update_SOURCES = \
 	src/analyze_fs.c \
 	src/chroot.c \
 	src/config.c \
+	src/create_update.c \
 	src/delta.c \
 	src/fullfiles.c \
 	src/globals.c \
@@ -22,7 +23,6 @@ swupd_create_update_SOURCES = \
 	src/helpers.c \
 	src/heuristics.c \
 	src/log.c \
-	src/main.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
@@ -96,24 +96,26 @@ noinst_HEADERS = \
 	include/swupd.h \
 	include/xattrs.h
 
-TEST_EXTENSIONS = .sh
+TEST_EXTENSIONS = .bats
 
 if ENABLE_TESTS
 tap_driver = env AM_TAP_AWK='$(AWK)' $(SHELL) \
 	$(top_srcdir)/tap-driver.sh
 
 LOG_DRIVER = $(tap_driver)
-SH_LOG_DRIVER = $(tap_driver)
+BATS_LOG_DRIVER = $(tap_driver)
 
 TESTS = $(dist_check_SCRIPTS)
 
 dist_check_SCRIPTS = \
-	test/functional/basic.bats \
-	test/functional/update.bats \
-	test/functional/fullfiles.bats \
-	test/functional/pack.bats \
-	test/functional/full-run.bats \
-	test/functional/full-run-delta.bats
+	test/functional/basic/test.bats \
+	test/functional/delete-no-version-bump/test.bats \
+	test/functional/include-version-bump/test.bats \
+	test/functional/update/test.bats \
+	test/functional/fullfiles/test.bats \
+	test/functional/pack/test.bats \
+	test/functional/full-run/test.bats \
+	test/functional/full-run-delta/test.bats
 endif
 
 if COVERAGE

basic_creator.sh

@@ -23,11 +23,9 @@ export LEAF_CERT="leaf.cert.pem"
 export CA_CHAIN_CERT="ca-chain.cert.pem"
 export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase"
 
-${SWUPDREPO}/swupd_create_update --osversion ${VER} --statedir ${UPDATEDIR}
+${SWUPDREPO}/swupd_create_update --osversion ${VER} --statedir ${UPDATEDIR} --format 3
 ${SWUPDREPO}/swupd_make_fullfiles --statedir ${UPDATEDIR} ${VER}
 
-pushd ${SWUPDREPO}
-
 # create zero packs
 MOM=${UPDATEDIR}/www/${VER}/Manifest.MoM
 if [ ! -e ${MOM} ]; then
@@ -51,8 +49,6 @@ done
 NUM_PACKS=2
 ${SWUPDREPO}/pack_maker.sh ${VER} ${NUM_PACKS}
 
-popd
-
 # expose the new build to staging / testing
 echo ${VER} > ${UPDATEDIR}/image/latest.version
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.1.2, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.2.2, timothy.c.pepper@linux.intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -13,7 +13,6 @@
 // SWUPD_NUM_PACKS is also "PREV_CHECK" in releas tool swupd_bb.py (change both)
 #define SWUPD_NUM_PACKS 4
 #define SWUPD_NUM_MANIFEST_DELTAS 25
-#define SWUPD_DEFAULT_FORMAT 3
 
 #define SWUPD_SERVER_STATE_DIR "/var/lib/update"
 
@@ -61,6 +60,7 @@
 #endif
 
 struct manifest {
+	unsigned long long int format;
 	int version;
 	int prevversion;
 	char *component;
@@ -70,6 +70,8 @@ struct manifest {
 	GList *manifests; /* as struct file */
 
 	GList *submanifests; /* as struct manifest */
+
+	GList *includes; /* struct manifests for all bundles included into this one */
 };
 
 struct file;
@@ -134,7 +136,7 @@ struct packdata {
 extern int current_version;
 extern int newversion;
 extern int minversion;
-extern char *format_string;
+extern unsigned long long int format;
 extern bool enable_signing;
 
 extern char *state_dir;
@@ -144,7 +146,7 @@ extern char *staging_dir;
 
 extern bool init_globals(void);
 extern void free_globals(void);
-extern bool set_format_string(char *);
+extern bool set_format(char *);
 extern void check_root(void);
 extern bool set_state_dir(char *);
 extern bool init_state_globals(void);
@@ -173,10 +175,13 @@ extern void free_manifest(struct manifest *manifest);
 extern struct manifest *alloc_manifest(int version, char *module);
 extern int match_manifests(struct manifest *m1, struct manifest *m2);
 extern void sort_manifest_by_version(struct manifest *manifest);
+extern bool manifest_includes(struct manifest *manifest, char *component);
+extern bool changed_includes(struct manifest *old, struct manifest *new);
 extern int prune_manifest(struct manifest *manifest);
+extern int remove_old_deleted_files(struct manifest *m1, struct manifest *m2);
 extern void create_manifest_delta(int oldversion, int newversion, char *module);
 extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list);
-extern void subtract_manifests(struct manifest *m1, struct manifest *m2);
+extern void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2);
 extern void nest_manifest(struct manifest *parent, struct manifest *sub);
 extern void nest_manifest_file(struct manifest *parent, struct file *file);
 extern int manifest_subversion(struct manifest *parent, char *group);

src/analyze_fs.c

@@ -23,19 +23,19 @@
  */
 
 #define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
 #include <assert.h>
+#include <dirent.h>
 #include <errno.h>
 #include <glib.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-#include <openssl/hmac.h>
 #include <linux/limits.h>
+#include <openssl/hmac.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 #include "xattrs.h"
@@ -286,18 +286,15 @@ static bool illegal_characters(char *filename)
 	// these breaks the tar transform sed-like expression,
 	// hopefully can remove this check after moving to libtar
 	if (strncmp(filename, "+", 1) == 0) {
-		printf("WARNING: Filename %s begins with '+'! ...skipping.\n", filename);
 		return true;
 	}
 	if (strstr(filename, "+package+") != NULL) {
-		printf("WARNING: Filename %s contains \"+package+\"! ...skipping.\n", filename);
 		return true;
 	}
 
 	for (i = 0; i < BAD_CHAR_COUNT; i++) {
 		c = bad_chars[i];
 		if (strchr(filename, c) != NULL) {
-			printf("WARNING: Filename %s includes illegal character '%c'! ...skipping.\n", filename, c);
 			return true;
 		}
 	}
@@ -323,6 +320,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 
 	while (dir) {
 		struct file *file;
+		char *sub_filename;
 		char *fullname;
 
 		entry = readdir(dir);
@@ -331,8 +329,15 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		if ((strcmp(entry->d_name, ".") == 0) ||
-		    (strcmp(entry->d_name, "..") == 0) ||
-		    (illegal_characters(entry->d_name))) {
+		    (strcmp(entry->d_name, "..") == 0)) {
+			continue;
+		}
+
+		string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
+
+		if (illegal_characters(entry->d_name)) {
+			printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+			free(sub_filename);
 			continue;
 		}
 
@@ -342,7 +347,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		file->last_change = manifest->version;
-		string_or_die(&file->filename, "%s/%s", subpath, entry->d_name);
+		file->filename = sub_filename;
 
 		string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
 		populate_file_struct(file, fullname);
@@ -402,6 +407,58 @@ struct manifest *full_manifest_from_directory(int version)
 	return manifest;
 }
 
+/* Read includes from $manifest-includes file */
+GList *get_sub_manifest_includes(char *component, int version)
+{
+	FILE *infile;
+	GList *includes = NULL;
+	char *c;
+	char *conf;
+	char *filename;
+	char *included;
+	char line[8192];
+
+	conf = config_output_dir();
+	if (conf == NULL) {
+		assert(0);
+	}
+
+	string_or_die(&filename, "%s/%i/noship/%s-includes", conf, version, component);
+	free(conf);
+
+	LOG(NULL, "Reading includes", "%s", filename);
+	infile = fopen(filename, "rb");
+
+	if (infile == NULL) {
+		if (errno != ENOENT) {
+			LOG(NULL, "Cannot read includes", "%s (%s)\n", filename, strerror(errno));
+		}
+		free(filename);
+		return NULL;
+	}
+
+	line[0] = 0;
+	while (strcmp(line, "\n") != 0) {
+		line[0] = 0;
+		if (fgets(line, 8191, infile) == NULL) {
+			break;
+		}
+		c = strchr(line, '\n');
+		if (c) {
+			*c = 0;
+		}
+		if (strlen(line) == 0) {
+			break;
+		}
+		included = strdup(line);
+		includes = g_list_prepend(includes, included);
+	}
+	fclose(infile);
+	includes = g_list_sort(includes, (GCompareFunc)strcmp);
+
+	return includes;
+}
+
 struct manifest *sub_manifest_from_directory(char *component, int version)
 {
 	struct manifest *manifest;
@@ -419,6 +476,8 @@ struct manifest *sub_manifest_from_directory(char *component, int version)
 
 	manifest->files = g_list_sort(manifest->files, file_sort_filename);
 
+	manifest->includes = get_sub_manifest_includes(component, version);
+
 	return manifest;
 }
 

src/chroot.c

@@ -21,15 +21,14 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-
-#include <glib.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/config.c

@@ -20,11 +20,11 @@
  *
  */
 
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <assert.h>
-#include <glib.h>
 
 #include "swupd.h"
 

src/create_update.c

@@ -25,17 +25,17 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <getopt.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <glib.h>
-#include <errno.h>
 #include <sys/stat.h>
-#include <sys/wait.h>
 #include <sys/time.h>
-#include <getopt.h>
+#include <sys/wait.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
@@ -69,7 +69,7 @@ static void print_help(const char *name)
 	printf("Application Options:\n");
 	printf("   -o, --osversion         The OS version for which to create an update\n");
 	printf("   -m, --minversion        Optional minimum file version to write into manifests per file\n");
-	printf("   -F, --format            Optional format string [ default:=%d ]\n", SWUPD_DEFAULT_FORMAT);
+	printf("   -F, --format            Format number for the update\n");
 	printf("   -g, --getformat         Print current format string and exit\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
 	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
@@ -105,7 +105,7 @@ static bool parse_options(int argc, char **argv)
 			}
 			break;
 		case 'F':
-			if (!optarg || !set_format_string(optarg)) {
+			if (!optarg || !set_format(optarg)) {
 				printf("Invalid --format argument\n\n");
 				return false;
 			}
@@ -117,10 +117,10 @@ static bool parse_options(int argc, char **argv)
 			}
 			break;
 		case 'g':
-			if (format_string == NULL) {
-				printf("%d\n", SWUPD_DEFAULT_FORMAT);
+			if (format == 0) {
+				printf("No format specified\n");
 			} else {
-				printf("%s\n", format_string);
+				printf("%llu\n", format);
 				free_globals();
 			}
 			exit(0);
@@ -223,8 +223,11 @@ int main(int argc, char **argv)
 	struct manifest *old_full = NULL;
 	struct manifest *new_full = NULL;
 
+	GHashTable *new_manifests = g_hash_table_new(g_str_hash, g_str_equal);
+	GHashTable *old_manifests = g_hash_table_new(g_str_hash, g_str_equal);
 	GList *manifests_last_versions_list = NULL;
 	int newfiles = 0;
+	int old_deleted = 0;
 
 	struct timeval current_time;
 	struct timeval previous_time;
@@ -293,6 +296,13 @@ int main(int argc, char **argv)
 	apply_heuristics(old_full);
 	apply_heuristics(new_full);
 	match_manifests(old_full, new_full);
+
+	old_deleted = remove_old_deleted_files(old_full, new_full);
+	if (old_deleted > 0) {
+		LOG(NULL, "", "Old deleted files (%d) removed from full manifest", old_deleted);
+		printf("Old deleted files (%d) removed from full manifest\n", old_deleted);
+	}
+
 	apply_heuristics(new_full);
 #warning disabled rename detection for some simplicity
 	// rename_detection(new_full);
@@ -332,6 +342,7 @@ int main(int argc, char **argv)
 		/* Detect renamed files specifically for each pack */
 		// rename_detection(...);
 
+		old_deleted = remove_old_deleted_files(old_core, new_core);
 		sort_manifest_by_version(new_core);
 		newfiles = prune_manifest(new_core);
 		if (newfiles <= 0) {
@@ -339,8 +350,8 @@ int main(int argc, char **argv)
 			printf("Core component has not changed (after pruning), exiting\n");
 			goto exit;
 		}
-		LOG(NULL, "", "Core component has changes (%d), writing out new manifest", newfiles);
-		printf("Core component has changes (%d), writing out new manifest\n", newfiles);
+		LOG(NULL, "", "Core component has changes (%d new, %d deleted), writing out new manifest", newfiles, old_deleted);
+		printf("Core component has changes (%d new, %d deleted), writing out new manifest\n", newfiles, old_deleted);
 		if (write_manifest(new_core) != 0) {
 			LOG(NULL, "", "Core component manifest write failed");
 			printf("Core component manifest write failed\n");
@@ -353,6 +364,43 @@ int main(int argc, char **argv)
 
 	/* Phase 3: the functional bundles */
 	printf("Entering phase 3: The bundles\n");
+	while (1) {
+		char *group = next_group();
+
+		if (!group) {
+			break;
+		}
+
+		(void)g_hash_table_insert(new_manifests, group, sub_manifest_from_directory(group, newversion));
+		(void)g_hash_table_insert(old_manifests, group, manifest_from_file(manifest_subversion(old_MoM, group), group));
+	}
+	while (1) {
+		GList *manifest_includes = NULL;
+		GList *name_includes;
+		char *group = next_group();
+		struct manifest *manifest;
+
+		if (!group) {
+			break;
+		}
+		manifest = g_hash_table_lookup(new_manifests, group);
+		name_includes = manifest->includes;
+		while (name_includes) {
+			char *name = name_includes->data;
+			name_includes = g_list_next(name_includes);
+			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(new_manifests, name));
+		}
+		manifest->includes = manifest_includes;
+		manifest_includes = NULL;
+		manifest = g_hash_table_lookup(old_manifests, group);
+		name_includes = manifest->includes;
+		while (name_includes) {
+			char *name = name_includes->data;
+			name_includes = g_list_next(name_includes);
+			manifest_includes = g_list_prepend(manifest_includes, g_hash_table_lookup(old_manifests, name));
+		}
+		manifest->includes = manifest_includes;
+	}
 	while (1) {
 		char *group = next_group();
 		struct manifest *oldm;
@@ -369,19 +417,27 @@ int main(int argc, char **argv)
 		printf("Processing bundle %s\n", group);
 
 		/* Step 4: Make a manifest for this functonal group */
-		oldm = manifest_from_file(manifest_subversion(old_MoM, group), group);
-		newm = sub_manifest_from_directory(group, newversion);
+		oldm = g_hash_table_lookup(old_manifests, group);
+		newm = g_hash_table_lookup(new_manifests, group);
 		add_component_hashes_to_manifest(newm, new_full);
 		apply_heuristics(oldm);
 		apply_heuristics(newm);
 		newm->prevversion = oldm->version;
 
+		/* add os-core as an included manifest */
+		if (!manifest_includes(oldm, "os-core")) {
+			oldm->includes = g_list_prepend(oldm->includes, old_core);
+		}
+		if (!manifest_includes(newm, "os-core")) {
+			newm->includes = g_list_prepend(newm->includes, new_core);
+		}
+
 		/* Step 5: Subtract the core files from the manifest */
-		subtract_manifests(oldm, old_core);
-		subtract_manifests(newm, new_core);
+		subtract_manifests_frontend(oldm, oldm);
+		subtract_manifests_frontend(newm, newm);
 
 		/* Step 6: Compare manifest to the previous version... */
-		if (match_manifests(oldm, newm) == 0) {
+		if (match_manifests(oldm, newm) == 0 && !changed_includes(oldm, newm)) {
 			LOG(NULL, "", "%s components have not changed, no new manifest", group);
 			printf("%s components have not changed, no new manifest\n", group);
 			/* Step 6a: if nothing changed, stay at the old version */
@@ -390,12 +446,13 @@ int main(int argc, char **argv)
 			apply_heuristics(newm);
 #warning missing rename_detection here
 			/* Step 6b: otherwise, write out the manifest */
+			old_deleted = remove_old_deleted_files(oldm, newm);
 			sort_manifest_by_version(newm);
 			type_change_detection(newm);
 			newfiles = prune_manifest(newm);
-			if (newfiles > 0) {
-				LOG(NULL, "", "%s component has changes (%d), writing out new manifest", group, newfiles);
-				printf("%s component has changes (%d), writing out new manifest\n", group, newfiles);
+			if (newfiles > 0 || old_deleted > 0 || changed_includes(oldm, newm)) {
+				LOG(NULL, "", "%s component has changes (%d new, %d deleted), writing out new manifest", group, newfiles, old_deleted);
+				printf("%s component has changes (%d new, %d deleted), writing out new manifest\n", group, newfiles, old_deleted);
 				if (write_manifest(newm) != 0) {
 					LOG(NULL, "", "%s component manifest write failed", group);
 					printf("%s component manifest write failed\n", group);

src/delta.c

@@ -21,18 +21,16 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <bsdiff.h>
+#include <errno.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <errno.h>
-
-#include <bsdiff.h>
-
-#include <glib.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 #include "xattrs.h"

src/fullfiles.c

@@ -22,18 +22,17 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <glib.h>
+#include <libgen.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <stdint.h>
-#include <errno.h>
-#include <libgen.h>
-
-#include <glib.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
@@ -92,8 +91,7 @@ static void create_fullfile(struct file *file)
 			assert(0);
 		}
 
-		string_or_die(&tarcommand, TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -cf - --exclude='%s/?*' './%s' 2> /dev/null | "
-			      TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -xf - 2> /dev/null",
+		string_or_die(&tarcommand, TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -cf - --exclude='%s/?*' './%s' 2> /dev/null | " TAR_COMMAND " -C %s " TAR_PERM_ATTR_ARGS " -xf - 2> /dev/null",
 			      dir, base, base, rename_tmpdir);
 		if (system(tarcommand) != 0) {
 			LOG(NULL, "Failed to run command:", "%s", tarcommand);

src/globals.c

@@ -21,19 +21,19 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
 int newversion = -1;
 int minversion = 0;
-char *format_string = NULL;
+unsigned long long int format = 0;
 bool enable_signing = false;
 
 char *state_dir = NULL;
@@ -41,20 +41,17 @@ char *packstage_dir = NULL;
 char *image_dir = NULL;
 char *staging_dir = NULL;
 
-bool set_format_string(char *userinput)
+bool set_format(char *userinput)
 {
-	int version;
+	unsigned long long int user_format;
 
-	// expect a positive integer
+	// format string shall be a positive integer
 	errno = 0;
-	version = strtoull(userinput, NULL, 10);
-	if ((errno < 0) || (version <= 0)) {
+	user_format = strtoull(userinput, NULL, 10);
+	if ((errno < 0) || (user_format == 0)) {
 		return false;
 	}
-	if (format_string) {
-		free(format_string);
-	}
-	string_or_die(&format_string, "%d", version);
+	format = user_format;
 
 	return true;
 }
@@ -81,8 +78,9 @@ bool set_state_dir(char *dir)
 
 bool init_globals(void)
 {
-	if (format_string == NULL) {
-		string_or_die(&format_string, "%d", SWUPD_DEFAULT_FORMAT);
+	if (format == 0) {
+		printf("Error: Missing format parameter. Please specify a format with -F.\n");
+		return false;
 	}
 
 	if (!init_state_globals()) {
@@ -101,7 +99,6 @@ bool init_globals(void)
 
 void free_globals(void)
 {
-	free(format_string);
 	free(state_dir);
 	free(packstage_dir);
 	free(image_dir);

src/groups.c

@@ -23,11 +23,11 @@
  *
  */
 
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <assert.h>
-#include <glib.h>
 
 #include "swupd.h"
 

src/helpers.c

@@ -21,19 +21,18 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/time.h>
+#include <sys/types.h>
 #include <sys/wait.h>
-#include <fcntl.h>
-#include <errno.h>
-
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/heuristics.c

@@ -21,13 +21,12 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/log.c

@@ -22,15 +22,14 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <stdarg.h>
 #include <sys/time.h>
-
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/make_fullfiles.c

@@ -21,13 +21,12 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <getopt.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-
-#include <getopt.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/make_packs.c

@@ -21,20 +21,18 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <getopt.h>
+#include <getopt.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <dirent.h>
-#include <getopt.h>
-
-#include <glib.h>
-
-#include <getopt.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/manifest.c

@@ -23,19 +23,17 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <bsdiff.h>
+#include <errno.h>
+#include <glib.h>
+#include <libgen.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <libgen.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-
-#include <bsdiff.h>
-
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 #include "xattrs.h"
@@ -101,6 +99,7 @@ struct manifest *alloc_manifest(int version, char *component)
 
 	manifest->version = version;
 	manifest->component = strdup(component);
+	manifest->format = format;
 
 	return manifest;
 }
@@ -108,11 +107,13 @@ struct manifest *alloc_manifest(int version, char *component)
 struct manifest *manifest_from_file(int version, char *component)
 {
 	FILE *infile;
+	GList *includes = NULL;
 	char line[8192], *c, *c2;
 	int count = 0;
 	struct manifest *manifest;
 	char *filename, *conf;
 	int previous = 0;
+	unsigned long long int format_number;
 
 	conf = config_output_dir();
 	if (conf == NULL) {
@@ -144,7 +145,9 @@ struct manifest *manifest_from_file(int version, char *component)
 		return NULL;
 	}
 	c = &line[9];
-	if (strtoull(c, NULL, 10) == 0) {
+	format_number = strtoull(c, NULL, 10);
+	if ((errno < 0) || (format_number == 0)) {
+		//format string shall be a positive integer
 		printf("Unknown file format version in MANIFEST line: %s\n", c);
 		fclose(infile);
 		return NULL;
@@ -178,10 +181,18 @@ struct manifest *manifest_from_file(int version, char *component)
 		if (strncmp(line, "previous:", 9) == 0) {
 			previous = strtoull(c, NULL, 10);
 		}
+		if (strncmp(line, "includes:", 9) == 0) {
+			includes = g_list_prepend(includes, strdup(c));
+			if (!includes->data) {
+				abort();
+			}
+		}
 	}
 
 	manifest = alloc_manifest(version, component);
+	manifest->format = format_number;
 	manifest->prevversion = previous;
+	manifest->includes = includes;
 
 	/* empty line */
 	while (!feof(infile)) {
@@ -509,29 +520,57 @@ int match_manifests(struct manifest *m1, struct manifest *m2)
 	return count;
 }
 
-/* removes all files from m2 from m1 */
+static GList *get_unique_includes(struct manifest *manifest)
+{
+	GHashTable *unique_includes = g_hash_table_new(g_str_hash, g_str_equal);
+	GHashTableIter iter;
+	GList *includes = NULL;
+	GList *l1, *l2;
+	gpointer k, v;
+
+	l1 = g_list_first(manifest->includes);
+	while (l1) {
+		struct manifest *m = l1->data;
+		l1 = g_list_next(l1);
+		(void)g_hash_table_replace(unique_includes, m->component, m);
+		l2 = get_unique_includes(m);
+		while (l2) {
+			struct manifest *m = l2->data;
+			l2 = g_list_next(l2);
+			(void)g_hash_table_replace(unique_includes, m->component, m);
+		}
+		g_list_free(l2);
+	}
+	g_hash_table_iter_init(&iter, unique_includes);
+	while (g_hash_table_iter_next(&iter, &k, &v)) {
+		includes = g_list_prepend(includes, v);
+	}
+
+	return includes;
+}
+
+/*
+ * removes all files from m2 from m1
+ * This will recurse over all included manifests to also
+ * subtract those from the m1 manifest.
+ *
+ * As a special convenient semantics,
+ * subtract_manifests(M, M)
+ * will subtract all included manifests from M,
+ * but will not subtract M from M itself.
+ */
 void subtract_manifests(struct manifest *m1, struct manifest *m2)
 {
 	GList *list1, *list2;
 	struct file *file1, *file2;
 
-	if (!m1) {
-		printf("Subtracting manifests failed: No m1 manifest!\n");
-		return;
-	}
-
-	if (!m2) {
-		printf("Subtracting manifests failed: No m2 manifest!\n");
-		return;
-	}
-
 	m1->files = g_list_sort(m1->files, file_sort_filename);
 	m2->files = g_list_sort(m2->files, file_sort_filename);
 
 	list1 = g_list_first(m1->files);
 	list2 = g_list_first(m2->files);
 
-	while (list1 && list2) {
+	while (list1 && list2 && m1 != m2) {
 		int ret;
 		file1 = list1->data;
 		file2 = list2->data;
@@ -555,6 +594,31 @@ void subtract_manifests(struct manifest *m1, struct manifest *m2)
 	}
 }
 
+void subtract_manifests_frontend(struct manifest *m1, struct manifest *m2)
+{
+	GList *includes;
+	struct manifest *m;
+
+	if (!m1) {
+		printf("Subtracting manifests failed: No m1 manifest!\n");
+		return;
+	}
+
+	if (!m2) {
+		printf("Subtracting manifests failed: No m2 manifest!\n");
+		return;
+	}
+
+	subtract_manifests(m1, m2);
+
+	includes = get_unique_includes(m2);
+	while (includes) {
+		m = includes->data;
+		includes = g_list_next(includes);
+		subtract_manifests(m1, m);
+	}
+}
+
 char *file_type_to_string(struct file *file)
 {
 	static char type[5];
@@ -657,6 +721,7 @@ exit:
 /* Returns 0 == success, -1 == failure */
 static int write_manifest_plain(struct manifest *manifest)
 {
+	GList *includes;
 	GList *list;
 	struct file *file;
 	FILE *out = NULL;
@@ -687,13 +752,19 @@ static int write_manifest_plain(struct manifest *manifest)
 		goto exit;
 	}
 
-	fprintf(out, "MANIFEST\t%s\n", format_string);
+	fprintf(out, "MANIFEST\t%llu\n", format);
 	fprintf(out, "version:\t%i\n", manifest->version);
 	fprintf(out, "previous:\t%i\n", manifest->prevversion);
 	fprintf(out, "filecount:\t%i\n", manifest->count);
 	fprintf(out, "timestamp:\t%i\n", (int)time(NULL));
 	compute_content_size(manifest);
 	fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
+	includes = manifest->includes;
+	while (includes) {
+		struct manifest *sub = includes->data;
+		includes = g_list_next(includes);
+		fprintf(out, "includes:\t%s\n", sub->component);
+	}
 	fprintf(out, "\n");
 
 	list = g_list_first(manifest->files);
@@ -747,12 +818,12 @@ static int write_manifest_tar(struct manifest *manifest)
 	/* and put the signature of the plain manifest into the archive, too */
 	if (enable_signing) {
 		string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
-				       "%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed",
+						   "%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed",
 			      conf, manifest->version, conf, manifest->version, manifest->component,
 			      manifest->component, manifest->component);
 	} else {
 		string_or_die(&tarcmd, TAR_COMMAND " --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf "
-				       "%s/%i/Manifest.%s.tar Manifest.%s",
+						   "%s/%i/Manifest.%s.tar Manifest.%s",
 			      conf, manifest->version, conf, manifest->version, manifest->component,
 			      manifest->component);
 	}
@@ -811,6 +882,115 @@ void sort_manifest_by_version(struct manifest *manifest)
 	}
 }
 
+bool manifest_includes(struct manifest *manifest, char *component)
+{
+	GList *includes = g_list_first(manifest->includes);
+
+	while (includes) {
+		if (strcmp(((struct manifest *)includes->data)->component,
+			   component) == 0) {
+			return true;
+		}
+		includes = g_list_next(includes);
+	}
+
+	return false;
+}
+
+/* This requires the manifest to have the includes sorted.
+ * The manifests will be sorted when constructed from
+ * get_sub_manifest_includes (and then replacing the manifest
+ * strings with actual manifest structs in the same order as
+ * is done in the main.c's main()).
+ */
+bool changed_includes(struct manifest *old, struct manifest *new)
+{
+	GList *includes_old = g_list_first(old->includes);
+	GList *includes_new = g_list_first(new->includes);
+
+	if (g_list_length(includes_old) != g_list_length(includes_new)) {
+		return true;
+	}
+
+	while (includes_old && includes_new) {
+		char *bundle_old = ((struct manifest *)includes_old->data)->component;
+		char *bundle_new = ((struct manifest *)includes_new->data)->component;
+		if (strcmp(bundle_old, bundle_new) != 0) {
+			return true;
+		}
+
+		includes_old = g_list_next(includes_old);
+		includes_new = g_list_next(includes_new);
+	}
+
+	return false;
+}
+
+/* For a format bump, it's convenient to remove deleted files from manifests
+ * that last changed prior to the format bump, since they are no longer
+ * considered for deletion as part of an update.
+ *
+ * Note: this function should be called after match_manifests().
+ */
+int remove_old_deleted_files(struct manifest *m1, struct manifest *m2)
+{
+	GList *list1, *list2;
+	struct file *file1, *file2;
+	int count = 0;
+
+	if (!m1) {
+		printf("No old manifest!\n");
+		return -1;
+	}
+
+	if (!m2) {
+		printf("No new manifest!\n");
+		return -1;
+	}
+
+	/* This is the common case, so bail early, reporting no deletions */
+	if (m1->format == m2->format) {
+		return 0;
+	}
+
+	/* At this point, the manifest formats mismatch, and it's assumed that
+	 * m1 is the old manifest, and m2 is the new.
+	 */
+
+	m1->files = g_list_sort(m1->files, file_sort_filename);
+	m2->files = g_list_sort(m2->files, file_sort_filename);
+
+	list1 = g_list_first(m1->files);
+	list2 = g_list_first(m2->files);
+
+	while (list1 && list2) {
+		int ret;
+		file1 = list1->data;
+		file2 = list2->data;
+
+		ret = strcmp(file1->filename, file2->filename);
+		if (ret == 0) {
+			if (file1->is_deleted && file2->is_deleted) {
+				GList *to_delete = list2;
+				list1 = g_list_next(list1);
+				list2 = g_list_next(list2);
+				m2->files = g_list_delete_link(m2->files, to_delete);
+				m2->count--;
+				count++;
+				continue;
+			}
+			list1 = g_list_next(list1);
+			list2 = g_list_next(list2);
+		} else if (ret < 0) {
+			list1 = g_list_next(list1);
+		} else {
+			list2 = g_list_next(list2);
+		}
+	}
+
+	return count;
+}
+
 /* Conditionally remove some things from a manifest.
  * Returns > 0 when the pruned manifest has new files.
  * Returns 0 when the pruned manifest no longer has new files.
@@ -1001,7 +1181,7 @@ static void maximize_version_manifests(struct manifest *m1, struct manifest *m2)
 
 		ret = strcmp(file1->filename, file2->filename);
 		if (ret == 0) {
-			if (file1->last_change > file2->last_change) {
+			if (!file1->is_deleted && file1->last_change > file2->last_change) {
 				LOG(file1, "Update", "Moving %s to version %i", file1->filename, file1->last_change);
 				file2->last_change = file1->last_change;
 			}

src/pack.c

@@ -24,18 +24,17 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <dirent.h>
+#include <errno.h>
+#include <glib.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <dirent.h>
-
-#include <glib.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
@@ -118,8 +117,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 		 * the resulting pack is slightly smaller, and in addition, we're saving CPU
 		 * time on the client...
 		 */
-		string_or_die(&tar, TAR_COMMAND " --directory=%s/%s/%i_to_%i/staged " TAR_WARN_ARGS " "
-			            TAR_PERM_ATTR_ARGS " -xf %s",
+		string_or_die(&tar, TAR_COMMAND " --directory=%s/%s/%i_to_%i/staged " TAR_WARN_ARGS " " TAR_PERM_ATTR_ARGS " -xf %s",
 			      packstage_dir, module, from_version, to_version, path);
 		ret = system(tar);
 		if (!ret) {
@@ -454,7 +452,6 @@ static int make_final_pack(struct packdata *pack)
 			create_manifest_delta(pack->from, pack->to, pack->module);
 		}
 
-
 		string_or_die(&to, "%s/%s/%i_to_%i/Manifest-%s-delta-from-%i", packstage_dir,
 			      pack->module, pack->from, pack->to, pack->module, pack->from);
 
@@ -495,7 +492,7 @@ static int make_final_pack(struct packdata *pack)
 	/* tar the staging directory up */
 	LOG(NULL, "starting tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to);
 	string_or_die(&tar, TAR_COMMAND " " TAR_PERM_ATTR_ARGS " --directory=%s/%s/%i_to_%i/ "
-		            "--numeric-owner -Jcf %s/%i/pack-%s-from-%i.tar delta staged",
+					"--numeric-owner -Jcf %s/%i/pack-%s-from-%i.tar delta staged",
 		      packstage_dir, pack->module, pack->from, pack->to, staging_dir, pack->to,
 		      pack->module, pack->from);
 	ret = system(tar);

src/rename.c

@@ -22,16 +22,16 @@
  */
 
 #define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
 #include <assert.h>
 #include <ctype.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <glib.h>
 #include <libgen.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/signature.c

@@ -21,6 +21,7 @@
  */
 
 #define _GNU_SOURCE
+#include <err.h>
 #include <errno.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -29,7 +30,6 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <err.h>
 
 #include "swupd.h"
 

src/stats.c

@@ -22,13 +22,12 @@
  */
 
 #define _GNU_SOURCE
+#include <assert.h>
+#include <glib.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <assert.h>
-
-#include <glib.h>
+#include <unistd.h>
 
 #include "swupd.h"
 

src/versions.c

@@ -23,17 +23,16 @@
 
 #define _GNU_SOURCE
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
 #include <assert.h>
-#include <sys/stat.h>
 #include <ctype.h>
 #include <dirent.h>
-
+#include <errno.h>
 #include <glib.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
 #include "swupd.h"
 
@@ -140,7 +139,7 @@ void write_cookiecrumbs_to_download_area(int version)
 	}
 	free(cmd);
 
-	string_or_die(&cmd, "echo %s > %s/%i/format", format_string, conf, version);
+	string_or_die(&cmd, "echo %llu > %s/%i/format", format, conf, version);
 	if (system(cmd) != 0) {
 		assert(0);
 	}

src/xattrs.c

@@ -20,15 +20,15 @@
  *
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
-#include <assert.h>
-#include <sys/xattr.h>
-#include <sys/types.h>
 #include <sys/stat.h>
-#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/xattr.h>
 
 #include "swupd.h"
 #include "xattrs.h"

test/functional/basic/test.bats

@@ -1,23 +1,25 @@
 #!/usr/bin/env bats
 
+load "../swupdlib"
+
 @test "create_update help output" {
-  run $srcdir/swupd_create_update --help
+  run $CREATE_UPDATE --help
   [ "$status" -eq 1 ]
-  run $srcdir/swupd_create_update -h
+  run $CREATE_UPDATE -h
   [ "$status" -eq 1 ]
 }
 
 @test "make_fullfiles help output" {
-  run $srcdir/swupd_make_fullfiles --help
+  run $MAKE_FULLFILES --help
   [ "$status" -eq 1 ]
-  run $srcdir/swupd_make_fullfiles -h
+  run $MAKE_FULLFILES -h
   [ "$status" -eq 1 ]
 }
 
 @test "make_pack help output" {
-  run $srcdir/swupd_make_pack --help
+  run $MAKE_PACK --help
   [ "$status" -eq 1 ]
-  run $srcdir/swupd_make_pack -h
+  run $MAKE_PACK -h
   [ "$status" -eq 1 ]
 }
 

test/functional/delete-no-version-bump/test.bats

@@ -0,0 +1,48 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle1 test-bundle2
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle1
+  set_os_release 10 test-bundle2
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle1
+  track_bundle 10 test-bundle2
+
+  set_os_release 20 os-core
+  set_os_release 20 test-bundle1
+  set_os_release 20 test-bundle2
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle1
+  track_bundle 20 test-bundle2
+
+  gen_file_plain 10 test-bundle1 foo
+  gen_file_plain 10 test-bundle2 foo
+
+  gen_file_plain 20 test-bundle1 foo
+}
+
+@test "delete no version bump update creation" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+
+  set_latest_ver 10
+
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+
+  [ 1 -eq $(grep '10	/foo' $DIR/www/10/Manifest.full | wc -l) ]
+  [ 1 -eq $(grep '10	/foo' $DIR/www/20/Manifest.full | wc -l) ]
+  [ 0 -eq $(grep '20	/foo' $DIR/www/20/Manifest.full | wc -l) ]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run-delta.bats

@@ -1,58 +0,0 @@
-#!/usr/bin/env bats
-
-# common functions
-load swupdlib
-
-setup() {
-  DIR=$(init_web_dir "$srcdir/web-dir")
-  export DIR
-
-  init_server_ini
-  set_latest_ver 0
-  init_groups_ini os-core test-bundle
-
-  set_os_release 10 os-core
-  track_bundle 10 os-core
-  track_bundle 10 test-bundle
-  set_os_release 20 os-core
-  track_bundle 20 os-core
-  track_bundle 20 test-bundle
-
-  gen_file_to_delta 10 4096 20 4 test-bundle
-}
-
-@test "full run update creation with delta packs" {
-  # build the first version
-  sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR
-  sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 os-core
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 test-bundle
-
-  set_latest_ver 10
-
-  # then the second version...
-  sudo $srcdir/swupd_create_update --osversion 20 --statedir $DIR
-  sudo $srcdir/swupd_make_fullfiles --statedir $DIR 20
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 os-core
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 20 test-bundle
-
-  # and with delta packs this time
-  sudo $srcdir/swupd_make_pack --statedir $DIR 10 20 os-core
-  sudo $srcdir/swupd_make_pack --statedir $DIR 10 20 test-bundle
-
-  # zero packs should exist (non-zero size) for both versions
-  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
-  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
-  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
-  [ -s $DIR/www/20/pack-test-bundle-from-0.tar ]
-
-  # and delta packs should exist (non-zero size) for the latest version
-  [ -s $DIR/www/20/pack-os-core-from-10.tar ]
-  [ -s $DIR/www/20/pack-test-bundle-from-10.tar ]
-}
-
-teardown() {
-  sudo rm -rf $DIR
-}
-
-# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run-delta/test.bats

@@ -0,0 +1,84 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle included included-two included-nested
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  track_bundle 20 included
+  track_bundle 20 included-two
+  track_bundle 20 included-nested
+
+  gen_file_to_delta 10 4096 20 4 test-bundle
+
+  gen_file_plain 10 test-bundle foo
+  gen_file_plain 10 test-bundle foobarbaz
+  gen_file_plain 20 test-bundle foo
+  gen_file_plain 20 included foo
+  gen_file_plain 20 included-two foo
+  gen_file_plain 20 included-two foobar
+  gen_file_plain 20 included-nested foobarbaz
+
+  gen_includes_file test-bundle 20 included included-two
+  gen_includes_file included 20 included-nested
+}
+
+@test "full run update creation with delta packs" {
+  # build the first version
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  # then the second version...
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+
+  # and with delta packs this time
+  sudo $MAKE_PACK --statedir $DIR 10 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 10 20 test-bundle
+
+  # zero packs should exist (non-zero size) for both versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-0.tar ]
+
+  # and delta packs should exist (non-zero size) for the latest version
+  [ -s $DIR/www/20/pack-os-core-from-10.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-10.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(grep '/foobar$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-two$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^.d..	0000000000000000000000000000000000000000000000000000000000000000	20	/foo$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^.d..	0000000000000000000000000000000000000000000000000000000000000000	20	/foobarbaz$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/20/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '/foobar$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/20/Manifest.included-nested | wc -l) ]]
+  [[ 6 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
+  [[ 4 -eq $(tar -tf $DIR/www/20/pack-test-bundle-from-0.tar | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run.bats

@@ -1,35 +0,0 @@
-#!/usr/bin/env bats
-
-# common functions
-load swupdlib
-
-setup() {
-  DIR=$(init_web_dir "$srcdir/web-dir")
-  export DIR
-
-  init_server_ini
-  set_latest_ver 0
-  init_groups_ini os-core test-bundle
-
-  set_os_release 10 os-core
-  set_os_release 10 test-bundle
-  track_bundle 10 os-core
-  track_bundle 10 test-bundle
-}
-
-@test "full run update creation" {
-  sudo $srcdir/swupd_create_update --osversion 10 --statedir $DIR
-  sudo $srcdir/swupd_make_fullfiles --statedir $DIR 10
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 os-core
-  sudo $srcdir/swupd_make_pack --statedir $DIR 0 10 test-bundle
-
-  # zero packs should exist (non-zero size) for the build
-  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
-  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
-}
-
-teardown() {
-  sudo rm -rf $DIR
-}
-
-# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/full-run/test.bats

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle
+
+  set_os_release 10 os-core
+  set_os_release 10 test-bundle
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  gen_file_plain 10 test-bundle foo
+}
+
+@test "full run update creation" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  # zero packs should exist (non-zero size) for the build
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles/test-bundle$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles/os-core$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/lib$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
+  [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
+  [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/fullfiles/test.bats

@@ -1,18 +1,20 @@
 #!/usr/bin/env bats
 
+load "../swupdlib"
+
 @test "make_fullfiles required arg" {
-  run $srcdir/swupd_make_fullfiles
+  run $MAKE_FULLFILES
   [ "$status" -eq 1 ]
 }
 
 @test "make_fullfiles too many arguments" {
   # exactly one argument must be passed
-  run $srcdir/swupd_make_fullfiles foo bar
+  run $MAKE_FULLFILES foo bar
   [ "$status" -eq 1 ]
 }
 
 @test "make_fullfiles root priv check" {
-  run $srcdir/swupd_make_fullfiles foo
+  run $MAKE_FULLFILES foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]
 }

test/functional/include-version-bump/test.bats

@@ -0,0 +1,99 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core test-bundle included included-two included-nested
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+  track_bundle 10 test-bundle
+
+  set_os_release 20 os-core
+  track_bundle 20 os-core
+  track_bundle 20 test-bundle
+  track_bundle 20 included
+  track_bundle 20 included-two
+
+  set_os_release 30 os-core
+  track_bundle 30 os-core
+  track_bundle 30 test-bundle
+  track_bundle 30 included
+  track_bundle 30 included-two
+  track_bundle 30 included-nested
+
+  gen_file_plain 10 test-bundle foo
+  gen_file_plain 20 test-bundle foo
+  gen_file_plain 20 included bar
+  gen_file_plain 20 included-two baz
+  gen_file_plain 30 test-bundle foo
+  gen_file_plain 30 included bar
+  gen_file_plain 30 included-two baz
+  gen_file_plain 30 included-nested foobarbaz
+
+  gen_includes_file test-bundle 20 included included-two
+  gen_includes_file test-bundle 30 included included-two
+  gen_includes_file included 30 included-nested
+}
+
+@test "full run update creation with delta packs" {
+  # build the first version
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 10
+  sudo $MAKE_PACK --statedir $DIR 0 10 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 10 test-bundle
+
+  set_latest_ver 10
+
+  # then the second version...
+  sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 20
+  sudo $MAKE_PACK --statedir $DIR 0 20 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 20 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 0 20 included
+  sudo $MAKE_PACK --statedir $DIR 0 20 included-two
+
+  set_latest_ver 20
+
+  # then the third version...
+  sudo $CREATE_UPDATE --osversion 30 --statedir $DIR --format 3
+  sudo $MAKE_FULLFILES --statedir $DIR 30
+  sudo $MAKE_PACK --statedir $DIR 0 30 os-core
+  sudo $MAKE_PACK --statedir $DIR 0 30 test-bundle
+  sudo $MAKE_PACK --statedir $DIR 0 30 included
+  sudo $MAKE_PACK --statedir $DIR 0 30 included-two
+  sudo $MAKE_PACK --statedir $DIR 0 30 included-nested
+
+  # zero packs should exist (non-zero size) for both versions
+  [ -s $DIR/www/10/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/10/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/20/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/20/pack-included-from-0.tar ]
+  [ -s $DIR/www/20/pack-included-two-from-0.tar ]
+  [ -s $DIR/www/30/pack-os-core-from-0.tar ]
+  [ -s $DIR/www/30/pack-test-bundle-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-two-from-0.tar ]
+  [ -s $DIR/www/30/pack-included-nested-from-0.tar ]
+
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/foo$' $DIR/www/10/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	os-core$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-two$' $DIR/www/20/Manifest.test-bundle | wc -l) ]]
+  [[ 1 -eq $(grep '/bar$' $DIR/www/20/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/baz$' $DIR/www/20/Manifest.included-two | wc -l) ]]
+  [[ 1 -eq $(grep '^includes:	included-nested$' $DIR/www/30/Manifest.included | wc -l) ]]
+  [[ 1 -eq $(grep '/foobarbaz$' $DIR/www/30/Manifest.included-nested | wc -l) ]]
+  [[ 0 -eq $(ls $DIR/www/30/Manifest.test-bundle | wc -l) ]]
+  [[ 0 -eq $(ls $DIR/www/30/Manifest.included-two | wc -l) ]]
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/pack/test.bats

@@ -1,24 +1,26 @@
 #!/usr/bin/env bats
 
+load "../swupdlib"
+
 @test "make_pack required arg" {
-  run $srcdir/swupd_make_pack
+  run $MAKE_PACK
   [ "$status" -eq 1 ]
 }
 
 @test "make_pack too few arguments" {
-  run $srcdir/swupd_make_pack foo
+  run $MAKE_PACK foo
   [ "$status" -eq 1 ]
-  run $srcdir/swupd_make_pack foo bar
+  run $MAKE_PACK foo bar
   [ "$status" -eq 1 ]
 }
 
 @test "make_pack too many arguments" {
-  run $srcdir/swupd_make_pack foo bar foo bar
+  run $MAKE_PACK foo bar foo bar
   [ "$status" -eq 1 ]
 }
 
 @test "make_pack root priv check" {
-  run $srcdir/swupd_make_pack foo bar foo
+  run $MAKE_PACK foo bar foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]
 }

test/functional/swupdlib.bash

@@ -1,14 +1,28 @@
 # NOTE: source this file from a *.bats file
 
-init_web_dir() {
-  # absolute path is a hard requirement right now
-  local dir=$(realpath $1)
-  mkdir -p $dir/{image,www}
-  echo $dir
+# The location of the swupd_* binaries
+export SRCDIR="$BATS_TEST_DIRNAME/../../../"
+
+export CREATE_UPDATE="$SRCDIR/swupd_create_update"
+export MAKE_FULLFILES="$SRCDIR/swupd_make_fullfiles"
+export MAKE_PACK="$SRCDIR/swupd_make_pack"
+
+export DIR="$BATS_TEST_DIRNAME/web-dir"
+
+init_test_dir() {
+  local testdir="$BATS_TEST_DIRNAME"
+  mkdir -p "$testdir"/logs
+  mkdir -p $DIR/{image,www}
+  # run swupd_* inside the directory to dump the logs
+  cd "$testdir"/logs
+}
+
+clean_test_dir() {
+  sudo rm -rf $DIR "$BATS_TEST_DIRNAME"/logs
 }
 
 init_server_ini() {
-  cp $srcdir/server.ini $DIR
+  cp $SRCDIR/server.ini $DIR
   sed -i "s|/var/lib/update|$DIR|" $DIR/server.ini
 }
 
@@ -39,6 +53,18 @@ track_bundle() {
   touch $DIR/image/$ver/$bundle/usr/share/clear/bundles/$bundle
 }
 
+gen_includes_file() {
+  local bundle=$1
+  local ver=$2
+  local includes="${@:3}"
+  mkdir -p $DIR/www/$ver/noship
+  for b in $includes; do
+    cat >> $DIR/www/$ver/noship/"$bundle"-includes << EOF
+$b
+EOF
+  done
+}
+
 gen_file_to_delta() {
   local origver=$1
   local origsize=$2
@@ -58,4 +84,24 @@ gen_file_to_delta() {
   rm $TMP
 }
 
+gen_file_plain() {
+  local ver=$1
+  local bundle=$2
+  local name=$3
+
+  # Add plain text file into a bundle
+  mkdir -p $DIR/image/$ver/$bundle
+  echo $name > $DIR/image/$ver/$bundle/$name
+}
+
+gen_file_plain_change() {
+  local ver=$1
+  local bundle=$2
+  local name=$3
+
+  # Add plain text file into a bundle
+  mkdir -p $DIR/image/$ver/$bundle
+  echo $ver $name > $DIR/image/$ver/$bundle/$name
+}
+
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/update/test.bats

@@ -1,13 +1,21 @@
 #!/usr/bin/env bats
 
-@test "create_update required arg" {
-  run $srcdir/swupd_create_update
+load "../swupdlib"
+
+@test "create_update required format" {
+  run $CREATE_UPDATE
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "Missing format parameter" ]]
+}
+
+@test "create_update required version" {
+  run $CREATE_UPDATE -F 3
   [ "$status" -eq 1 ]
   [[ "$output" =~ "Missing version parameter:" ]]
 }
 
 @test "create_update root priv check" {
-  run $srcdir/swupd_create_update -o 10
+  run $CREATE_UPDATE -F 3 -o 10
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]
 }