Release v3.3.2

This release includes changes across various areas of server, specifically:
- Fixing fallthrough detection logic to detect files and directories correctly
  when checking if they are "state" files.
- Removing stale and unused signature creation code
- Adding functional tests and enabling travis-ci integration
- Create alternative input layout to save IO for some cases(backward compatible)
- Update parallelism code to make it more versatile and editable
- Fix log call and add logging to stdout instead of just logfiles
- Honor proxy and cert checking settings
- Fix extracting files with bsdtar
- Enable locales in all programs

Signed-off-by: Tudor Marcu <tudor.marcu@intel.com>

.travis.yml

@@ -0,0 +1,25 @@
+sudo: required
+dist: trusty
+language: c
+
+# Pre-install missing build dependencies:
+# - libmagic (pull from repo)
+# - libcheck 0.9.10 is slightly too old, since 0.9.12 adds TAP support
+# - bsdiff 1.* is the Clear Linux OS fork
+before_install:
+        - sudo apt-get -qq update
+        - sudo apt-get install -y libmagic-dev
+
+install:
+        - wget http://downloads.sourceforge.net/project/check/check/0.10.0/check-0.10.0.tar.gz
+        - tar -xvf check-0.10.0.tar.gz
+        - pushd check-0.10.0 && ./configure --prefix=/usr && make -j48 && sudo make install && popd
+        - wget https://github.com/clearlinux/bsdiff/releases/download/v1.0.2/bsdiff-1.0.2.tar.xz
+        - tar -xvf bsdiff-1.0.2.tar.xz
+        - pushd bsdiff-1.0.2 && ./configure --prefix=/usr --disable-tests && make -j48 && sudo make install && popd
+
+# Ubuntu's default umask is 0002, but this break's swupd hash calculations.
+script:
+        - sudo find test/functional -exec chmod g-w {} \;
+        - autoreconf --verbose --warnings=none --install --force && ./configure && make -j48 && sudo sh -c 'umask 0022 && make -j48 check'
+after_failure: cat test-suite.log

Makefile.am

@@ -26,7 +26,6 @@ swupd_create_update_SOURCES = \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/type_change.c \
 	src/versions.c \
@@ -37,13 +36,13 @@ swupd_make_pack_SOURCES = \
 	src/config.c \
 	src/delta.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_packs.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c
 
@@ -53,13 +52,13 @@ swupd_make_fullfiles_SOURCES = \
 	src/delta.c \
 	src/fullfiles.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_fullfiles.c \
 	src/manifest.c \
 	src/pack.c \
 	src/rename.c \
-	src/signature.c \
 	src/stats.c \
 	src/xattrs.c
 
@@ -116,7 +115,8 @@ dist_check_SCRIPTS = \
 	test/functional/pack/test.bats \
 	test/functional/full-run/test.bats \
 	test/functional/full-run-delta/test.bats \
-	test/functional/file-name-blacklisted/test.bats
+	test/functional/file-name-blacklisted/test.bats \
+	test/functional/state-file/test.bats
 endif
 
 if COVERAGE

basic_creator.sh

@@ -15,7 +15,7 @@ error() {
 # eg: all of openstack, pnp, bat, cloud stuff, non-basic scripting language bundles
 ${SWUPDREPO}/mk_groups_ini.sh
 
-PREVREL=`cat ${UPDATEDIR}/image/latest.version`
+PREVREL=`cat ${UPDATEDIR}/image/LAST_VER`
 
 export SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR:-"/root/swupd-certs"}
 export LEAF_KEY="leaf.key.pem"
@@ -46,7 +46,7 @@ for job in $(jobs -p); do
 done
 
 # expose the new build to staging / testing
-echo ${VER} > ${UPDATEDIR}/image/latest.version
+echo ${VER} > ${UPDATEDIR}/image/LAST_VER
 
 STAGING_FILE="${UPDATEDIR}/www/version/formatstaging/latest"
 if [ ! -f "${STAGING_FILE}" ]; then

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.2.4, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.3.2, tudor.marcu@intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -20,12 +20,12 @@
 #define TAR_COMMAND "bsdtar"
 #define TAR_XATTR_ARGS ""
 #define TAR_XATTR_ARGS_STRLIST
-#define TAR_WARN_ARGS ""
+#define TAR_WARN_ARGS_STRLIST
 #else
 #define TAR_COMMAND "tar"
 #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'"
 #define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'",
-#define TAR_WARN_ARGS "--warning=no-timestamp"
+#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp",
 #endif
 
 #if SWUPD_WITH_SELINUX
@@ -36,6 +36,8 @@
 #define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions"
 #endif
 
+#define SWUPD_HASH_DIRNAME "DIRECTORY"
+
 #if SWUPD_WITH_STATELESS
 #define OS_IS_STATELESS 1
 #else
@@ -141,7 +143,6 @@ extern int current_version;
 extern int newversion;
 extern int minversion;
 extern unsigned long long int format;
-extern bool enable_signing;
 
 extern char *state_dir;
 extern char *packstage_dir;
@@ -219,6 +220,7 @@ extern void read_group_file(char *filename);
 extern void release_group_file(void);
 extern char *group_groups(char *group);
 extern char *group_packages(char *group);
+extern char *group_status(char *group);
 extern char *next_group(void);
 
 extern void apply_heuristics(struct manifest *manifest);
@@ -242,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest);
 
 extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
 extern void link_renames(GList *newfiles, struct manifest *from_manifest);
-extern void __create_delta(struct file *file, int from_version);
+extern void __create_delta(struct file *file, int from_version, char *from_hash);
 
 extern void account_delta_hit(void);
 extern void account_delta_miss(void);
@@ -255,9 +257,6 @@ extern int system_argv(char *const argv[]);
 extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
 extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
 			    char *const argvp2[], int stdoutp2, int stderrp2);
-
-extern bool signature_initialize(void);
-extern void signature_terminate(void);
-extern bool signature_sign(const char *filename);
+extern int num_threads(float scaling);
 
 #endif

mk_groups_ini.sh

@@ -18,8 +18,8 @@ if [ ! -f "$SWUPD_SERVER_INI" ]; then
 	sed -i "s|/var/lib/update|$UPDATEDIR|" $SWUPD_SERVER_INI
 fi
 
-if [ ! -f "$UPDATEDIR/image/latest.version" ]; then
-	echo "0" > $UPDATEDIR/image/latest.version
+if [ ! -f "$UPDATEDIR/image/LAST_VER" ]; then
+	echo "0" > $UPDATEDIR/image/LAST_VER
 fi
 
 echo "rebuilding $SWUPD_GROUPS_INI based on $BUNDLEREPO"
@@ -27,7 +27,11 @@ rm -f $SWUPD_GROUPS_INI
 
 for bundle in $(ls $BUNDLEREPO/bundles)
 do
+	status=$(awk -F: '/^# .STATUS/ {print $2}' $BUNDLEREPO/$bundle | tr -cd '[[:alnum:]]')
 	echo "[$bundle]" >> $SWUPD_GROUPS_INI
 	echo "group=$bundle" >> $SWUPD_GROUPS_INI
+	if [ -n "$status" ]; then
+		echo "status=$status" >> $SWUPD_GROUPS_INI
+	fi
 	echo "" >> $SWUPD_GROUPS_INI
 done

packfsck.pl

@@ -7,7 +7,7 @@
 
 my $target = $ARGV[0];
 system("rm /tmp/Manifest");
-system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core");
+system("wget --quiet --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core");
 
 
 my $from = $target;
@@ -15,7 +15,7 @@ while ($from > $target - 100) {
  $from = $from - 10;
  print "Testing the $from-$target pack\n";
  system("rm /tmp/pack.tar");
- system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar");
+ system("wget --quiet ---output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar");
 
 
 

src/analyze_fs.c

@@ -222,7 +222,7 @@ int compute_hash(struct file *file, char *filename)
 		hmac_sha256_for_string(file->hash,
 				       (const unsigned char *)key,
 				       key_len,
-				       file->filename); //file->filename not filename
+				       SWUPD_HASH_DIRNAME); // Make independent of dirname
 		return 0;
 	}
 
@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data)
 /* disallow characters which can do unexpected things when the filename is
  * used on a tar command line via system("tar [args] filename [more args]");
  */
-static bool illegal_characters(char *filename)
+static bool illegal_characters(const char *filename)
 {
 	char c;
 	int i;
@@ -301,27 +301,151 @@ static bool illegal_characters(char *filename)
 	return false;
 }
 
+static struct file *add_file(struct manifest *manifest,
+                             const char *entry_name,
+                             char *sub_filename,
+                             char *fullname,
+                             bool do_hash)
+{
+	GError *err = NULL;
+	struct file *file;
+
+	if (illegal_characters(entry_name)) {
+		printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+		free(sub_filename);
+		free(fullname);
+		return NULL;
+	}
+
+	file = calloc(1, sizeof(struct file));
+	assert(file);
+
+	file->last_change = manifest->version;
+	file->filename = sub_filename;
+
+	populate_file_struct(file, fullname);
+	if (file->is_deleted) {
+		/*
+		 * populate_file_struct() logs a stat() failure, but
+		 * does not abort. When adding files that should
+		 * exist, this case is an error.
+		 */
+		LOG(NULL, "file not found", "%s", fullname);
+		assert(0);
+	}
+
+
+	/* if for some reason there is a file in the official build
+	 * which should not be included in the Manifest, then open a bug
+	 * to get it removed, and work around its presence by
+	 * excluding it here, eg:
+	 if (strncmp(file->filename, "/dev/", 5) == 0) {
+	 continue;
+	 }
+	*/
+
+	if (do_hash) {
+		/* compute the hash from a thread */
+		int ret;
+		ret = g_thread_pool_push(threadpool, file, &err);
+		if (ret == FALSE) {
+			printf("GThread hash computation push error\n");
+			printf("%s\n", err->message);
+			assert(0);
+		}
+	}
+	manifest->files = g_list_prepend(manifest->files, file);
+	manifest->count++;
+        return file;
+}
+
+
 static void iterate_directory(struct manifest *manifest, char *pathprefix,
 			      char *subpath, bool do_hash)
 {
 	DIR *dir;
 	struct dirent *entry;
 	char *fullpath;
-	int ret;
-	GError *err = NULL;
 
 	string_or_die(&fullpath, "%s/%s", pathprefix, subpath);
 
 	dir = opendir(fullpath);
 	if (!dir) {
+		bool fatal_error = errno != ENOENT;
+		FILE *content;
+
 		free(fullpath);
+		if (fatal_error) {
+			return;
+		}
+		/*
+		 * If there is a <dir>.content.txt instead of
+		 * the actual directory, then read that
+		 * file. It has a list of path names,
+		 * including all directories. The
+		 * corresponding file system entry is then
+		 * expected to be in a pre-populated "full"
+		 * directory.
+		 *
+		 * Only supported at top level (i.e. empty
+		 * subpath) to keep the code and testing
+		 * simpler.
+		 */
+		assert(!subpath[0]);
+		string_or_die(&fullpath, "%s.content.txt", pathprefix);
+		content = fopen(fullpath, "r");
+		free(fullpath);
+		fullpath = NULL;
+		if (content) {
+			char *line = NULL;
+			size_t len = 0;
+			ssize_t read;
+			const char *full;
+			int full_len;
+			/*
+			 * determine path to "full" directory: it is assumed to be alongside
+			 * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit,
+			 * so we have to strip the last path component.
+			 */
+			full = strrchr(pathprefix, '/');
+			if (full) {
+				full_len = full - pathprefix + 1;
+				full = pathprefix;
+			} else {
+				full = "";
+				full_len = 0;
+			}
+			while ((read = getline(&line, &len, content)) != -1) {
+				if (read) {
+					const char *entry_name = strrchr(line, '/');
+					if (entry_name) {
+						entry_name++;
+					} else {
+						entry_name = line;
+					}
+					if (line[read - 1] == '\n') {
+						line[read - 1] = 0;
+					}
+					string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line);
+					add_file(manifest,
+						 entry_name,
+						 strdup(line),
+						 fullpath,
+						 do_hash);
+				}
+			}
+			free(line);
+		}
+
+		// If both directory and content file are missing, silently (?)
+		// don't add anything to the manifest.
 		return;
 	}
 
 	while (dir) {
-		struct file *file;
 		char *sub_filename;
 		char *fullname;
+                struct file *file;
 
 		entry = readdir(dir);
 		if (!entry) {
@@ -334,50 +458,14 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		}
 
 		string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
-
-		if (illegal_characters(entry->d_name)) {
-			printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
-			free(sub_filename);
-			continue;
-		}
-
-		file = calloc(1, sizeof(struct file));
-		if (!file) {
-			break;
-		}
-
-		file->last_change = manifest->version;
-		file->filename = sub_filename;
-
 		string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
-		populate_file_struct(file, fullname);
-		free(fullname);
 
-		if (entry->d_type == DT_DIR) {
-			iterate_directory(manifest, pathprefix, file->filename, do_hash);
-		}
+		/* takes ownership of the strings, so we don't need to free it */
+		file = add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
 
-		/* if for some reason there is a file in the official build
-		 * which should not be included in the Manifest, then open a bug
-		 * to get it removed, and work around its presence by
-		 * excluding it here, eg:
-		if (strncmp(file->filename, "/dev/", 5) == 0) {
-			continue;
+                if (file && file->is_dir) {
+                    iterate_directory(manifest, pathprefix, file->filename, do_hash);
 		}
-		 */
-
-		if (do_hash) {
-			/* compute the hash from a thread */
-			ret = g_thread_pool_push(threadpool, file, &err);
-			if (ret == FALSE) {
-				printf("GThread hash computation push error\n");
-				printf("%s\n", err->message);
-				closedir(dir);
-				return;
-			}
-		}
-		manifest->files = g_list_prepend(manifest->files, file);
-		manifest->count++;
 	}
 	closedir(dir);
 	free(fullpath);
@@ -387,6 +475,7 @@ struct manifest *full_manifest_from_directory(int version)
 {
 	struct manifest *manifest;
 	char *dir;
+	int numthreads = num_threads(1.0);
 
 	LOG(NULL, "Computing hashes", "for %i/full", version);
 
@@ -394,7 +483,7 @@ struct manifest *full_manifest_from_directory(int version)
 
 	string_or_die(&dir, "%s/%i/full", image_dir, version);
 
-	threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL);
+	threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL);
 
 	iterate_directory(manifest, dir, "", true);
 
@@ -418,7 +507,7 @@ GList *get_sub_manifest_includes(char *component, int version)
 	char *included;
 	char line[8192];
 
-	conf = config_output_dir();
+	conf = config_image_base();
 	if (conf == NULL) {
 		assert(0);
 	}

src/chroot.c

@@ -39,15 +39,21 @@ void chroot_create_full(int newversion)
 	char *full_dir;
 
 	string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
+	if (!access(full_dir, R_OK|X_OK)) {
+		free(full_dir);
+		return;
+	}
 
 	g_mkdir_with_parents(full_dir, S_IRWXU);
 
 	/* start with base */
-	LOG(NULL, "Copying chroot os-core to full", "");
 	string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
-	char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
-	if (system_argv(rsynccmd) != 0) {
-		assert(0);
+	if (!access(param, F_OK)) {
+		LOG(NULL, "Copying chroot os-core to full", "");
+		char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
+		if (system_argv(rsynccmd) != 0) {
+			assert(0);
+		}
 	}
 	free(param);
 
@@ -58,11 +64,13 @@ void chroot_create_full(int newversion)
 			break;
 		}
 
-		LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
 		string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
-		char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
-		if (system_argv(rsynccmd) != 0) {
-			assert(0);
+		if (!access(param, F_OK)) {
+			LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
+			char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
+			if (system_argv(rsynccmd) != 0) {
+				assert(0);
+			}
 		}
 		free(param);
 	}

src/create_update.c

@@ -29,6 +29,7 @@
 #include <errno.h>
 #include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -49,12 +50,12 @@ static void banner(void)
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
 	{ "version", no_argument, 0, 'v' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "osversion", required_argument, 0, 'o' },
 	{ "minversion", required_argument, 0, 'm' },
 	{ "format", required_argument, 0, 'F' },
 	{ "getformat", no_argument, 0, 'g' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -67,12 +68,12 @@ static void print_help(const char *name)
 	printf("   -v, --version           Show software version\n");
 	printf("\n");
 	printf("Application Options:\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -o, --osversion         The OS version for which to create an update\n");
 	printf("   -m, --minversion        Optional minimum file version to write into manifests per file\n");
 	printf("   -F, --format            Format number for the update\n");
 	printf("   -g, --getformat         Print current format string and exit\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }
 
@@ -80,12 +81,15 @@ static bool parse_options(int argc, char **argv)
 {
 	int opt;
 
-	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:s", prog_opts, NULL)) != -1) {
+	while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:", prog_opts, NULL)) != -1) {
 		switch (opt) {
 		case '?':
 		case 'h':
 			print_help(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'v':
 			banner();
 			return false;
@@ -124,9 +128,6 @@ static bool parse_options(int argc, char **argv)
 				free_globals();
 			}
 			exit(0);
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}
 
@@ -140,13 +141,14 @@ static bool parse_options(int argc, char **argv)
 static void populate_dirs(int version)
 {
 	char *newversiondir;
+	char *newversiondircontent = NULL;
 
 	string_or_die(&newversiondir, "%s/%d", image_dir, version);
 
 	if ((access(newversiondir, F_OK | R_OK) != 0) && (version == 0)) {
 		char *latestpath = NULL;
 
-		string_or_die(&latestpath, "%s/latest.version", image_dir);
+		string_or_die(&latestpath, "%s/LAST_VER", image_dir);
 
 		printf("** %s does not exist... creating and populating\n", newversiondir);
 		if (mkdir(newversiondir, 0755) != 0) {
@@ -181,9 +183,11 @@ static void populate_dirs(int version)
 			}
 
 			string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group);
+			string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group);
 
 			/* Create the bundle directory(s) as needed */
-			if (access(newversiondir, F_OK | R_OK) != 0) {
+			if (access(newversiondir, F_OK | R_OK) != 0 &&
+			    access(newversiondircontent, F_OK | R_OK) != 0) {
 				printf("%s does not exist...creating\n", group);
 				if (mkdir(newversiondir, 0755) != 0) {
 					printf("Failed to create %s subdirectory\n", group);
@@ -192,6 +196,7 @@ static void populate_dirs(int version)
 		}
 	}
 	free(newversiondir);
+	free(newversiondircontent);
 }
 
 static int check_build_env(void)
@@ -212,6 +217,25 @@ static int check_build_env(void)
 	return 0;
 }
 
+static int check_group_file(void)
+{
+	int ret = -1;
+
+	// Ensure the os-core group is defined
+	while (1) {
+		char *group = next_group();
+		if (!group) {
+			break;
+		}
+		if (strcmp(group, "os-core") == 0) {
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -240,6 +264,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_globals();
 		return EXIT_FAILURE;
@@ -253,12 +282,6 @@ int main(int argc, char **argv)
 		goto exit;
 	}
 
-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		goto exit;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	if (!read_configuration_file(file_path)) {
 		printf("Failed to read %s configuration file!\n", state_dir);
@@ -270,8 +293,13 @@ int main(int argc, char **argv)
 	string_or_die(&file_path, "%s/groups.ini", state_dir);
 	read_group_file(file_path);
 	free(file_path);
+	ret = check_group_file();
+	if (ret != 0) {
+		printf("os-core bundle is not listed in groups.ini, this is required.\n");
+		goto exit;
+	}
 
-	read_current_version("latest.version");
+	read_current_version("LAST_VER");
 	printf("Last processed version is %i\n", current_version);
 
 	populate_dirs(newversion);
@@ -502,11 +530,10 @@ int main(int argc, char **argv)
 exit:
 	if (exit_status == EXIT_SUCCESS) {
 		write_cookiecrumbs_to_download_area(newversion);
-		//write_new_version("latest.version", newversion);
+		//write_new_version("LAST_VER", newversion);
 	}
 	release_configuration_data();
 	release_group_file();
-	signature_terminate();
 	g_list_free(manifests_last_versions_list);
 
 	close_log(newversion, exit_status);

src/delta.c

@@ -35,10 +35,9 @@
 #include "swupd.h"
 #include "xattrs.h"
 
-void __create_delta(struct file *file, int from_version)
+void __create_delta(struct file *file, int from_version, char *from_hash)
 {
-	char *original, *newfile, *outfile, *dotfile, *testnewfile;
-	char *conf, *param1, *param2;
+	char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf;
 	int ret;
 
 	if (file->is_link) {
@@ -62,9 +61,9 @@ void __create_delta(struct file *file, int from_version)
 
 	conf = config_output_dir();
 
-	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
-	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash);
+	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
+	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
+	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s-%s.testnewfile", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
 
 	LOG(file, "Making delta", "%s->%s", original, newfile);
 
@@ -98,7 +97,7 @@ void __create_delta(struct file *file, int from_version)
 		ret = 0;
 		goto out;
 	}
-	xattrs_copy(original, newfile);
+	xattrs_copy(original, testnewfile);
 
 	/* does xattrs have been correctly copied?*/
 	if (xattrs_compare(original, testnewfile) != 0) {
@@ -109,12 +108,8 @@ void __create_delta(struct file *file, int from_version)
 		goto out;
 	}
 
-	string_or_die(&param1, "%s", newfile);
-	string_or_die(&param2, "%s", testnewfile);
-	char *const sanitycheck[] = { "cmp", "-s", param1, param2, NULL };
+	char *const sanitycheck[] = { "cmp", "-s", newfile, testnewfile, NULL };
 	ret = system_argv(sanitycheck);
-	free(param1);
-	free(param2);
 	if (ret == -1 || ret == 2) {
 		printf("Sanity check system command failed %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);

src/fullfiles.c

@@ -72,7 +72,7 @@ static void create_fullfile(struct file *file)
 	string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
 	if (lstat(origin, &sbuf) < 0) {
 		/* no input file: means earlier phase of update creation failed */
-		LOG(NULL, "Failed to stat %s\n", origin);
+		LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno));
 		assert(0);
 	}
 
@@ -252,7 +252,7 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest)
 	manifest->files = g_list_sort(manifest->files, file_sort_hash);
 
 	list = g_list_first(manifest->files);
-	while (prev == NULL) {
+	while (prev == NULL && list != NULL) {
 		tmp = list->data;
 		list = g_list_next(list);
 
@@ -291,10 +291,11 @@ static void submit_fullfile_tasks(GList *files)
 	int ret;
 	int count = 0;
 	GError *err = NULL;
+	int numthreads = num_threads(3.0);
 
-	LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
+	LOG(NULL, "fullfile threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_fullfile_task, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN) * 3,
+				       numthreads,
 				       TRUE, NULL);
 
 	printf("Starting downloadable fullfiles data creation\n");

src/globals.c

@@ -34,7 +34,6 @@
 int newversion = -1;
 int minversion = 0;
 unsigned long long int format = 0;
-bool enable_signing = false;
 
 char *state_dir = NULL;
 char *packstage_dir = NULL;

src/groups.c

@@ -51,6 +51,13 @@ char *group_groups(char *group)
 	return g_key_file_get_value(groupfile, group, "groups", NULL);
 }
 
+char *group_status(char *group)
+{
+	assert(groupfile != NULL);
+
+	return g_key_file_get_value(groupfile, group, "status", NULL);
+}
+
 void read_group_file(char *filename)
 {
 	GError *error = NULL;

src/helpers.c

@@ -284,3 +284,28 @@ void check_root(void)
 		exit(EXIT_FAILURE);
 	}
 }
+
+int num_threads(float scaling)
+{
+	const char *var = getenv("SWUPD_NUM_THREADS");
+	int result = sysconf(_SC_NPROCESSORS_ONLN) * scaling;
+
+	if (var && *var) {
+		char *endptr;
+		long int value;
+
+		errno = 0;
+		value = strtol(var, &endptr, 0);
+
+		if ((errno != 0 && value == 0) || *endptr) {
+			LOG(NULL, "SWUPD_NUM_THREADS must be an integer", "%s", var);
+		} else if ((errno == ERANGE && (value == LONG_MAX || value == LONG_MIN)) ||
+			   value < 1 || value > INT_MAX) {
+			LOG(NULL, "SWUPD_NUM_THREADS out of range", "%s", var);
+		} else {
+			result = (int)value;
+		}
+	}
+
+	return result;
+}

src/log.c

@@ -33,7 +33,7 @@
 
 #include "swupd.h"
 
-static FILE *logfile;
+static FILE *logfile[2];
 
 static struct timeval start_time;
 
@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end)
 {
 	char *filename;
 	string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end);
-	logfile = fopen(filename, "w");
+	logfile[0] = fopen(filename, "w");
 	free(filename);
 	gettimeofday(&start_time, NULL);
 }
 void init_log_stdout(void)
 {
-	logfile = stdout;
+	logfile[1] = stdout;
 	gettimeofday(&start_time, NULL);
 }
 
@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 	char *logstring = NULL;
 	char filebuf[4096];
 	char filebuf2[4096];
+	int i;
 
-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}
 
@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
 		strcat(filebuf2, " ");
 	}
 
-	fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
-		(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+	for (i = 0; i < 2; i++) {
+		if (logfile[i]) {
+			fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
+				(int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
+			fflush(logfile[i]);
+		}
+	}
 
 	free(logstring);
 	free(buf);
-	fflush(logfile);
 }
 
 void close_log(int version, int exit_status)
@@ -133,7 +138,7 @@ void close_log(int version, int exit_status)
 	int t_sec;
 	int t_msec;
 
-	if (!logfile) {
+	if (!logfile[0] && !logfile[1]) {
 		return;
 	}
 
@@ -159,6 +164,8 @@ void close_log(int version, int exit_status)
 		printf("Update build failed for version %i\n", version);
 	}
 
-	fclose(logfile);
-	logfile = NULL;
+	if (logfile[0]) {
+		fclose(logfile[0]);
+		logfile[0] = NULL;
+	}
 }

src/make_fullfiles.c

@@ -23,6 +23,7 @@
 #define _GNU_SOURCE
 #include <assert.h>
 #include <getopt.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,6 +33,7 @@
 
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
 	{ 0, 0, 0, 0 }
 };
@@ -42,6 +44,7 @@ static void usage(const char *name)
 	printf("   %s <version>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
 	printf("\n");
 }
@@ -56,6 +59,9 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument '%s'\n\n", optarg);
@@ -88,6 +94,11 @@ int main(int argc, char **argv)
 	/* keep valgrind working well */
 	setenv("G_SLICE", "always-malloc", 0);
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;

src/make_packs.c

@@ -27,6 +27,7 @@
 #include <getopt.h>
 #include <getopt.h>
 #include <glib.h>
+#include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -45,8 +46,8 @@ static void banner(void)
 
 static const struct option prog_opts[] = {
 	{ "help", no_argument, 0, 'h' },
+	{ "log-stdout", no_argument, 0, 'l' },
 	{ "statedir", required_argument, 0, 'S' },
-	{ "signcontent", no_argument, 0, 's' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -56,8 +57,8 @@ static void usage(const char *name)
 	printf("   %s <start version> <latest version> <bundle>\n\n", name);
 	printf("Help options:\n");
 	printf("   -h, --help              Show help options\n");
+	printf("   -l, --log-stdout        Write log messages also to stdout\n");
 	printf("   -S, --statedir          Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
-	printf("   -s, --signcontent       Enables cryptographic signing of update content\n");
 	printf("\n");
 }
 
@@ -71,15 +72,15 @@ static bool parse_options(int argc, char **argv)
 		case 'h':
 			usage(argv[0]);
 			return false;
+		case 'l':
+			init_log_stdout();
+			break;
 		case 'S':
 			if (!optarg || !set_state_dir(optarg)) {
 				printf("Invalid --statedir argument ''%s'\n\n", optarg);
 				return false;
 			}
 			break;
-		case 's':
-			enable_signing = true;
-			break;
 		}
 	}
 
@@ -101,6 +102,11 @@ int main(int argc, char **argv)
 	int exit_status = EXIT_FAILURE;
 	char *file_path = NULL;
 
+	if (!setlocale(LC_ALL, "")) {
+		fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
+		return EXIT_FAILURE;
+	}
+
 	if (!parse_options(argc, argv)) {
 		free_state_globals();
 		return EXIT_FAILURE;
@@ -114,12 +120,6 @@ int main(int argc, char **argv)
 	banner();
 	check_root();
 
-	/* Initilize the crypto signature module */
-	if (!signature_initialize()) {
-		printf("Can't initialize the crypto signature module!\n");
-		return exit_status;
-	}
-
 	string_or_die(&file_path, "%s/server.ini", state_dir);
 	read_configuration_file(file_path);
 	free(file_path);
@@ -152,8 +152,6 @@ int main(int argc, char **argv)
 		exit_status = EXIT_SUCCESS;
 	}
 
-	signature_terminate();
-
 	printf("Pack creation %s (pack-%s %i to %li)\n",
 	       exit_status == EXIT_SUCCESS ? "complete" : "failed",
 	       module, start_version, end_version);

src/manifest.c

@@ -64,7 +64,7 @@ int file_sort_version(gconstpointer a, gconstpointer b)
 		return 1;
 	}
 
-	return strcmp(B->filename, A->filename);
+	return strcmp(A->filename, B->filename);
 }
 
 int file_sort_filename(gconstpointer a, gconstpointer b)
@@ -695,29 +695,6 @@ static void compute_content_size(struct manifest *manifest)
 	}
 }
 
-/* Returns 0 == success, -1 == failure */
-static int write_manifest_signature(struct manifest *manifest, const char *suffix)
-{
-	char *conf = config_output_dir();
-	char *filename = NULL;
-	int ret = -1;
-
-	if (conf == NULL) {
-		assert(0);
-	}
-	string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version,
-		      manifest->component, suffix);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	free(conf);
-	return ret;
-}
-
 /* Returns 0 == success, -1 == failure */
 static int write_manifest_plain(struct manifest *manifest)
 {
@@ -728,6 +705,7 @@ static int write_manifest_plain(struct manifest *manifest)
 	char *base = NULL, *dir;
 	char *conf = config_output_dir();
 	char *filename = NULL;
+	char *status = NULL;
 	char *submanifest_filename = NULL;
 	char *manifest_tempdir = NULL;
 	char *tempmanifest = NULL;
@@ -762,6 +740,10 @@ static int write_manifest_plain(struct manifest *manifest)
 	compute_content_size(manifest);
 	fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
 	includes = manifest->includes;
+	status = group_status(manifest->component);
+	if (status) {
+		fprintf(out, "status:\t%s\n", status);
+	}
 	while (includes) {
 		struct manifest *sub = includes->data;
 		includes = g_list_next(includes);
@@ -846,7 +828,7 @@ exit:
 static int write_manifest_tar(struct manifest *manifest)
 {
 	char *conf = config_output_dir();
-	char *directory, *manifesttar, *manifestcomp, *manifestsigned;
+	char *directory, *manifesttar, *manifestcomp;
 	int ret = 0;
 
 	if (conf == NULL) {
@@ -856,19 +838,11 @@ static int write_manifest_tar(struct manifest *manifest)
 	string_or_die(&directory, "--directory=%s/%i", conf, manifest->version);
 	string_or_die(&manifesttar, "%s/%i/Manifest.%s.tar", conf, manifest->version, manifest->component);
 	string_or_die(&manifestcomp, "Manifest.%s", manifest->component);
-	string_or_die(&manifestsigned, "Manifest.%s.signed", manifest->component);
 
 	/* now, tar the thing up for efficient full file download */
-	/* and put the signature of the plain manifest into the archive, too */
-	if (enable_signing) {
-		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-					 manifesttar, manifestcomp, manifestsigned, NULL };
-		ret = system_argv(tarcmd);
-	} else {
-		char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
-					 manifesttar, manifestcomp, NULL };
-		ret = system_argv(tarcmd);
-	}
+	char *const tarcmd[] = { TAR_COMMAND, directory, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf",
+				 manifesttar, manifestcomp, NULL };
+	ret = system_argv(tarcmd);
 	if (ret) {
 		fprintf(stderr, "Creation of Manifest.tar failed\n");
 	}
@@ -876,7 +850,6 @@ static int write_manifest_tar(struct manifest *manifest)
 	free(directory);
 	free(manifesttar);
 	free(manifestcomp);
-	free(manifestsigned);
 	free(conf);
 	return ret;
 }
@@ -906,9 +879,7 @@ bool compute_hash_with_xattrs(const char *filename)
 int write_manifest(struct manifest *manifest)
 {
 	if (write_manifest_plain(manifest) == 0 &&
-	    write_manifest_signature(manifest, "") == 0 &&
-	    write_manifest_tar(manifest) == 0 &&
-	    write_manifest_signature(manifest, ".tar") == 0) {
+	    write_manifest_tar(manifest) == 0) {
 		return 0;
 	}
 	return -1;
@@ -1110,9 +1081,6 @@ void create_manifest_delta(int oldversion, int newversion, char *module)
 			}
 			LOG(NULL, "Failed to rename", "");
 		}
-		if (!signature_sign(outfile)) {
-			fprintf(stderr, "Creating signature for '%s' failed\n", outfile);
-		}
 	} else {
 		sleep(1); /* we raced. whatever. sleep for a bit to get the other guy to make progress */
 	}

src/pack.c

@@ -116,7 +116,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
 		 * time on the client...
 		 */
 		string_or_die(&param, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version);
-		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
+		char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
 		if (system_argv(tarcmd) == 0) {
 			unlink(path);
 		}
@@ -254,8 +254,8 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 			continue;
 		}
 
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
 
 		ret = stat(from, &stat_delta);
 		if (ret && !find_file_in_list(files, file)) {
@@ -274,7 +274,7 @@ static void create_delta(gpointer data, __unused__ gpointer user_data)
 
 	/* if the file was not found in the from version, skip delta creation */
 	if (file->peer) {
-		__create_delta(file, file->peer->last_change);
+		__create_delta(file, file->peer->last_change, file->peer->hash);
 	}
 }
 
@@ -285,10 +285,11 @@ static void make_pack_deltas(GList *files)
 	struct file *file;
 	int ret;
 	GError *err = NULL;
+	int numthreads = num_threads(1.0);
 
-	LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
+	LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
 	threadpool = g_thread_pool_new(create_delta, NULL,
-				       sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
+				       numthreads, FALSE, NULL);
 
 	item = g_list_first(files);
 	while (item) {
@@ -307,24 +308,6 @@ static void make_pack_deltas(GList *files)
 	g_thread_pool_free(threadpool, FALSE, TRUE);
 }
 
-/* Returns 0 == success, -1 == failure */
-static int write_pack_signature(struct packdata *pack)
-{
-	char *filename = NULL;
-	int ret = -1;
-
-	string_or_die(&filename, "%s/%i/pack-%s-from-%i.tar",
-		      staging_dir, pack->to, pack->module, pack->from);
-	if (!signature_sign(filename)) {
-		fprintf(stderr, "Creating signature for '%s' failed\n", filename);
-		goto exit;
-	}
-	ret = 0;
-exit:
-	free(filename);
-	return ret;
-}
-
 /* Returns 0 == success, other == failure */
 static int make_final_pack(struct packdata *pack)
 {
@@ -353,11 +336,11 @@ static int make_final_pack(struct packdata *pack)
 
 		/* for each file changed since <X> */
 		/* locate delta, check if the diff it's from is >= <X> */
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
-		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s", packstage_dir,
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
+		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s-%s", packstage_dir,
 			      pack->module, pack->from, pack->to, file->peer->last_change,
-			      file->last_change, file->hash);
+			      file->last_change, file->peer->hash, file->hash);
 		string_or_die(&tarfrom, "%s/%i/files/%s.tar", staging_dir,
 			      file->last_change, file->hash);
 		string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
@@ -507,13 +490,6 @@ static int make_final_pack(struct packdata *pack)
 	if ((ret != 0) && (ret != 1)) {
 		fprintf(stderr, "Unexpected return value (%d) creating tar of pack %s from %i to %i\n",
 			ret, pack->module, pack->from, pack->to);
-	} else {
-		/* Write the signature file */
-		ret = write_pack_signature(pack);
-		if (ret != 0) {
-			fprintf(stderr, "Failure creating signature of pack %s from %i to %i\n",
-				pack->module, pack->from, pack->to);
-		}
 	}
 
 	/* and clean up */

src/signature.c

@@ -1,173 +0,0 @@
-/*
- *   Software Updater - server side
- *
- *      Copyright © 2012-2016 Intel Corporation.
- *
- *   This program is free software: you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, version 2 or later of the License.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *   Authors:
- *         Tom Keel <thomas.keel@intel.com>
- *
- */
-
-#define _GNU_SOURCE
-#include <err.h>
-#include <errno.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "swupd.h"
-
-static char *make_filename(const char *, const char *, const char *);
-
-static char *leaf_key = NULL;
-static char *leaf_cert = NULL;
-static char *ca_chain_cert = NULL;
-static char *passphrase = NULL;
-
-static bool initialized = false;
-
-/*
- * Initialize this module.
- * @return true <=> success
- */
-bool signature_initialize(void)
-{
-	if (!enable_signing) {
-		return true;
-	}
-
-	char *cdir;
-	char *pphr;
-	struct stat s;
-
-	if (initialized) {
-		return true;
-	}
-	cdir = getenv("SWUPD_CERTS_DIR");
-	if (cdir == NULL || cdir[0] == '\0') {
-		printf("No certificates directory specified\n");
-		goto err;
-	}
-	if (stat(cdir, &s)) {
-		printf("Can't stat certificates directory '%s' (%s)\n", cdir,
-		       strerror(errno));
-		goto err;
-	}
-	leaf_key = make_filename(cdir, "LEAF_KEY", "leaf key");
-	if (leaf_key == NULL) {
-		goto err;
-	}
-	leaf_cert = make_filename(cdir, "LEAF_CERT", "leaf certificate");
-	if (leaf_cert == NULL) {
-		goto err;
-	}
-	ca_chain_cert = make_filename(cdir, "CA_CHAIN_CERT", "CA chain certificate");
-	if (ca_chain_cert == NULL) {
-		goto err;
-	}
-	pphr = getenv("PASSPHRASE");
-	if (pphr == NULL || (passphrase = strdup(pphr)) == NULL) {
-		goto err;
-	}
-	if (stat(passphrase, &s)) {
-		printf("Can't stat '%s' (%s)\n", passphrase,
-		       strerror(errno));
-		goto err;
-	}
-	initialized = true;
-	return true;
-err:
-	signature_terminate();
-	return false;
-}
-
-/* Make filename from dir name and env variable containing basename */
-static char *make_filename(const char *dir, const char *env, const char *desc)
-{
-	char *fn = getenv(env);
-	char *result = NULL;
-	struct stat s;
-
-	if (fn == NULL || fn[0] == '\0') {
-		printf("No %s file specified\n", desc);
-		return NULL;
-	}
-	string_or_die(&result, "%s/%s", dir, fn);
-	if (stat(result, &s)) {
-		printf("Can't stat %s '%s' (%s)\n", desc, result, strerror(errno));
-		free(result);
-		return NULL;
-	}
-	return result;
-}
-
-/*
- * Terminate this module, free resources.
- */
-void signature_terminate(void)
-{
-	if (!enable_signing) {
-		return;
-	}
-
-	free(leaf_key);
-	free(leaf_cert);
-	free(ca_chain_cert);
-	free(passphrase);
-
-	leaf_key = NULL;
-	leaf_cert = NULL;
-	ca_chain_cert = NULL;
-	passphrase = NULL;
-
-	initialized = false;
-}
-
-/*
- * Write the signature file corresponding to the given data file.
- * The name of the signature file is the name of the data file with suffix
- * ".signed" appended.
- */
-bool signature_sign(const char *filename)
-{
-	char *param1, *param2;
-	int status;
-
-	if (!enable_signing) {
-		return true;
-	}
-
-	if (!initialized) {
-		return false;
-	}
-	string_or_die(&param1, "%s.signed", filename);
-	string_or_die(&param2, "file:%s", passphrase);
-	char *const opensslcmd[] = { "openssl", "smime", "-sign", "-in", (char *)filename, "-binary",
-				     "-out", param1, "-outform", " PEM", "-md", "sha256", "-inkey",
-				     leaf_key, "-signer", leaf_cert, "-certfile", ca_chain_cert,
-				     "-passin", param2, NULL };
-	status = system_argv(opensslcmd);
-	if (status != 0) {
-		printf("Bad status %d from signing command\n", status);
-	}
-	free(param1);
-	free(param2);
-
-	return status == 0;
-}

test/functional/full-run/test.bats

@@ -39,7 +39,7 @@ setup() {
   [[ 1 -eq $(grep '/usr/share/clear$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
-  [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+  [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/fullfiles/test.bats

@@ -14,6 +14,7 @@ load "../swupdlib"
 }
 
 @test "make_fullfiles root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $MAKE_FULLFILES foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]

test/functional/pack/test.bats

@@ -20,6 +20,7 @@ load "../swupdlib"
 }
 
 @test "make_pack root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $MAKE_PACK foo bar foo
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]

test/functional/state-file/test.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+# common functions
+load "../swupdlib"
+
+setup() {
+  clean_test_dir
+  init_test_dir
+
+  init_server_ini
+  set_latest_ver 0
+  init_groups_ini os-core
+
+  set_os_release 10 os-core
+  track_bundle 10 os-core
+
+  gen_file_plain 10 os-core "/var/lib/test"
+}
+
+@test "state file marked in manifest" {
+  sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3
+
+  # a file and dir installed in /var should be marked state
+  grep '^D\.s\..*/var/lib$' $DIR/www/10/Manifest.os-core
+  grep '^F\.s\..*/var/lib/test$' $DIR/www/10/Manifest.os-core
+}
+
+# vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/swupdlib.bash

@@ -27,7 +27,7 @@ init_server_ini() {
 }
 
 set_latest_ver() {
-  echo "$1" > $DIR/image/latest.version
+  echo "$1" > $DIR/image/LAST_VER
 }
 
 init_groups_ini() {
@@ -35,6 +35,7 @@ init_groups_ini() {
     cat >> $DIR/groups.ini << EOF
 [$bundle]
 group=$bundle
+status=ACTIVE
 EOF
   done
 }
@@ -57,9 +58,9 @@ gen_includes_file() {
   local bundle=$1
   local ver=$2
   local includes="${@:3}"
-  mkdir -p $DIR/www/$ver/noship
+  mkdir -p $DIR/image/$ver/noship
   for b in $includes; do
-    cat >> $DIR/www/$ver/noship/"$bundle"-includes << EOF
+    cat >> $DIR/image/$ver/noship/"$bundle"-includes << EOF
 $b
 EOF
   done
@@ -90,7 +91,7 @@ gen_file_plain() {
   local name="$3"
 
   # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
+  mkdir -p $DIR/image/$ver/$bundle/$(dirname "$name")
   echo "$name" > $DIR/image/$ver/$bundle/"$name"
 }
 
@@ -100,7 +101,7 @@ gen_file_plain_change() {
   local name="$3"
 
   # Add plain text file into a bundle
-  mkdir -p $DIR/image/$ver/$bundle
+  mkdir -p $DIR/image/$ver/$bundle/$(dirname "$name")
   echo "$ver $name" > $DIR/image/$ver/$bundle/"$name"
 }
 

test/functional/update/test.bats

@@ -15,6 +15,7 @@ load "../swupdlib"
 }
 
 @test "create_update root priv check" {
+  [ $EUID -eq 0 ] && skip "test can only be run as non-root"
   run $CREATE_UPDATE -F 3 -o 10
   [ "$status" -eq 1 ]
   [[ "$output" =~ "not being run as root.. exiting" ]]