Release v3.3.0

This release contains various changes to fix segfaults and memory misuse,
simplifying logic when iterating directories and populating file structs
from manifests, and changing the secondary sort to be lexographically sorted
filenames for version-sorted manifests.

Signed-off-by: Tudor Marcu <tudor.marcu@intel.com>

Makefile.am

@@ -37,6 +37,7 @@ swupd_make_pack_SOURCES = \
 	src/config.c \
 	src/delta.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_packs.c \
@@ -53,6 +54,7 @@ swupd_make_fullfiles_SOURCES = \
 	src/delta.c \
 	src/fullfiles.c \
 	src/globals.c \
+	src/groups.c \
 	src/helpers.c \
 	src/log.c \
 	src/make_fullfiles.c \

basic_creator.sh

@@ -15,7 +15,7 @@ error() {
 # eg: all of openstack, pnp, bat, cloud stuff, non-basic scripting language bundles
 ${SWUPDREPO}/mk_groups_ini.sh
 
-PREVREL=`cat ${UPDATEDIR}/image/latest.version`
+PREVREL=`cat ${UPDATEDIR}/image/LAST_VER`
 
 export SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR:-"/root/swupd-certs"}
 export LEAF_KEY="leaf.key.pem"
@@ -46,7 +46,7 @@ for job in $(jobs -p); do
 done
 
 # expose the new build to staging / testing
-echo ${VER} > ${UPDATEDIR}/image/latest.version
+echo ${VER} > ${UPDATEDIR}/image/LAST_VER
 
 STAGING_FILE="${UPDATEDIR}/www/version/formatstaging/latest"
 if [ ! -f "${STAGING_FILE}" ]; then

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.66])
-AC_INIT(swupd-server, 3.2.4, timothy.c.pepper@linux.intel.com)
+AC_INIT(swupd-server, 3.3.0, timothy.c.pepper@linux.intel.com)
 AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects])
 AM_SILENT_RULES([yes])
 AC_PROG_CC

include/swupd.h

@@ -36,6 +36,8 @@
 #define TAR_PERM_ATTR_ARGS_STRLIST TAR_XATTR_ARGS_STRLIST "--preserve-permissions"
 #endif
 
+#define SWUPD_HASH_DIRNAME "DIRECTORY"
+
 #if SWUPD_WITH_STATELESS
 #define OS_IS_STATELESS 1
 #else
@@ -219,6 +221,7 @@ extern void read_group_file(char *filename);
 extern void release_group_file(void);
 extern char *group_groups(char *group);
 extern char *group_packages(char *group);
+extern char *group_status(char *group);
 extern char *next_group(void);
 
 extern void apply_heuristics(struct manifest *manifest);
@@ -242,7 +245,7 @@ extern void type_change_detection(struct manifest *manifest);
 
 extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
 extern void link_renames(GList *newfiles, struct manifest *from_manifest);
-extern void __create_delta(struct file *file, int from_version);
+extern void __create_delta(struct file *file, int from_version, char *from_hash);
 
 extern void account_delta_hit(void);
 extern void account_delta_miss(void);

mk_groups_ini.sh

@@ -18,8 +18,8 @@ if [ ! -f "$SWUPD_SERVER_INI" ]; then
 	sed -i "s|/var/lib/update|$UPDATEDIR|" $SWUPD_SERVER_INI
 fi
 
-if [ ! -f "$UPDATEDIR/image/latest.version" ]; then
+if [ ! -f "$UPDATEDIR/image/LAST_VER" ]; then
-	echo "0" > $UPDATEDIR/image/latest.version
+	echo "0" > $UPDATEDIR/image/LAST_VER
 fi
 
 echo "rebuilding $SWUPD_GROUPS_INI based on $BUNDLEREPO"
@@ -27,7 +27,11 @@ rm -f $SWUPD_GROUPS_INI
 
 for bundle in $(ls $BUNDLEREPO/bundles)
 do
+	status=$(awk -F: '/^# .STATUS/ {print $2}' $BUNDLEREPO/$bundle | tr -cd '[[:alnum:]]')
 	echo "[$bundle]" >> $SWUPD_GROUPS_INI
 	echo "group=$bundle" >> $SWUPD_GROUPS_INI
+	if [ -n "$status" ]; then
+		echo "status=$status" >> $SWUPD_GROUPS_INI
+	fi
 	echo "" >> $SWUPD_GROUPS_INI
 done

src/analyze_fs.c

@@ -222,7 +222,7 @@ int compute_hash(struct file *file, char *filename)
 		hmac_sha256_for_string(file->hash,
 				       (const unsigned char *)key,
 				       key_len,
-				       file->filename); //file->filename not filename
+				       SWUPD_HASH_DIRNAME); // Make independent of dirname
 		return 0;
 	}
 
@@ -353,7 +353,7 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
 		populate_file_struct(file, fullname);
 		free(fullname);
 
-		if (entry->d_type == DT_DIR) {
+		if (file->is_dir) {
 			iterate_directory(manifest, pathprefix, file->filename, do_hash);
 		}
 
@@ -418,7 +418,7 @@ GList *get_sub_manifest_includes(char *component, int version)
 	char *included;
 	char line[8192];
 
-	conf = config_output_dir();
+	conf = config_image_base();
 	if (conf == NULL) {
 		assert(0);
 	}

src/create_update.c

@@ -146,7 +146,7 @@ static void populate_dirs(int version)
 	if ((access(newversiondir, F_OK | R_OK) != 0) && (version == 0)) {
 		char *latestpath = NULL;
 
-		string_or_die(&latestpath, "%s/latest.version", image_dir);
+		string_or_die(&latestpath, "%s/LAST_VER", image_dir);
 
 		printf("** %s does not exist... creating and populating\n", newversiondir);
 		if (mkdir(newversiondir, 0755) != 0) {
@@ -212,6 +212,25 @@ static int check_build_env(void)
 	return 0;
 }
 
+static int check_group_file(void)
+{
+	int ret = -1;
+
+	// Ensure the os-core group is defined
+	while (1) {
+		char *group = next_group();
+		if (!group) {
+			break;
+		}
+		if (strcmp(group, "os-core") == 0) {
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
 int main(int argc, char **argv)
 {
 	struct manifest *new_core = NULL;
@@ -270,8 +289,13 @@ int main(int argc, char **argv)
 	string_or_die(&file_path, "%s/groups.ini", state_dir);
 	read_group_file(file_path);
 	free(file_path);
+	ret = check_group_file();
+	if (ret != 0) {
+		printf("os-core bundle is not listed in groups.ini, this is required.\n");
+		goto exit;
+	}
 
-	read_current_version("latest.version");
+	read_current_version("LAST_VER");
 	printf("Last processed version is %i\n", current_version);
 
 	populate_dirs(newversion);
@@ -502,7 +526,7 @@ int main(int argc, char **argv)
 exit:
 	if (exit_status == EXIT_SUCCESS) {
 		write_cookiecrumbs_to_download_area(newversion);
-		//write_new_version("latest.version", newversion);
+		//write_new_version("LAST_VER", newversion);
 	}
 	release_configuration_data();
 	release_group_file();

src/delta.c

@@ -35,10 +35,9 @@
 #include "swupd.h"
 #include "xattrs.h"
 
-void __create_delta(struct file *file, int from_version)
+void __create_delta(struct file *file, int from_version, char *from_hash)
 {
-	char *original, *newfile, *outfile, *dotfile, *testnewfile;
+	char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf;
-	char *conf, *param1, *param2;
 	int ret;
 
 	if (file->is_link) {
@@ -62,9 +61,9 @@ void __create_delta(struct file *file, int from_version)
 
 	conf = config_output_dir();
 
-	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
+	string_or_die(&outfile, "%s/%i/delta/%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
-	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash);
+	string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s-%s", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
-	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash);
+	string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s-%s.testnewfile", conf, file->last_change, from_version, file->last_change, from_hash, file->hash);
 
 	LOG(file, "Making delta", "%s->%s", original, newfile);
 
@@ -98,7 +97,7 @@ void __create_delta(struct file *file, int from_version)
 		ret = 0;
 		goto out;
 	}
-	xattrs_copy(original, newfile);
+	xattrs_copy(original, testnewfile);
 
 	/* does xattrs have been correctly copied?*/
 	if (xattrs_compare(original, testnewfile) != 0) {
@@ -109,12 +108,8 @@ void __create_delta(struct file *file, int from_version)
 		goto out;
 	}
 
-	string_or_die(&param1, "%s", newfile);
+	char *const sanitycheck[] = { "cmp", "-s", newfile, testnewfile, NULL };
-	string_or_die(&param2, "%s", testnewfile);
-	char *const sanitycheck[] = { "cmp", "-s", param1, param2, NULL };
 	ret = system_argv(sanitycheck);
-	free(param1);
-	free(param2);
 	if (ret == -1 || ret == 2) {
 		printf("Sanity check system command failed %i. \n", ret);
 		printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile);

src/fullfiles.c

@@ -252,7 +252,7 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest)
 	manifest->files = g_list_sort(manifest->files, file_sort_hash);
 
 	list = g_list_first(manifest->files);
-	while (prev == NULL) {
+	while (prev == NULL && list != NULL) {
 		tmp = list->data;
 		list = g_list_next(list);
 

src/groups.c

@@ -51,6 +51,13 @@ char *group_groups(char *group)
 	return g_key_file_get_value(groupfile, group, "groups", NULL);
 }
 
+char *group_status(char *group)
+{
+	assert(groupfile != NULL);
+
+	return g_key_file_get_value(groupfile, group, "status", NULL);
+}
+
 void read_group_file(char *filename)
 {
 	GError *error = NULL;

src/heuristics.c

@@ -44,15 +44,15 @@ static void runtime_state_heuristics(struct file *file)
 {
 	/* these are shipped directories that are not themselves state,
 	 * rather only their contents are state */
-	if ((strcmp(file->filename, "/usr/src/debug") == 0) ||
+	if ((strncmp(file->filename, "/usr/src/debug", 14) == 0) ||
-	    (strcmp(file->filename, "/dev") == 0) ||
+	    (strncmp(file->filename, "/dev", 4) == 0) ||
-	    (strcmp(file->filename, "/home") == 0) ||
+	    (strncmp(file->filename, "/home", 5) == 0) ||
-	    (strcmp(file->filename, "/proc") == 0) ||
+	    (strncmp(file->filename, "/proc", 5) == 0) ||
-	    (strcmp(file->filename, "/root") == 0) ||
+	    (strncmp(file->filename, "/root", 5) == 0) ||
-	    (strcmp(file->filename, "/run") == 0) ||
+	    (strncmp(file->filename, "/run", 4) == 0) ||
-	    (strcmp(file->filename, "/sys") == 0) ||
+	    (strncmp(file->filename, "/sys", 4) == 0) ||
-	    (strcmp(file->filename, "/tmp") == 0) ||
+	    (strncmp(file->filename, "/tmp", 4) == 0) ||
-	    (strcmp(file->filename, "/var") == 0)) {
+	    (strncmp(file->filename, "/var", 4) == 0)) {
 		return;
 	}
 

src/manifest.c

@@ -728,6 +728,7 @@ static int write_manifest_plain(struct manifest *manifest)
 	char *base = NULL, *dir;
 	char *conf = config_output_dir();
 	char *filename = NULL;
+	char *status = NULL;
 	char *submanifest_filename = NULL;
 	char *manifest_tempdir = NULL;
 	char *tempmanifest = NULL;
@@ -762,6 +763,10 @@ static int write_manifest_plain(struct manifest *manifest)
 	compute_content_size(manifest);
 	fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize);
 	includes = manifest->includes;
+	status = group_status(manifest->component);
+	if (status) {
+		fprintf(out, "status:\t%s\n", status);
+	}
 	while (includes) {
 		struct manifest *sub = includes->data;
 		includes = g_list_next(includes);

src/pack.c

@@ -254,8 +254,8 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 			continue;
 		}
 
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
 
 		ret = stat(from, &stat_delta);
 		if (ret && !find_file_in_list(files, file)) {
@@ -274,7 +274,7 @@ static void create_delta(gpointer data, __unused__ gpointer user_data)
 
 	/* if the file was not found in the from version, skip delta creation */
 	if (file->peer) {
-		__create_delta(file, file->peer->last_change);
+		__create_delta(file, file->peer->last_change, file->peer->hash);
 	}
 }
 
@@ -353,11 +353,11 @@ static int make_final_pack(struct packdata *pack)
 
 		/* for each file changed since <X> */
 		/* locate delta, check if the diff it's from is >= <X> */
-		string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change,
+		string_or_die(&from, "%s/%i/delta/%i-%i-%s-%s", staging_dir, file->last_change,
-			      file->peer->last_change, file->last_change, file->hash);
+			      file->peer->last_change, file->last_change, file->peer->hash, file->hash);
-		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s", packstage_dir,
+		string_or_die(&to, "%s/%s/%i_to_%i/delta/%i-%i-%s-%s", packstage_dir,
 			      pack->module, pack->from, pack->to, file->peer->last_change,
-			      file->last_change, file->hash);
+			      file->last_change, file->peer->hash, file->hash);
 		string_or_die(&tarfrom, "%s/%i/files/%s.tar", staging_dir,
 			      file->last_change, file->hash);
 		string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,

test/functional/full-run/test.bats

@@ -39,7 +39,7 @@ setup() {
   [[ 1 -eq $(grep '/usr/share/clear$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 1 -eq $(grep '/usr/share/clear/bundles$' $DIR/www/10/Manifest.os-core | wc -l) ]]
   [[ 4 -eq $(tar -tf $DIR/www/10/pack-test-bundle-from-0.tar | wc -l) ]]
-  [[ 9 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
+  [[ 5 -eq $(tar -tf $DIR/www/10/pack-os-core-from-0.tar | wc -l) ]]
 }
 
 # vi: ft=sh ts=8 sw=2 sts=2 et tw=80

test/functional/swupdlib.bash

@@ -27,7 +27,7 @@ init_server_ini() {
 }
 
 set_latest_ver() {
-  echo "$1" > $DIR/image/latest.version
+  echo "$1" > $DIR/image/LAST_VER
 }
 
 init_groups_ini() {
@@ -35,6 +35,7 @@ init_groups_ini() {
     cat >> $DIR/groups.ini << EOF
 [$bundle]
 group=$bundle
+status=ACTIVE
 EOF
   done
 }
@@ -57,9 +58,9 @@ gen_includes_file() {
   local bundle=$1
   local ver=$2
   local includes="${@:3}"
-  mkdir -p $DIR/www/$ver/noship
+  mkdir -p $DIR/image/$ver/noship
   for b in $includes; do
-    cat >> $DIR/www/$ver/noship/"$bundle"-includes << EOF
+    cat >> $DIR/image/$ver/noship/"$bundle"-includes << EOF
 $b
 EOF
   done