Update to support new state if system is on a mix

This patch adds the logic required to handle files and manifests
differently if it is detected that the system is on a mix, or the
content being provided is local mix content.

Signed-off-by: Tudor Marcu <tudor.marcu@intel.com>

include/signature.h

@@ -21,6 +21,6 @@ void terminate_signature(void);
  * @param data_url - the URL from which the data came
  * @param data_filename - the file containing the data
  */
-bool download_and_verify_signature(const char *data_url, const char *data_filename);
+bool download_and_verify_signature(const char *data_url, const char *data_filename, int version, bool mix_exists);
 
 #endif /* SIGNATURE_H_ */

include/swupd.h

@@ -21,6 +21,8 @@ extern "C" {
 #define SWUPD_VERSION_IS_DEVEL(v) (((v) % SWUPD_VERSION_INCR) == 8)
 #define SWUPD_VERSION_IS_RESVD(v) (((v) % SWUPD_VERSION_INCR) == 9)
 
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+
 #ifndef LINE_MAX
 #define LINE_MAX _POSIX2_LINE_MAX
 #endif
@@ -33,6 +35,9 @@ extern "C" {
 #define MAX_TRIES 3
 
 #define SWUPD_HASH_DIRNAME "DIRECTORY"
+#define MIX_DIR "/usr/share/mix/"
+#define MIX_STATE_DIR MIX_DIR "update/www/"
+#define MIX_BUNDLES_DIR MIX_STATE_DIR "mix-bundles/"
 
 struct sub {
 	/* name of bundle/component/subscription */
@@ -57,6 +62,7 @@ struct manifest {
 	struct list *submanifests; /* struct manifest for subscribed manifests */
 	struct list *includes;     /* manifest names included in manifest */
 	char *component;
+	unsigned int is_mix : 1;
 };
 
 struct version_container {
@@ -111,6 +117,7 @@ struct file {
 	unsigned int is_boot : 1;
 	unsigned int is_rename : 1;
 	unsigned int is_orphan : 1;
+	unsigned int is_mix : 1;
 	unsigned int do_not_update : 1;
 
 	struct file *peer;      /* same file in another manifest */
@@ -192,8 +199,9 @@ extern void apply_heuristics(struct file *file);
 
 extern int file_sort_filename(const void *a, const void *b);
 extern int file_sort_filename_reverse(const void *a, const void *b);
-extern struct manifest *load_mom(int version, bool latest);
+extern struct manifest *load_mom(int version, bool latest, bool mix_exists);
 extern struct manifest *load_manifest(int current, int version, struct file *file, struct manifest *mom, bool header_only);
+extern struct manifest *load_manifest_full(int version, bool mix);
 extern struct list *create_update_list(struct manifest *current, struct manifest *server);
 extern void link_manifests(struct manifest *m1, struct manifest *m2);
 extern void link_submanifests(struct manifest *m1, struct manifest *m2, struct list *subs1, struct list *subs2, bool server);
@@ -214,7 +222,7 @@ static inline int bsearch_helper(const void *A, const void *B)
 
 static inline int bsearch_file_helper(const void *A, const void *B)
 {
-	struct file *key = (struct file *) A;
+	struct file *key = (struct file *)A;
 	struct file *elem = (*(struct file **)B);
 	return strcmp(key->filename, elem->filename);
 }
@@ -262,7 +270,7 @@ static inline void account_delta_miss(void)
 
 extern void print_statistics(int version1, int version2);
 
-extern int download_subscribed_packs(struct list *subs, bool required);
+extern int download_subscribed_packs(struct list *subs, struct manifest *mom, bool required);
 
 extern void try_delta(struct file *file);
 extern void full_download(struct file *file);
@@ -292,8 +300,7 @@ extern int swupd_curl_get_file(const char *url, char *filename, struct file *fil
 extern CURLcode swupd_curl_set_basic_options(CURL *curl, const char *url);
 
 extern void free_subscriptions(struct list **subs);
-extern void read_subscriptions(void);
-extern void read_subscriptions_alt(struct list **subs);
+extern void read_subscriptions_alt(struct list **subs, bool is_mix);
 extern int component_subscribed(struct list *subs, char *component);
 extern void set_subscription_versions(struct manifest *latest, struct manifest *current, struct list **subs);
 
@@ -381,7 +388,11 @@ extern void telemetry(telem_prio_t level, const char *class, const char *fmt, ..
 
 extern struct file **manifest_files_to_array(struct manifest *manifest);
 extern void print_manifest_array(struct file **array, int filecount);
-int enforce_compliant_manifest(struct file **a, struct file **b, int size);
+extern int enforce_compliant_manifest(struct file **a, struct file **b, int searchsize, int size);
+extern void free_manifest_array(struct file **array);
+
+extern bool check_mix_exists(void);
+extern int check_mix_versions(int *current_version, int *server_version, char *path_prefix);
 
 /* some disk sizes constants for the various features:
  *   ...consider adding build automation to catch at build time

src/bundle.c

@@ -74,7 +74,7 @@ int list_installable_bundles()
 
 	swupd_curl_set_current_version(current_version);
 
-	MoM = load_mom(current_version, false);
+	MoM = load_mom(current_version, false, false);
 	if (!MoM) {
 		v_lockfile(lock_fd);
 		return EMOM_NOTFOUND;
@@ -108,7 +108,7 @@ static int load_bundle_manifest(const char *bundle_name, struct list *subs, int
 	*submanifest = NULL;
 
 	swupd_curl_set_current_version(version);
-	mom = load_mom(version, false);
+	mom = load_mom(version, false, false);
 	if (!mom) {
 		return EMOM_NOTFOUND;
 	}
@@ -224,7 +224,7 @@ int show_included_bundles(char *bundle_name)
 		goto out;
 	}
 
-	mom = load_mom(current_version, false);
+	mom = load_mom(current_version, false, false);
 	if (!mom) {
 		fprintf(stderr, "Cannot load official manifest MoM for version %i\n", current_version);
 		ret = EMOM_NOTFOUND;
@@ -322,7 +322,7 @@ int show_bundle_reqd_by(const char *bundle_name, bool server)
 		goto out;
 	}
 
-	current_manifest = load_mom(version, server);
+	current_manifest = load_mom(version, server, false);
 	if (!current_manifest) {
 		fprintf(stderr, "Unable to download/verify %d Manifest.MoM\n", version);
 		ret = EMOM_NOTFOUND;
@@ -345,7 +345,7 @@ int show_bundle_reqd_by(const char *bundle_name, bool server)
 
 	} else {
 		/* load all tracked bundles into memory */
-		read_subscriptions_alt(&subs);
+		read_subscriptions(&subs);
 		/* now popout the one to be processed */
 		ret = unload_tracked_bundle(bundle_name, &subs);
 		if (ret != 0) {
@@ -462,7 +462,7 @@ int remove_bundle(const char *bundle_name)
 
 	swupd_curl_set_current_version(current_version);
 
-	current_mom = load_mom(current_version, false);
+	current_mom = load_mom(current_version, false, false);
 	if (!current_mom) {
 		fprintf(stderr, "Unable to download/verify %d Manifest.MoM\n", current_version);
 		ret = EMOM_NOTFOUND;
@@ -476,7 +476,7 @@ int remove_bundle(const char *bundle_name)
 	}
 
 	/* load all tracked bundles into memory */
-	read_subscriptions_alt(&subs);
+	read_subscriptions_alt(&subs, false);
 	/* now popout the one to be removed */
 	ret = unload_tracked_bundle(bundle_name, &subs);
 	if (ret != 0) {
@@ -601,6 +601,7 @@ int add_subscriptions(struct list *bundles, struct list **subs, int current_vers
 		}
 
 	retry_manifest_download:
+		printf("add_subscriptions: load_manifest(): %s %d\n", file->filename, file->is_mix);
 		manifest = load_manifest(current_version, file->last_change, file, mom, true);
 		if (!manifest) {
 			if (retries < MAX_TRIES) {
@@ -692,7 +693,7 @@ static int install_bundles(struct list *bundles, struct list **subs, int current
 	(void)rm_staging_dir_contents("download");
 
 download_subscribed_packs:
-	if (download_subscribed_packs(*subs, true)) {
+	if (download_subscribed_packs(*subs, mom, true)) {
 		if (retries < MAX_TRIES) {
 			increment_retries(&retries, &timeout);
 			printf("\nRetry #%d downloading subscribed packs\n", retries);
@@ -703,7 +704,7 @@ download_subscribed_packs:
 
 	/* step 3: Add tracked bundles */
 	grabtime_start(&times, "Add tracked bundles");
-	read_subscriptions_alt(subs);
+	read_subscriptions_alt(subs, false);
 	set_subscription_versions(mom, NULL, subs);
 	mom->submanifests = recurse_manifest(mom, *subs, NULL, false);
 	if (!mom->submanifests) {
@@ -854,7 +855,7 @@ int install_bundles_frontend(char **bundles)
 
 	swupd_curl_set_current_version(current_version);
 
-	mom = load_mom(current_version, false);
+	mom = load_mom(current_version, false, false);
 	if (!mom) {
 		fprintf(stderr, "Cannot load official manifest MoM for version %i\n", current_version);
 		ret = EMOM_NOTFOUND;
@@ -929,3 +930,4 @@ void read_local_bundles(struct list **list_bundles)
 
 	free(path);
 }
+

src/curl.c

@@ -342,6 +342,7 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file,
 				abort();
 			}
 		}
+
 		local->staging = filename;
 
 		if (lstat(filename, &stat) == 0) {

src/manifest.c

@@ -162,7 +162,7 @@ static struct manifest *alloc_manifest(int version, char *component)
 	return manifest;
 }
 
-static struct manifest *manifest_from_file(int version, char *component, bool header_only, bool latest)
+static struct manifest *manifest_from_file(int version, char *component, bool header_only, bool latest, bool is_mix)
 {
 	FILE *infile;
 	char line[MANIFEST_LINE_MAXLEN], *c, *c2;
@@ -171,12 +171,18 @@ static struct manifest *manifest_from_file(int version, char *component, bool he
 	struct manifest *manifest;
 	struct list *includes = NULL;
 	char *filename;
+	char *basedir;
 	uint64_t filecount = 0;
 	uint64_t contentsize = 0;
 	int manifest_hdr_version;
 	int manifest_enc_version;
 
-	string_or_die(&filename, "%s/%i/Manifest.%s", state_dir, version, component);
+	if (!is_mix) {
+		basedir = state_dir;
+	} else {
+		basedir = MIX_STATE_DIR;
+	}
+	string_or_die(&filename, "%s/%i/Manifest.%s", basedir, version, component);
 
 	infile = fopen(filename, "rbm");
 	if (infile == NULL) {
@@ -335,6 +341,9 @@ static struct manifest *manifest_from_file(int version, char *component, bool he
 
 		if (c[3] == 'r') {
 			file->is_rename = 1;
+		} else if (c[3] == 'm') {
+			file->is_mix = 1;
+			manifest->is_mix = 1;
 		} else if (c[3] != '.') { /* unknown modifier #3 */
 			free(file);
 			goto err;
@@ -375,6 +384,12 @@ static struct manifest *manifest_from_file(int version, char *component, bool he
 			abort();
 		}
 
+		/* Mark every file in a mix manifest as also being mix content since we do not
+		 * have another flag to check for like we do in the MoM */
+		if (is_mix && strcmp(component, "MoM") != 0) {
+			file->is_mix = 1;
+		}
+
 		if (file->is_manifest) {
 			manifest->manifests = list_prepend_data(manifest->manifests, file);
 		} else {
@@ -507,15 +522,22 @@ out:
 }
 
 /* TODO: This should deal with nested manifests better */
-static int retrieve_manifests(int current, int version, char *component, struct file *file)
+static int retrieve_manifests(int current, int version, char *component, struct file *file, bool is_mix)
 {
 	char *url = NULL;
 	char *filename;
 	char *dir;
+	char *basedir;
 	int ret = 0;
 	char *tar;
 	struct stat sb;
 
+	if (!is_mix) {
+		basedir = state_dir;
+	} else {
+		basedir = MIX_STATE_DIR;
+	}
+
 	/* Check for fullfile only, we will not be keeping the .tar around */
 	string_or_die(&filename, "%s/%i/Manifest.%s", state_dir, version, component);
 	if (stat(filename, &sb) == 0) {
@@ -523,7 +545,6 @@ static int retrieve_manifests(int current, int version, char *component, struct
 		goto out;
 	}
 	free(filename);
-	string_or_die(&filename, "%s/%i/Manifest.%s.tar", state_dir, version, component);
 
 	if (swupd_curl_check_network()) {
 		ret = -ENOSWUPDSERVER;
@@ -538,12 +559,30 @@ static int retrieve_manifests(int current, int version, char *component, struct
 	free(dir);
 
 	/* FILE is not set for a MoM, only for bundle manifests */
-	if (file && current < version) {
+	if (file && current < version && strcmp(component, "full") != 0) {
 		if (try_delta_manifest_download(current, version, component, file) == 0) {
 			return 0;
 		}
 	}
 
+	/* If it's mix content just hardlink instead of curl download */
+	if (is_mix) {
+		string_or_die(&filename, "%s/%i/Manifest.%s.tar", state_dir, version, component);
+		string_or_die(&url, "%s/%i/Manifest.%s.tar", basedir, version, component);
+		ret = link(url, filename);
+		/* Try doing a regular rename if hardlink fails */
+		if (ret) {
+			ret = rename(url, filename);
+		}
+		if (ret == 0) {
+			goto untar;
+		}
+		free(filename);
+		free(url);
+	}
+
+	/* Either we're not on mix or it failed, try curl-ing the file if link didn't work */
+	string_or_die(&filename, "%s/%i/Manifest.%s.tar", state_dir, version, component);
 	string_or_die(&url, "%s/%i/Manifest.%s.tar", content_url, version, component);
 
 	ret = swupd_curl_get_file(url, filename, NULL, NULL, false);
@@ -552,6 +591,19 @@ static int retrieve_manifests(int current, int version, char *component, struct
 		goto out;
 	}
 
+	if (ret) {
+		free(filename);
+		string_or_die(&filename, "%s/%i/Manifest.%s.tar", state_dir, version, component);
+		string_or_die(&url, "%s/%i/Manifest.%s.tar", content_url, version, component);
+
+		ret = swupd_curl_get_file(url, filename, NULL, NULL, false);
+		if (ret) {
+			unlink(filename);
+			goto out;
+		}
+	}
+
+untar:
 	string_or_die(&tar, TAR_COMMAND " -C %s/%i -xf %s/%i/Manifest.%s.tar 2> /dev/null",
 		      state_dir, version, state_dir, version, component);
 
@@ -564,7 +616,9 @@ static int retrieve_manifests(int current, int version, char *component, struct
 	if (ret != 0) {
 		goto out;
 	} else {
-		unlink(filename);
+		if (!is_mix) {
+			unlink(filename);
+		}
 	}
 
 out:
@@ -624,23 +678,30 @@ void remove_manifest_files(char *filename, int version, char *hash)
  * Note that if the manifest fails to download, or if the manifest fails to be
  * loaded into memory, this function will return NULL.
  */
-struct manifest *load_mom(int version, bool latest)
+struct manifest *load_mom(int version, bool latest, bool mix_exists)
 {
 	struct manifest *manifest = NULL;
 	int ret = 0;
+	char *basedir;
 	char *filename;
 	char *url;
 	char *log_cmd = NULL;
 	bool retried = false;
 
+	if (mix_exists) {
+		basedir = MIX_STATE_DIR;
+	} else {
+		basedir = state_dir;
+	}
+
 verify_mom:
-	ret = retrieve_manifests(version, version, "MoM", NULL);
+	ret = retrieve_manifests(version, version, "MoM", NULL, mix_exists);
 	if (ret != 0) {
 		fprintf(stderr, "Failed to retrieve %d MoM manifest\n", version);
 		return NULL;
 	}
 
-	manifest = manifest_from_file(version, "MoM", false, latest);
+	manifest = manifest_from_file(version, "MoM", false, latest, mix_exists);
 
 	if (manifest == NULL) {
 		if (retried == false) {
@@ -654,15 +715,15 @@ verify_mom:
 
 	string_or_die(&filename, "%s/%i/Manifest.MoM", state_dir, version);
 	string_or_die(&url, "%s/%i/Manifest.MoM", content_url, version);
-	if (!download_and_verify_signature(url, filename)) {
+	if (!download_and_verify_signature(url, filename, version, mix_exists)) {
 		if (sigcheck) {
 			/* cleanup and try one more time, statedir could have got corrupt/stale */
-			if (retried == false) {
+			if (retried == false && !mix_exists) {
 				remove_manifest_files("MoM", version, NULL);
 				retried = true;
 				goto verify_mom;
 			}
-			fprintf(stderr, "WARNING!!! FAILED TO VERIFY SIGNATURE OF Manifest.MoM\n");
+			fprintf(stderr, "WARNING!!! FAILED TO VERIFY SIGNATURE OF Manifest.MoM version %d\n", version);
 			free(filename);
 			free(url);
 			return NULL;
@@ -678,12 +739,14 @@ verify_mom:
 			free(log_cmd);
 		}
 	}
+	printf("MOM IS GOODDDD\n");
 	free(filename);
 	free(url);
-
+	printf("manifest is %s\n", manifest->component);
 	return manifest;
 
 out:
+	printf("why\n");
 	return NULL;
 }
 
@@ -706,8 +769,9 @@ struct manifest *load_manifest(int current, int version, struct file *file, stru
 	struct manifest *manifest = NULL;
 	int ret = 0;
 	bool retried = false;
+
 retry_load:
-	ret = retrieve_manifests(current, version, file->filename, file);
+	ret = retrieve_manifests(current, version, file->filename, file, file->is_mix);
 	if (ret != 0) {
 		fprintf(stderr, "Failed to retrieve %d %s manifest\n", version, file->filename);
 		return NULL;
@@ -725,9 +789,11 @@ retry_load:
 		}
 		return NULL;
 	}
-	retried = false;
+	if (!retried) {
+		retried = false;
+	}
 
-	manifest = manifest_from_file(version, file->filename, header_only, false);
+	manifest = manifest_from_file(version, file->filename, header_only, false, file->is_mix);
 
 	if (manifest == NULL) {
 		if (retried == false) {
@@ -744,6 +810,28 @@ retry_load:
 	return manifest;
 }
 
+/* Special case manifest for mixer content enforcement */
+struct manifest *load_manifest_full(int version, bool mix)
+{
+	struct manifest *manifest = NULL;
+	int ret = 0;
+
+	ret = retrieve_manifests(version, version, "full", NULL, mix);
+	if (ret != 0) {
+		fprintf(stderr, "Failed to retrieve %d Manifest.full\n", version);
+		return NULL;
+	}
+
+	manifest = manifest_from_file(version, "full", false, false, false);
+
+	if (manifest == NULL) {
+		fprintf(stderr, "Failed to load %d Manifest.full\n", version);
+		return NULL;
+	}
+
+	return manifest;
+}
+
 /* Checks to see if the given file is installed under the path_prefix. */
 static bool is_installed_and_verified(struct file *file)
 {
@@ -1443,8 +1531,9 @@ struct file **manifest_files_to_array(struct manifest *manifest)
 void print_manifest_array(struct file **array, int filecount)
 {
 	for (int i = 0; i < filecount; i++) {
-		printf("%s\n", array[i]->filename);
+		printf("[%d]/%d %s\n", i, filecount, array[i]->filename);
 	}
+	printf("SIZE: %d\n", filecount);
 }
 
 void free_manifest_array(struct file **array)
@@ -1457,23 +1546,28 @@ static int cmpnames(const void *a, const void *b)
 	return strcmp((*(struct file **)a)->filename, (*(struct file **)b)->filename);
 }
 
-int enforce_compliant_manifest(struct file **a, struct file **b, int size)
+int enforce_compliant_manifest(struct file **a, struct file **b, int searchsize, int size)
 {
 	struct file **found;
+	int ret = 0;
 
-	//qsort(a, size, sizeof(char *), cmpnames);
-	print_manifest_array(a, size);
+	qsort(a, searchsize, sizeof(struct file *), cmpnames);
+	qsort(b, size, sizeof(struct file *), cmpnames);
 	printf("Checking manifest uniqueness...\n");
-	for (int i = 0; i < size; i++) {
+	for (int i = 0; i < searchsize; i++) {
 		found = bsearch(a[i], b, size, sizeof(struct file *), bsearch_file_helper);
 		if (found) {
+			if (strcmp(a[i]->filename, "/usr/lib/os-release") == 0 ||
+			    strcmp(a[i]->filename, "/usr/share/clear/version") == 0 ||
+			    strcmp(a[i]->filename, "/usr/share/clear/versionstamp") == 0) {
+				continue;
+			}
 			if (hash_equal(a[i]->hash, (*found)->hash)) {
 				continue;
 			}
-			fprintf(stderr, "ERROR: Conflict found for file: %s & %s\n", a[i]->filename, (*found)->filename);
-			return 1;
+			fprintf(stderr, "ERROR: Conflict found for file: %s\n", a[i]->filename);
+			ret++;
 		}
 	}
-	return 0; // No collisions were found, so manifest is purely additive
+	return ret; // No collisions were found, so manifest is purely additive
 }
-

src/packs.c

@@ -35,7 +35,7 @@
 #include "swupd-build-variant.h"
 #include "swupd.h"
 
-static int download_pack(int oldversion, int newversion, char *module)
+static int download_pack(int oldversion, int newversion, char *module, int is_mix)
 {
 	FILE *tarfile = NULL;
 	char *tar = NULL;
@@ -52,10 +52,24 @@ static int download_pack(int oldversion, int newversion, char *module)
 		return 0;
 	}
 
-	string_or_die(&url, "%s/%i/pack-%s-from-%i.tar", content_url, newversion, module, oldversion);
+	if (is_mix) {
+		string_or_die(&url, "%s/%i/pack-%s-from-%i.tar", MIX_STATE_DIR, newversion, module, oldversion);
+		link(url, filename);
+		printf("Linked %s to %s\n", url, filename);
+		free(url);
+	} else {
+		string_or_die(&url, "%s/%i/pack-%s-from-%i.tar", content_url, newversion, module, oldversion);
+
+		err = swupd_curl_get_file(url, filename, NULL, NULL, true);
+		if (err) {
+			free(url);
+			if ((lstat(filename, &stat) == 0) && (stat.st_size == 0)) {
+				unlink(filename);
+			}
+			free(filename);
+			return err;
+		}
 
-	err = swupd_curl_get_file(url, filename, NULL, NULL, true);
-	if (err) {
 		free(url);
 		if ((lstat(filename, &stat) == 0) && (stat.st_size == 0)) {
 			unlink(filename);
@@ -92,7 +106,7 @@ static int download_pack(int oldversion, int newversion, char *module)
 }
 
 /* pull in packs for base and any subscription */
-int download_subscribed_packs(struct list *subs, bool required)
+int download_subscribed_packs(struct list *subs, struct manifest *mom, bool required)
 {
 	struct list *iter;
 	struct sub *sub = NULL;

src/search.c

@@ -428,7 +428,7 @@ static int download_manifests(struct manifest **MoM, struct list **subs)
 
 	swupd_curl_set_current_version(current_version);
 
-	*MoM = load_mom(current_version, false);
+	*MoM = load_mom(current_version, false, false);
 	if (!(*MoM)) {
 		fprintf(stderr, "Cannot load official manifest MoM for version %i\n", current_version);
 		return EMOM_NOTFOUND;

src/signature.c

@@ -414,8 +414,9 @@ int verify_callback(int ok, X509_STORE_CTX *stor)
  * returns: true if signature verification succeeded, false if verification
  * failed, or the signature download failed
  */
-bool download_and_verify_signature(const char *data_url, const char *data_filename)
+bool download_and_verify_signature(const char *data_url, const char *data_filename, int version, bool mix_exists)
 {
+	char *local;
 	char *sig_url;
 	char *sig_filename;
 	int ret;
@@ -426,8 +427,8 @@ bool download_and_verify_signature(const char *data_url, const char *data_filena
 	}
 
 	string_or_die(&sig_url, "%s.sig", data_url);
-
 	string_or_die(&sig_filename, "%s.sig", data_filename);
+	string_or_die(&local, "%s/%i/Manifest.MoM.sig", MIX_STATE_DIR, version);
 
 	// Try verifying a local copy of the signature first
 	result = verify_signature(data_filename, sig_filename, false);
@@ -436,7 +437,12 @@ bool download_and_verify_signature(const char *data_url, const char *data_filena
 	}
 
 	// Else, download a fresh signature, and verify
-	ret = swupd_curl_get_file(sig_url, sig_filename, NULL, NULL, false);
+	if (mix_exists) {
+		ret = link(local, sig_filename);
+	} else {
+		ret = swupd_curl_get_file(sig_url, sig_filename, NULL, NULL, false);
+	}
+
 	if (ret == 0) {
 		result = verify_signature(data_filename, sig_filename, true);
 	} else {

src/subscriptions.c

@@ -70,14 +70,23 @@ static int subscription_sort_component(const void *a, const void *b)
 	return ret;
 }
 
-void read_subscriptions_alt(struct list **subs)
+/* Custom content comes from a different location and is not required to
+ * have os-core added */
+void read_subscriptions_alt(struct list **subs, bool is_mix)
 {
 	bool have_os_core = false;
 	char *path = NULL;
+	char *bundlesdir = NULL;
 	DIR *dir;
 	struct dirent *ent;
 
-	string_or_die(&path, "%s/%s", path_prefix, BUNDLES_DIR);
+	if (!is_mix) {
+		bundlesdir = BUNDLES_DIR;
+	} else {
+		bundlesdir = MIX_BUNDLES_DIR;
+	}
+
+	string_or_die(&path, "%s/%s", path_prefix, bundlesdir);
 
 	dir = opendir(path);
 	if (dir) {
@@ -90,8 +99,10 @@ void read_subscriptions_alt(struct list **subs)
 					/*  This is considered odd since means two files same name on same folder */
 					continue;
 				}
-				if (strcmp(ent->d_name, "os-core") == 0) {
-					have_os_core = true;
+				if (!is_mix) {
+					if (strcmp(ent->d_name, "os-core") == 0) {
+						have_os_core = true;
+					}
 				}
 
 				create_and_append_subscription(subs, ent->d_name);
@@ -104,7 +115,7 @@ void read_subscriptions_alt(struct list **subs)
 	free(path);
 
 	/* Always add os-core */
-	if (!have_os_core) {
+	if (!is_mix && !have_os_core) {
 		create_and_append_subscription(subs, "os-core");
 	}
 

src/update.c

@@ -67,6 +67,9 @@ static struct list *full_download_loop(struct list *updates, int isfailed)
 {
 	struct list *iter;
 	struct file *file;
+	char *filename;
+	char *url;
+	int ret = -1;
 	unsigned int list_length = list_len(updates);
 	unsigned int complete = 0;
 
@@ -79,7 +82,23 @@ static struct list *full_download_loop(struct list *updates, int isfailed)
 		if (file->is_deleted) {
 			continue;
 		}
+		/* Mix content is local, so don't queue files up for curl downloads */
+		if (file->is_mix) {
+			string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash);
+			string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash);
+			file->staging = filename;
+			ret = link(url, filename);
+			/* Try doing a regular rename if hardlink fails */
+			if (ret) {
+				if (rename(url, filename) != 0) {
+					fprintf(stderr, "Failed to copy local mix file: %s\n", filename);
+					continue;
+				}
+			}
 
+			untar_full_download(file);
+			continue;
+		}
 		full_download(file);
 		print_progress(complete, list_length);
 	}
@@ -93,6 +112,39 @@ static struct list *full_download_loop(struct list *updates, int isfailed)
 	return end_full_download();
 }
 
+/* This loads the upstream Clear Manifest.Full and local Manifest.full, and then checks
+ * that there are no conflicts between the files they both include */
+int check_manifests_uniqueness(int clrver, int mixver)
+{
+	struct manifest *clear = load_manifest_full(clrver, false);
+	struct manifest *mixer = load_manifest_full(mixver, true);
+	if (!clear || !mixer) {
+		fprintf(stderr, "ERROR: Could not load full manifests\n");
+		return -1;
+	}
+
+	struct file **clearfull = manifest_files_to_array(clear);
+	struct file **mixerfull = manifest_files_to_array(mixer);
+
+	if (clearfull == NULL || mixerfull == NULL) {
+		fprintf(stderr, "Could not convert full manifest to array\n");
+		abort();
+	}
+
+	int ret = enforce_compliant_manifest(mixerfull, clearfull, mixer->filecount, clear->filecount);
+
+	free_manifest_array(clearfull);
+	free_manifest_array(mixerfull);
+	free_manifest(clear);
+	free_manifest(mixer);
+
+	return ret;
+}
+
+void setup_mix_update()
+{
+}
+
 static int update_loop(struct list *updates, struct manifest *server_manifest)
 {
 	int ret;
@@ -221,29 +273,47 @@ int add_included_manifests(struct manifest *mom, int current, struct list **subs
 	return ret;
 }
 
-static int re_exec_update(bool versions_match)
+static int re_exec_update(bool versions_match)                                  
+{                                                                               
+    if (!versions_match) {                                                      
+        fprintf(stderr, "ERROR: Inconsistency between version files, exiting now.\n");
+        return 1;                                                               
+    }                                                                           
+                                                                                
+    if (!swupd_cmd) {                                                           
+        fprintf(stderr, "ERROR: Unable to determine re-update command, exiting now.\n");
+        return 1;                                                               
+    }                                                                           
+                                                                                
+    /* Run the swupd_cmd saved from main */                                     
+    if (system(swupd_cmd) != 0) {                                               
+        return 1;                                                               
+    }                                                                           
+                                                                                
+    return 0;                                                                   
+}  
+
+static bool system_on_mix(void)
 {
-	if (!versions_match) {
-		fprintf(stderr, "ERROR: Inconsistency between version files, exiting now.\n");
-		return 1;
-	}
+	bool ret = (access("/usr/share/defaults/swupd/mixed", R_OK) != 0);
+	return ret;
+}
 
-	if (!swupd_cmd) {
-		fprintf(stderr, "ERROR: Unable to determine re-update command, exiting now.\n");
-		return 1;
+static bool need_new_upstream(int server)
+{
+	if (!access(MIX_DIR ".clearversion", R_OK)) {
+		int version = read_mix_version_file(MIX_DIR ".clearversion", path_prefix);
+		if (version < server) {
+			return true;
+		}
 	}
-
-	/* Run the swupd_cmd saved from main */
-	if (system(swupd_cmd) != 0) {
-		return 1;
-	}
-
-	return 0;
+	return false;
 }
 
 int main_update()
 {
 	int current_version = -1, server_version = -1;
+	int mix_current_version = -1, mix_server_version = -1;
 	struct manifest *current_manifest = NULL, *server_manifest = NULL;
 	struct list *updates = NULL;
 	struct list *current_subs = NULL;
@@ -255,6 +325,7 @@ int main_update()
 	struct timespec ts_start, ts_stop; // For main swupd update time
 	timelist times;
 	double delta;
+	bool mix_exists;
 	bool re_update = false;
 	bool versions_match = false;
 
@@ -278,20 +349,57 @@ int main_update()
 		goto clean_curl;
 	}
 
+	mix_exists = check_mix_exists();
+
 	fprintf(stderr, "Update started.\n");
 
 	grabtime_start(&times, "Update Step 1: get versions");
 
-	read_subscriptions_alt(&current_subs);
-
-	/* Step 1: get versions */
+	read_subscriptions_alt(&current_subs, mix_exists);
 
+/* Step 1: get versions */
+version_check:
 	ret = check_versions(&current_version, &server_version, path_prefix);
 
 	if (ret < 0) {
 		ret = EXIT_FAILURE;
 		goto clean_curl;
 	}
+
+	if (mix_exists) {
+		ret = check_mix_versions(&mix_current_version, &mix_server_version, path_prefix);
+		if (ret < 0) {
+			ret = EXIT_FAILURE;
+			goto clean_curl;
+		}
+		/* Check if a new upstream version is available so we can update to it still */
+		if (need_new_upstream(server_version)) {
+			printf("NEW CLEAR AVAILABLE %d\n", server_version);
+			ret = check_manifests_uniqueness(server_version, mix_server_version);
+			if (ret) {
+				printf("\n\t!! %i collisions were found between mix and upstream, please re-create mix !!\n", ret);
+			}
+
+			/* Update the clearversion that will be used to generate the new mix content */
+			FILE *verfile = fopen(MIX_DIR ".clearversion", "w+");
+			if (!verfile) {
+				fprintf(stderr, "ERROR: fopen() returned %s\n", strerror(errno));
+			} else {
+				fprintf(verfile, "%d", server_version);
+				fclose(verfile);
+			}
+
+			if (system("/usr/bin/add-pkg.sh") != 0) {
+				fprintf(stderr, "ERROR: Could not execute add-pkg.sh\n");
+				ret = EXIT_FAILURE;
+				goto clean_curl;
+			}
+			goto version_check;
+		}
+		current_version = mix_current_version;
+		server_version = mix_server_version;
+	}
+
 	if (server_version <= current_version) {
 		fprintf(stderr, "Version on server (%i) is not newer than system version (%i)\n", server_version, current_version);
 		ret = EXIT_SUCCESS;
@@ -314,7 +422,11 @@ load_current_mom:
 	/* Step 3: setup manifests */
 
 	/* get the from/to MoM manifests */
-	current_manifest = load_mom(current_version, false);
+	if (system_on_mix()) {
+		current_manifest = load_mom(current_version, false, mix_exists);
+	} else {
+		current_manifest = load_mom(current_version, false, false);
+	}
 	if (!current_manifest) {
 		/* TODO: possibly remove this as not getting a "from" manifest is not fatal
 		 * - we just don't apply deltas */
@@ -335,7 +447,7 @@ load_current_mom:
 load_server_mom:
 	grabtime_stop(&times); // Close step 2
 	grabtime_start(&times, "Recurse and Consolidate Manifests");
-	server_manifest = load_mom(server_version, true);
+	server_manifest = load_mom(server_version, true, mix_exists);
 	if (!server_manifest) {
 		if (retries < MAX_TRIES) {
 			increment_retries(&retries, &timeout);
@@ -378,6 +490,7 @@ load_current_submanifests:
 	latest_subs = list_clone(current_subs);
 	set_subscription_versions(server_manifest, current_manifest, &latest_subs);
 	link_submanifests(current_manifest, server_manifest, current_subs, latest_subs, false);
+
 	/* The new subscription is seeded from the list of currently installed bundles
 	 * This calls add_subscriptions which recurses for new includes */
 	grabtime_start(&times, "Add Included Manifests");
@@ -427,7 +540,7 @@ load_server_submanifests:
 
 download_packs:
 	/* Step 5: get the packs and untar */
-	ret = download_subscribed_packs(latest_subs, false);
+	ret = download_subscribed_packs(latest_subs, server_manifest, false);
 	if (ret) {
 		// packs don't always exist, tolerate that but not ENONET
 		if (retries < MAX_TRIES) {
@@ -483,6 +596,16 @@ download_packs:
 	run_scripts(re_update);
 	grabtime_stop(&times);
 
+	/* Create the state file that will tell swupd it's on a mix on future runs */
+	if (mix_exists && !system_on_mix()) {
+		int fd = open("/usr/share/defaults/swupd/mixed", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+		if (fd == -1) {
+			fprintf(stderr, "ERROR: Failed to create 'mixed' statefile\n");
+			ret = -1;
+		}
+		close(fd);
+	}
+
 clean_exit:
 	list_free_list(updates);
 	free_manifest(current_manifest);

src/verify.c

@@ -255,7 +255,7 @@ static int get_all_files(struct manifest *official_manifest, struct list *subs)
 	struct list *iter;
 
 	/* for install we need everything so synchronously download zero packs */
-	ret = download_subscribed_packs(subs, true);
+	ret = download_subscribed_packs(subs, official_manifest, true);
 	if (ret < 0) { // require zero pack
 		/* If we hit this point, we know we have a network connection, therefore
 		 * 	the error is server-side. This is also a critical error, so detailed
@@ -321,6 +321,26 @@ static struct list *download_loop(struct list *files, int isfailed)
 		}
 
 		if (hash_needs_work(file, local.hash)) {
+			/* Mix content is local, so don't queue files up for curl downloads */
+			if (file->is_mix) {
+				char *filename;
+				char *url;
+				string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash);
+				string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash);
+				file->staging = filename;
+				ret = link(url, filename);
+				/* Try doing a regular rename if hardlink fails */
+				if (ret) {
+					if (rename(url, filename) != 0) {
+						fprintf(stderr, "Failed to copy local mix file: %s\n", filename);
+						continue;
+					}
+				}
+				untar_full_download(file);
+				free(filename);
+				free(url);
+				continue;
+			}
 			full_download(file);
 		} else {
 			/* mark the file as good to save time later */
@@ -651,6 +671,7 @@ int verify_main(int argc, char **argv)
 	int timeout = 10;
 	struct list *subs = NULL;
 	timelist times;
+	bool mix_exists;
 
 	copyright_header("software verify");
 
@@ -695,7 +716,9 @@ int verify_main(int argc, char **argv)
 		goto clean_and_exit;
 	}
 
-	read_subscriptions_alt(&subs);
+	mix_exists = check_mix_exists();
+
+	read_subscriptions_alt(&subs, mix_exists);
 
 	/*
 	 * FIXME: We need a command line option to override this in case the
@@ -710,7 +733,7 @@ int verify_main(int argc, char **argv)
 	times = init_timelist();
 
 	grabtime_start(&times, "Load and recurse Manifests");
-	official_manifest = load_mom(version, false);
+	official_manifest = load_mom(version, false, mix_exists);
 
 	if (!official_manifest) {
 		/* This is hit when or if an OS version is specified for --fix which

src/version.c

@@ -97,25 +97,23 @@ int get_current_version(char *path_prefix)
 		if (fgets(line, LINE_MAX, file) == NULL) {
 			break;
 		}
-
 		if (strncmp(line, "VERSION_ID=", 11) == 0) {
 			src = &line[11];
-
-			/* Drop quotes and newline in value */
-			dest = src;
-			while (*src) {
-				if (*src == '\'' || *src == '"' || *src == '\n') {
-					++src;
-				} else {
-					*dest = *src;
-					++dest;
-					++src;
+				/* Drop quotes and newline in value */
+				dest = src;
+				while (*src) {
+					if (*src == '\'' || *src == '"' || *src == '\n') {
+						++src;
+					} else {
+						*dest = *src;
+						++dest;
+						++src;
+					}
 				}
-			}
-			*dest = 0;
+				*dest = 0;
 
-			v = strtoull(&line[11], NULL, 10);
-			break;
+				v = strtoull(&line[11], NULL, 10);
+				break;
 		}
 	}
 
@@ -170,6 +168,44 @@ int check_versions(int *current_version,
 	return 0;
 }
 
+int read_mix_version_file(char *filename, char *path_prefix)
+{
+	char line[LINE_MAX];
+	FILE *file;
+	int v = -1;
+	char *buildstamp;
+
+	string_or_die(&buildstamp, "%s%s", path_prefix, filename);
+	file = fopen(buildstamp, "rm");
+	if (!file) {
+		free(buildstamp);
+		return v;
+	}
+
+	while (!feof(file)) {
+		line[0] = 0;
+		if (fgets(line, LINE_MAX, file) == NULL) {
+			break;
+		}
+
+		/* Drop newline in value */
+		char *c = strchr(line, '\n');
+		if (c) {
+			*c = '\0';
+		}
+
+		v = strtoull(line, NULL, 10);
+	}
+	free(buildstamp);
+	fclose(file);
+	return v;
+}
+
+int check_mix_versions(int *current_version, int *server_version, char *path_prefix)
+{
+	*current_version = read_mix_version_file("/usr/share/clear/version", path_prefix);
+	*server_version = read_mix_version_file(MIX_STATE_DIR "version/format1/latest", path_prefix);
+}
 int update_device_latest_version(int version)
 {
 	FILE *file = NULL;