diff --git a/src/bundle.c b/src/bundle.c index 578bed17..76119916 100644 --- a/src/bundle.c +++ b/src/bundle.c @@ -777,7 +777,7 @@ download_subscribed_packs: /* step 2.5: Download missing files */ grabtime_start(×, "Download missing files"); - ret = download_fullfiles(to_install_files, MAX_TRIES, 10); + ret = download_fullfiles(to_install_files, NULL); if (ret) { fprintf(stderr, "ERROR: Could not download some files from bundles, aborting bundle installation.\n"); goto out; diff --git a/src/curl-internal.h b/src/curl-internal.h new file mode 100644 index 00000000..6522cdb8 --- /dev/null +++ b/src/curl-internal.h @@ -0,0 +1,43 @@ +#ifndef __INCLUDE_GUARD_CURL_INTERNAL_H +#define __INCLUDE_GUARD_CURL_INTERNAL_H + +/* Define internal curl functions to be used by curl.c and download.c. + * Not expected to be used by users of swupd_curl_* functions. + */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct curl_file { + char *path; /* output name used during download */ + FILE *fh; /* file written into during downloading */ +}; + +/* + * Create a file in disk to start the download. + */ +extern CURLcode swupd_download_file_create(struct curl_file *file); + +/* + * Create a file in disk to start the download, using append mode. + */ +extern CURLcode swupd_download_file_append(struct curl_file *file); + +/* + * Close file after the download is finished. + */ +extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct curl_file *file); + +/* + * Set swupd default basic options to curl handler. + */ +extern CURLcode swupd_curl_set_basic_options(CURL *curl, const char *url); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/curl.c b/src/curl.c index a50e3f90..600dbdff 100644 --- a/src/curl.c +++ b/src/curl.c @@ -44,8 +44,13 @@ #include #include "config.h" +#include "curl-internal.h" #include "swupd.h" +#define SWUPD_CURL_LOW_SPEED_LIMIT 1 +#define SWUPD_CURL_CONNECT_TIMEOUT 30 +#define SWUPD_CURL_RCV_TIMEOUT 120 + static CURL *curl = NULL; /* alternative CA Path */ @@ -227,34 +232,34 @@ static size_t swupd_download_file_to_memory(void *ptr, size_t size, size_t nmemb return data_len; } -CURLcode swupd_download_file_start(struct file *file) +CURLcode swupd_download_file_create(struct curl_file *file) { - file->fh = fopen(file->staging, "w"); + file->fh = fopen(file->path, "w"); if (!file->fh) { fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", - file->staging, strerror(errno)); + file->path, strerror(errno)); return CURLE_WRITE_ERROR; } return CURLE_OK; } -CURLcode swupd_download_file_append(struct file *file) +CURLcode swupd_download_file_append(struct curl_file *file) { - file->fh = fopen(file->staging, "a"); + file->fh = fopen(file->path, "a"); if (!file->fh) { fprintf(stderr, "Cannot open file for append \\*outfile=\"%s\",strerror=\"%s\"*\\\n", - file->staging, strerror(errno)); + file->path, strerror(errno)); return CURLE_WRITE_ERROR; } return CURLE_OK; } -CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file) +CURLcode swupd_download_file_complete(CURLcode curl_ret, struct curl_file *file) { if (file->fh) { if (fclose(file->fh)) { fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", - file->staging, strerror(errno)); + file->path, strerror(errno)); if (curl_ret == CURLE_OK) { curl_ret = CURLE_WRITE_ERROR; } @@ -284,7 +289,7 @@ int swupd_curl_get_file_full(const char *url, char *filename, CURLcode curl_ret; long ret = 0; int err = -1; - struct file local = { 0 }; + struct curl_file local = { 0 }; bool local_download = strncmp(url, "file://", 7) == 0; if (!curl) { @@ -297,7 +302,7 @@ restart_download: // normal file download struct stat stat; - local.staging = filename; + local.path = filename; if (resume_ok && resume_download_supported && lstat(filename, &stat) == 0) { curl_ret = curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)stat.st_size); @@ -307,7 +312,7 @@ restart_download: curl_ret = swupd_download_file_append(&local); } else { - curl_ret = swupd_download_file_start(&local); + curl_ret = swupd_download_file_create(&local); } curl_ret = curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)&local); @@ -439,7 +444,7 @@ exit: /* * Download a single file SYNCHRONOUSLY - + * * Returns: Zero on success or a standard < 0 status code on errors. */ int swupd_curl_get_file(const char *url, char *filename) diff --git a/src/download.c b/src/download.c index d7d7a9a0..ad2c47fd 100644 --- a/src/download.c +++ b/src/download.c @@ -29,29 +29,28 @@ #include #include #include -#include #include #include #include "config.h" +#include "curl-internal.h" #include "hashmap.h" -#include "swupd-build-variant.h" #include "swupd.h" -/* This file provides a managed download facility for code that needs a set of - * update tar files. Such code starts with one call to "start_full_download()", - * then makes a series of calls to "full_download()" once per desired file, and - * then finishes with one call to "end_full_download()" which will block until - * the previously queued downloads are completed and and untarred. - */ +#define MAX_RETRIES 5 +#define RETRY_TIMEOUT 10 -static CURLM *mcurl = NULL; -static size_t mcurl_size = 0; -static struct list *failed = NULL; +/* This file provides a managed download facility for parallelizing download + * files. To use this functionality, create a handler using + * swupd_curl_parallel_download_start(), enqueue files using + * swupd_curl_parallel_download_enqueue() and finish the process using + * swupd_curl_parallel_download_end(), which will block until the previously + * queued downloads are completed. + */ /* * The following represents a hash data structure used inside download.c - * to track file downloads asynchonously via libcurl. In the past a + * to track file downloads in parallel via libcurl. In the past a * single linked list was used, but we may have a very large number of * files and repeatedly scan the list could become expensive. The hashmap * gives info on what HASH.tar files will be downloaded. A de-duplicated @@ -74,252 +73,111 @@ static struct list *failed = NULL; */ #define HASH_TO_KEY(hash) (HASH_VALUE(hash[0]) << 4 | HASH_VALUE(hash[1])) -static struct hashmap *swupd_curl_hashmap = NULL; +struct swupd_curl_parallel_handle { + int timeout; /* Retry timeout */ + size_t mcurl_size, max_xfer, max_xfer_bottom; /* hysteresis parameters */ -/* try to insert the file into the hashmap download queue - * returns 1 if no download is needed - * returns 0 if download is needed - * returns -1 if error */ -static int swupd_curl_hashmap_insert(struct file *file) -{ - char *tar_dotfile; - char *targetfile; - struct stat stat; + CURLM *mcurl; /* Curl handle */ + struct list *failed; /* List of failed downloads */ + struct hashmap *curl_hashmap; /* Hashmap mentioned above */ + swupd_curl_download_cb success_cb; /* Callback to sucess function */ +}; - if (!hashmap_put(swupd_curl_hashmap, file)) { - // hash already in download queue - return 1; - } +/* + * Struct represinting a single file being downloaded. + */ +struct multi_curl_file { + struct curl_file file; /* Curl file information */ - // if valid target file is already here, no need to download - string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash); + char retries; /* Number of retried performed so far */ + CURL *curl; /* curl handle if downloading */ + char *url; /* The url to be downloaded from */ + size_t hash_key; /* hash_key of this file */ + const char *hash; /* Unique identifier of this file. */ - if (lstat(targetfile, &stat) == 0) { - /* file exists */ - if (verify_file(file, targetfile)) { - /* hash matches, no download necessary */ - free_string(&targetfile); - return 1; - } else { - /* hash mismatch, remove the staged file to enable re-download */ - unlink(targetfile); - } - } - - free_string(&targetfile); - - // hash not in queue and not present in staged - - // clean up in case any prior download failed in a partial state - string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash); - unlink(tar_dotfile); - free_string(&tar_dotfile); - - return 0; -} - -/* hysteresis thresholds */ -static size_t MAX_XFER = 25; -static size_t MAX_XFER_BOTTOM = 15; + void *data; /* user's data */ +}; static bool file_hash_cmp(const void *a, const void *b) { - const struct file *fa = a; - const struct file *fb = b; + const struct multi_curl_file *fa = a; + const struct multi_curl_file *fb = b; + if (!fa->hash || !fb->hash) { + return strcmp(fa->file.path, fb->file.path) == 0; + } return hash_equal(fa->hash, fb->hash); } static size_t file_hash_value(const void *data) { - return HASH_VALUE(((struct file *)data)->hash[0]); + return ((struct multi_curl_file *)data)->hash_key; } -int start_full_download(void) +static void free_curl_file(struct swupd_curl_parallel_handle *h, struct multi_curl_file *file) { - if (swupd_curl_hashmap) { - return -1; - } - - failed = NULL; - swupd_curl_hashmap = hashmap_new(SWUPD_CURL_HASH_BUCKETS, file_hash_cmp, file_hash_value); - - mcurl = curl_multi_init(); - if (mcurl == NULL) { - return -1; - } - - curl_multi_setopt(mcurl, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX | CURLPIPE_HTTP1); - - fprintf(stderr, "Starting download of remaining update content. This may take a while...\n"); - - return 0; -} - -static void free_curl_list_data(void *data) -{ - struct file *file = (struct file *)data; CURL *curl = file->curl; - (void)swupd_download_file_complete(CURLE_OK, file); + swupd_download_file_complete(CURLE_OK, &file->file); + + free_string(&file->url); + free_string(&file->file.path); if (curl != NULL) { /* Must remove handle out of multi queue first!*/ - curl_multi_remove_handle(mcurl, curl); + curl_multi_remove_handle(h->mcurl, curl); curl_easy_cleanup(curl); file->curl = NULL; } + free(file); } -/* list the tarfile content, and verify it contains only one line equal to the expected hash. - * loop through all the content to detect the case where archive contains more than one file. +/* + * Start a parallel download element. + * + * Parameters: + * - max_xfer: The maximum number of simultaneos downloads. + * - max_xfer_bottom: Minimum number of simultaneos downloads before starting + * - sucess_cb(): Callback called for each sucessfull download. + * + * Parallel download handler will retry MAX_TRIES times to download each file, + * ading a timeout between each try. + * + * Note: This function is non-blocking. */ -static int check_tarfile_content(struct file *file, const char *tarfilename) +void *swupd_curl_parallel_download_start(size_t max_xfer, size_t max_xfer_bottom, swupd_curl_download_cb success_cb) { - int err; - char *tarcommand; - FILE *tar; - int count = 0; + struct swupd_curl_parallel_handle *h = calloc(1, sizeof(struct swupd_curl_parallel_handle)); - string_or_die(&tarcommand, TAR_COMMAND " -tf %s/download/%s.tar 2> /dev/null", state_dir, file->hash); - - err = access(tarfilename, R_OK); - if (err) { - goto free_tarcommand; + h->mcurl = curl_multi_init(); + if (h->mcurl == NULL) { + goto error; } - tar = popen(tarcommand, "r"); - if (tar == NULL) { - err = -1; - goto free_tarcommand; - } + curl_multi_setopt(h->mcurl, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX | CURLPIPE_HTTP1); - while (!feof(tar)) { - char *c; - char buffer[PATH_MAXLEN]; + h->max_xfer = max_xfer; + h->max_xfer_bottom = max_xfer_bottom; + h->success_cb = success_cb; + h->curl_hashmap = hashmap_new(SWUPD_CURL_HASH_BUCKETS, file_hash_cmp, file_hash_value); + h->timeout = RETRY_TIMEOUT; - if (fgets(buffer, PATH_MAXLEN, tar) == NULL) { - if (count != 1) { - err = -1; - } - break; - } + fprintf(stderr, "Starting download of remaining update content. This may take a while...\n"); - c = strchr(buffer, '\n'); - if (c) { - *c = 0; - } - if (c && (c != buffer) && (*(c - 1) == '/')) { - /* strip trailing '/' from directory tar */ - *(c - 1) = 0; - } - if (strcmp(buffer, file->hash) != 0) { - err = -1; - break; - } - count++; - } - - pclose(tar); -free_tarcommand: - free_string(&tarcommand); - - return err; + return h; +error: + free(h); + return NULL; } -/* This function will break if the same HASH.tar full file is downloaded - * multiple times in parallel. */ -int untar_full_download(void *data) -{ - struct file *file = data; - char *tarfile; - char *tar_dotfile; - char *targetfile; - struct stat stat; - int err; - - string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash); - string_or_die(&tarfile, "%s/download/%s.tar", state_dir, file->hash); - string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash); - - /* If valid target file already exists, we're done. - * NOTE: this should NEVER happen given the checking that happens - * ahead of queueing a download. But... */ - if (lstat(targetfile, &stat) == 0) { - if (verify_file(file, targetfile)) { - unlink(tar_dotfile); - unlink(tarfile); - free_string(&tar_dotfile); - free_string(&tarfile); - free_string(&targetfile); - return 0; - } else { - unlink(tarfile); - unlink(targetfile); - } - } else if (lstat(tarfile, &stat) == 0) { - /* remove tar file from possible past failure */ - unlink(tarfile); - } - - err = rename(tar_dotfile, tarfile); - if (err) { - free_string(&tar_dotfile); - goto exit; - } - free_string(&tar_dotfile); - - err = check_tarfile_content(file, tarfile); - if (err) { - goto exit; - } - - /* modern tar will automatically determine the compression type used */ - char *outputdir; - string_or_die(&outputdir, "%s/staged", state_dir); - err = extract_to(tarfile, outputdir); - free_string(&outputdir); - if (err) { - fprintf(stderr, "ignoring tar extract failure for fullfile %s.tar (ret %d)\n", - file->hash, err); - goto exit; - /* TODO: respond to ARCHIVE_RETRY error codes - * libarchive returns ARCHIVE_RETRY when tar extraction fails but the - * operation is retry-able. We need to determine if it is worth our time - * to retry in these situations. */ - } else { - /* Only unlink when tar succeeded, so we can examine the tar file - * in the failure case. */ - unlink(tarfile); - } - - err = lstat(targetfile, &stat); - if (!err && !verify_file(file, targetfile)) { - /* Download was successful but the hash was bad. This is fatal*/ - fprintf(stderr, "Error: File content hash mismatch for %s (bad server data?)\n", targetfile); - exit(EXIT_FAILURE); - } - -exit: - free_string(&tarfile); - free_string(&targetfile); - if (err) { - unlink_all_staged_content(file); - } - return err; -} - -/* Try to process at most COUNT messages from the curl multi-stack, and enforce - * the hysteresis when BOUNDED is true. The COUNT value is a best guess about - * the number of messages ready to process, because it may include file - * descriptors internal to libcurl as well (see curl_multi_wait(3)). */ -static int perform_curl_io_and_complete(int count, bool bounded) +// Try to process at most COUNT messages from the curl multi-stack. +static int perform_curl_io_and_complete(struct swupd_curl_parallel_handle *h, int count) { CURLMsg *msg; - long ret; + long response; CURLcode curl_ret; while (count > 0) { CURL *handle; - struct file *file; + struct multi_curl_file *file; char *url = NULL; bool local_download; @@ -331,7 +189,7 @@ static int perform_curl_io_and_complete(int count, bool bounded) * curl_multi_info_read() does not accept NULL for the second * argument to indicate an unused value. */ int unused; - msg = curl_multi_info_read(mcurl, &unused); + msg = curl_multi_info_read(h->mcurl, &unused); if (!msg) { /* Either there were fewer than COUNT messages to * process, or the multi-stack is now empty. */ @@ -342,8 +200,8 @@ static int perform_curl_io_and_complete(int count, bool bounded) } handle = msg->easy_handle; - ret = 404; - curl_ret = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &ret); + response = 404; + curl_ret = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response); if (curl_ret != CURLE_OK) { continue; } @@ -363,45 +221,21 @@ static int perform_curl_io_and_complete(int count, bool bounded) /* Get error code from easy handle and augment it if * completing the download encounters further problems. */ - curl_ret = msg->data.result; - curl_ret = swupd_download_file_complete(curl_ret, file); + curl_ret = swupd_download_file_complete(msg->data.result, &file->file); - /* The easy handle may have an error set, even if the server returns - * HTTP 200, so retry the download for this case. */ - if (ret == 200 && curl_ret != CURLE_OK) { - fprintf(stderr, "Error for %s download: %s\n", file->hash, - curl_easy_strerror(msg->data.result)); - failed = list_prepend_data(failed, file); - } else if (ret == 200) { - /* When both web server and CURL report success, only then - * proceed to uncompress. */ - if (untar_full_download(file)) { - fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n", - file->hash, state_dir); - failed = list_prepend_data(failed, file); - } - } else if (ret == 0) { - /* When using the FILE:// protocol, 0 indicates success. - * Otherwise, it means the web server hasn't responded yet. - */ - if (local_download) { - if (untar_full_download(file)) { - fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n", - file->hash, state_dir); - failed = list_prepend_data(failed, file); - } - } else { - fprintf(stderr, "Error for %s download: No response received\n", - file->hash); - failed = list_prepend_data(failed, file); + /* if the server returns HTTP 200 the download is sucessful. + * When using the FILE:// protocol, 0 also indicates success. */ + if ((response == 200 && curl_ret == CURLE_OK) || (local_download && response == 0)) { + if (!h->success_cb(file->data)) { + // Retry download if cb return is false. File probably corrupted + + h->failed = list_prepend_data(h->failed, file); } } else { - fprintf(stderr, "Error for %s download: Received %ld response\n", file->hash, ret); - failed = list_prepend_data(failed, file); - - unlink_all_staged_content(file); + h->failed = list_prepend_data(h->failed, file); + fprintf(stderr, "Error for %s download: Response %ld - %s\n", + file->file.path, response, curl_easy_strerror(msg->data.result)); } - free_string(&file->staging); /* NOTE: Intentionally no removal of file from hashmap. All * needed files need determined and queued in one complete @@ -412,19 +246,14 @@ static int perform_curl_io_and_complete(int count, bool bounded) * and HASH and staged to the _multiple_ filenames with that * hash. */ - curl_multi_remove_handle(mcurl, handle); + curl_multi_remove_handle(h->mcurl, handle); curl_easy_cleanup(handle); file->curl = NULL; - - /* "bounded" is false when the remainder of the multi-stack is - * to be processed, ignoring the hysteresis bound. */ - if (bounded) { - count--; - } + count++; /* A response has been completely processed by this point, so * make the stack shorter. */ - mcurl_size--; + h->mcurl_size--; } return 0; @@ -437,28 +266,28 @@ static int perform_curl_io_and_complete(int count, bool bounded) * add new transfer up until the queue reaches the high threshold. At this point * we don't return to the caller and instead process the queue until its len * gets below the low threshold */ -static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low) +static int poll_fewer_than(struct swupd_curl_parallel_handle *h, size_t xfer_queue_high, size_t xfer_queue_low) { CURLMcode curlm_ret; int running; - curlm_ret = curl_multi_perform(mcurl, &running); + curlm_ret = curl_multi_perform(h->mcurl, &running); if (curlm_ret != CURLM_OK) { return -1; } // There are not enough transfers in queue yet - if (mcurl_size <= xfer_queue_high) { + if (h->mcurl_size <= xfer_queue_high) { return 0; } /* When the high bound for the hysteresis is reached, process events * until the low bound is reached */ - while (mcurl_size > xfer_queue_low) { + while (h->mcurl_size > xfer_queue_low) { int numfds = 0; // Wait for activity on the multi-stack - curlm_ret = curl_multi_wait(mcurl, NULL, 0, 500, &numfds); + curlm_ret = curl_multi_wait(h->mcurl, NULL, 0, 500, &numfds); if (curlm_ret != CURLM_OK) { return -1; } @@ -466,7 +295,7 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low) /* Either a timeout was hit, or no events to process. * Note: curl_multi_wait() never sets numfds < 0 */ if (numfds == 0) { - curlm_ret = curl_multi_perform(mcurl, &running); + curlm_ret = curl_multi_perform(h->mcurl, &running); if (curlm_ret != CURLM_OK) { return -1; } @@ -479,21 +308,21 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low) } // Do more work before processing the queue - curlm_ret = curl_multi_perform(mcurl, &running); + curlm_ret = curl_multi_perform(h->mcurl, &running); if (curlm_ret != CURLM_OK) { return -1; } // Instead of using "numfds" as a hint for how many transfers // to process, try to drain the queue to the lower bound. - int remaining = mcurl_size - xfer_queue_low; + int remaining = h->mcurl_size - xfer_queue_low; - if (perform_curl_io_and_complete(remaining, true) != 0) { + if (perform_curl_io_and_complete(h, remaining) != 0) { return -1; } // Do more work, in preparation for the next curl_multi_wait() - curlm_ret = curl_multi_perform(mcurl, &running); + curlm_ret = curl_multi_perform(h->mcurl, &running); if (curlm_ret != CURLM_OK) { return -1; } @@ -511,38 +340,22 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low) // activity is reported the next BUFFER times. When an error is // returned, the most recent attempted transfer is added to the failed // list, later to be retried. - if (xfer_queue_low != xfer_queue_high && mcurl_size > xfer_queue_high + 1 + BUFFER) { + if (xfer_queue_low != xfer_queue_high && h->mcurl_size > xfer_queue_high + 1 + BUFFER) { return -1; } return 0; } -extern int nonpack; - -/* full_download() attempts to enqueue a file for later asynchronous download - - NOTE: See swupd_curl_get_file() for single file synchronous downloads. */ -void full_download(struct file *file) +static int process_download(struct swupd_curl_parallel_handle *h, struct multi_curl_file *file) { - char *url = NULL; CURL *curl = NULL; - int ret = -EFULLDOWNLOAD; - char *filename = NULL; CURLMcode curlm_ret = CURLM_OK; CURLcode curl_ret = CURLE_OK; - file->fh = NULL; - ret = swupd_curl_hashmap_insert(file); - if (ret > 0) { /* no download needed */ - /* File already exists - report success */ - ret = 0; - goto out_good; - } else if (ret < 0) { /* error */ - goto out_bad; - } /* else (ret == 0) download needed */ - - /* If we get here the pack is missing a file so we have to download it */ - nonpack++; + //TODO: Add support to download resume + // clean up in case any prior download failed in a partial state + unlink(file->file.path); curl = curl_easy_init(); if (curl == NULL) { @@ -550,76 +363,164 @@ void full_download(struct file *file) } file->curl = curl; - ret = poll_fewer_than(MAX_XFER, MAX_XFER_BOTTOM); - if (ret != 0) { - goto out_bad; - } - - string_or_die(&url, "%s/%i/files/%s.tar", content_url, file->last_change, file->hash); - - string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash); - file->staging = filename; - - curl_ret = curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - if (curl_ret != CURLE_OK) { - goto out_bad; - } - curl_ret = curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)file); if (curl_ret != CURLE_OK) { goto out_bad; } - curl_ret = swupd_download_file_start(file); + curl_ret = swupd_download_file_create(&file->file); if (curl_ret != CURLE_OK) { goto out_bad; } - curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->fh); + curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->file.fh); if (curl_ret != CURLE_OK) { goto out_bad; } - curl_ret = swupd_curl_set_basic_options(curl, url); + curl_ret = swupd_curl_set_basic_options(curl, file->url); if (curl_ret != CURLE_OK) { goto out_bad; } - curlm_ret = curl_multi_add_handle(mcurl, curl); + curlm_ret = curl_multi_add_handle(h->mcurl, curl); if (curlm_ret != CURLM_OK) { goto out_bad; } /* The current multi-stack size is not exposed by libcurl, so we track * it with a counter variable. */ - mcurl_size++; + h->mcurl_size++; - ret = poll_fewer_than(MAX_XFER + 10, MAX_XFER); - if (ret != 0) { - goto out_bad; - } - - ret = 0; - goto out_good; + return poll_fewer_than(h, h->max_xfer, h->max_xfer_bottom); out_bad: - failed = list_prepend_data(failed, file); - free_curl_list_data(file); - free_string(&filename); -out_good: - free_string(&url); + return -1; } -struct list *end_full_download(void) +/* + * Enqueue a file to be downloaded. If the number of current downloads is higher + * than max_xfer, this function will be blocked for downloads until the number of + * current downloads reach max_xfer_bottom. + * + * Parameters: + * - handle: Handle created with swupd_curl_parallel_download_start(). + * - url: The url to be downloaded. + * - filename: Full path of the filename to save the download content. + * - hash: Optional hex string with hash to be used as unique identifier of this + * file. If NULL, filename will be used as the identifier. String MUST contain + * only characters in '0123456789abcdef'. + * - data: User data to be informed to success_cb(). + * + * Note: This function MAY be blocked. + */ +int swupd_curl_parallel_download_enqueue(void *handle, const char *url, const char *filename, const char *hash, void *data) { - fprintf(stderr, "Finishing download of update content...\n"); + struct swupd_curl_parallel_handle *h; + struct multi_curl_file *file; - if (poll_fewer_than(0, 0) == 0) { - // The multi-stack is now emptied. - perform_curl_io_and_complete(1, false); + if (!handle) { + fprintf(stderr, "Invalid parallel download handle\n"); + return -1; + } + h = handle; + + file = calloc(1, sizeof(struct multi_curl_file)); + file->file.path = strdup_or_die(filename); + file->url = strdup_or_die(url); + file->data = data; + if (hash) { + file->hash = hash; + file->hash_key = HASH_TO_KEY(hash); + } else { + file->hash_key = hashmap_hash_from_string(filename); } - hashmap_free_hash_and_data(swupd_curl_hashmap, free_curl_list_data); - swupd_curl_hashmap = NULL; + if (!hashmap_put(h->curl_hashmap, file)) { + // hash already in download queue + free_curl_file(h, file); + return 0; + } - curl_multi_cleanup(mcurl); - return failed; + return process_download(h, file); +} + +/* + * Finish all pending downloads and free memory allocated by parallel download + * handler. + * + * Parameters: + * - handle: Handle created with swupd_curl_parallel_download_start(). + * - num_downloads: Optional int pointer to be filled with the number of + * files enqueued for download using this handler. Include failed downloads. + * + * Note: This function MAY be blocked. + */ +int swupd_curl_parallel_download_end(void *handle, int *num_downloads) +{ + struct swupd_curl_parallel_handle *h; + struct multi_curl_file *file; + int i, downloads = 0; + struct list *l; + bool retry = true; + + if (!handle) { + fprintf(stderr, "Invalid parallel download handle\n"); + return -1; + } + + h = handle; + fprintf(stderr, "Finishing download of update content...\n"); + + while (poll_fewer_than(h, 0, 0) == 0 && retry) { + retry = false; + + // The multi-stack is now emptied. + perform_curl_io_and_complete(h, h->mcurl_size); + + //Retry failed downloads + for (l = h->failed; l;) { + struct multi_curl_file *file = l->data; + + if (file->retries < MAX_RETRIES) { + struct list *next; + + file->retries++; + + //Remove item + if (l == h->failed) { + h->failed = l->next; + } + next = l->next; + list_free_item(l, NULL); + l = next; + + fprintf(stderr, "Starting download retry #%d for %s\n", file->retries, file->url); + process_download(h, file); + retry = true; + continue; + } + l = l->next; + } + if (retry) { + sleep(h->timeout); + h->timeout *= 2; + } + } + + curl_multi_cleanup(h->mcurl); + + list_free_list(h->failed); + + HASHMAP_FOREACH(h->curl_hashmap, i, l, file) + { + downloads++; + free_curl_file(h, file); + } + hashmap_free(h->curl_hashmap); + + free(h); + + if (num_downloads) { + *num_downloads = downloads; + } + return 0; } diff --git a/src/fullfile.c b/src/fullfile.c index 6a00ee1e..cb7a9413 100644 --- a/src/fullfile.c +++ b/src/fullfile.c @@ -21,11 +21,53 @@ */ #define _GNU_SOURCE +#include #include #include "swupd.h" -static struct list *download_loop(struct list *files, bool free_list) +/* hysteresis thresholds */ +#define MAX_XFER 25 +#define MAX_XFER_BOTTOM 15 + +static void download_mix_file(struct file *file) +{ + char *url, *filename; + + string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash); + string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash); + + /* Mix content is local, so don't queue files up for curl downloads */ + if (link_or_rename(url, filename) == 0) { + untar_full_download(file); + } else { + fprintf(stderr, "Failed to copy local mix file: %s\n", file->staging); + } + + free_string(&url); + free_string(&filename); +} + +static void download_file(void *download_handle, struct file *file) +{ + char *url, *filename; + char *targetfile; + struct stat stat; + + string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash); + + // Just download if the target file doesn't exist + if (lstat(targetfile, &stat) != 0 || !verify_file(file, targetfile)) { + string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash); + string_or_die(&url, "%s/%i/files/%s.tar", content_url, file->last_change, file->hash); + swupd_curl_parallel_download_enqueue(download_handle, url, filename, file->hash, file); + free_string(&url); + free_string(&filename); + } + free_string(&targetfile); +} + +static int download_loop(void *download_handle, struct list *files, int *num_downloads) { struct list *iter; unsigned int complete = 0; @@ -43,70 +85,52 @@ static struct list *download_loop(struct list *files, bool free_list) continue; } - /* Mix content is local, so don't queue files up for curl downloads */ if (file->is_mix) { - char *url; - - if (file->staging) { - free_string(&file->staging); - } - - string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash); - string_or_die(&file->staging, "%s/download/.%s.tar", state_dir, file->hash); - - if (link_or_rename(url, file->staging) != 0) { - fprintf(stderr, "Failed to copy local mix file: %s\n", file->staging); - continue; - } - untar_full_download(file); - free_string(&url); - continue; + download_mix_file(file); + } else { + download_file(download_handle, file); } - full_download(file); print_progress(complete, list_length); } print_progress(list_length, list_length); /* Force out 100% */ printf("\n"); - if (free_list) { - list_free_list(files); + return swupd_curl_parallel_download_end(download_handle, num_downloads); +} + +static bool download_successful(void *data) +{ + if (!data) { + return false; } - return end_full_download(); + if (untar_full_download(data) != 0) { + fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n", + ((struct file *)data)->hash, state_dir); + } + return true; } /* - * Download all fullfiles from the files list. - * Return 0 on success or a negative number if any file was not downloaded successfully + * Download fullfiles from the list of files. + * + * Return 0 on success or a negative number or errors. */ -int download_fullfiles(struct list *files, int num_retries, int timeout) +int download_fullfiles(struct list *files, int *num_downloads) { - int ret; - int retries = 0; + void *download_handle; - while (files) { - ret = start_full_download(); - if (ret != 0) { - /* If we hit this point, the network is accessible but we were - * unable to download the needed files. This is a terminal error - * and we need good logging */ - return -EFULLDOWNLOAD; - } - - files = download_loop(files, retries > 0); - - /* Set retries only if failed downloads exist, and only retry a fixed - amount of × */ - if (list_head(files)) { - if (retries >= num_retries) { - list_free_list(files); - return -EFULLDOWNLOAD; - } - - increment_retries(&retries, &timeout); - fprintf(stderr, "Starting download retry #%d\n", retries); - } + if (!files) { + return 0; } - return 0; + download_handle = swupd_curl_parallel_download_start(MAX_XFER, MAX_XFER_BOTTOM, download_successful); + if (!download_handle) { + /* If we hit this point, the network is accessible but we were + * unable to download the needed files. This is a terminal error + * and we need good logging */ + return -EFULLDOWNLOAD; + } + + return download_loop(download_handle, files, num_downloads); } diff --git a/src/hashmap.c b/src/hashmap.c index edb7be46..f581ce84 100644 --- a/src/hashmap.c +++ b/src/hashmap.c @@ -31,6 +31,18 @@ #define HASH_MASK(bits) (HASH_SIZE(bits) - 1) +size_t hashmap_hash_from_string(const char *key) +{ + size_t hash = 0; + + while (key) { + hash = hash * 29 /* a prime number */ + *key; + key++; + } + + return hash; +} + static inline struct list **get_hashmap_list(struct hashmap *hashmap, const void *data) { return &hashmap->map[hashmap->hash(data) || HASH_MASK(hashmap->mask_bits)]; diff --git a/src/hashmap.h b/src/hashmap.h index ea3e47d8..2d1678db 100644 --- a/src/hashmap.h +++ b/src/hashmap.h @@ -65,6 +65,10 @@ void hashmap_free(struct hashmap *hashmap); */ void hashmap_free_hash_and_data(struct hashmap *hashmap, free_data_fn_t free_data); +/* + * Hash function helper to calculate a good hash to be used with strings. + */ +size_t hashmap_hash_from_string(const char *key); /* * Loop through all elements in the hashmap. * diff --git a/src/helpers.c b/src/helpers.c index 442c00c6..b08cf79b 100644 --- a/src/helpers.c +++ b/src/helpers.c @@ -38,6 +38,7 @@ #include "config.h" #include "signature.h" +#include "swupd-build-variant.h" #include "swupd.h" void check_root(void) @@ -953,3 +954,141 @@ int link_or_rename(const char *orig, const char *dest) return 0; } + +/* list the tarfile content, and verify it contains only one line equal to the expected hash. + * loop through all the content to detect the case where archive contains more than one file. + */ +static int check_tarfile_content(struct file *file, const char *tarfilename) +{ + int err; + char *tarcommand; + FILE *tar; + int count = 0; + + string_or_die(&tarcommand, TAR_COMMAND " -tf %s/download/%s.tar 2> /dev/null", state_dir, file->hash); + + err = access(tarfilename, R_OK); + if (err) { + goto free_tarcommand; + } + + tar = popen(tarcommand, "r"); + if (tar == NULL) { + err = -1; + goto free_tarcommand; + } + + while (!feof(tar)) { + char *c; + char buffer[PATH_MAXLEN]; + + if (fgets(buffer, PATH_MAXLEN, tar) == NULL) { + if (count != 1) { + err = -1; + } + break; + } + + c = strchr(buffer, '\n'); + if (c) { + *c = 0; + } + if (c && (c != buffer) && (*(c - 1) == '/')) { + /* strip trailing '/' from directory tar */ + *(c - 1) = 0; + } + if (strcmp(buffer, file->hash) != 0) { + err = -1; + break; + } + count++; + } + + pclose(tar); +free_tarcommand: + free_string(&tarcommand); + + return err; +} + +/* This function will break if the same HASH.tar full file is downloaded + * multiple times in parallel. */ +int untar_full_download(void *data) +{ + struct file *file = data; + char *tarfile; + char *tar_dotfile; + char *targetfile; + struct stat stat; + int err; + + string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash); + string_or_die(&tarfile, "%s/download/%s.tar", state_dir, file->hash); + string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash); + + /* If valid target file already exists, we're done. + * NOTE: this should NEVER happen given the checking that happens + * ahead of queueing a download. But... */ + if (lstat(targetfile, &stat) == 0) { + if (verify_file(file, targetfile)) { + unlink(tar_dotfile); + unlink(tarfile); + free_string(&tar_dotfile); + free_string(&tarfile); + free_string(&targetfile); + return 0; + } else { + unlink(tarfile); + unlink(targetfile); + } + } else if (lstat(tarfile, &stat) == 0) { + /* remove tar file from possible past failure */ + unlink(tarfile); + } + + err = rename(tar_dotfile, tarfile); + if (err) { + free_string(&tar_dotfile); + goto exit; + } + free_string(&tar_dotfile); + + err = check_tarfile_content(file, tarfile); + if (err) { + goto exit; + } + + /* modern tar will automatically determine the compression type used */ + char *outputdir; + string_or_die(&outputdir, "%s/staged", state_dir); + err = extract_to(tarfile, outputdir); + free_string(&outputdir); + if (err) { + fprintf(stderr, "ignoring tar extract failure for fullfile %s.tar (ret %d)\n", + file->hash, err); + goto exit; + /* TODO: respond to ARCHIVE_RETRY error codes + * libarchive returns ARCHIVE_RETRY when tar extraction fails but the + * operation is retry-able. We need to determine if it is worth our time + * to retry in these situations. */ + } else { + /* Only unlink when tar succeeded, so we can examine the tar file + * in the failure case. */ + unlink(tarfile); + } + + err = lstat(targetfile, &stat); + if (!err && !verify_file(file, targetfile)) { + /* Download was successful but the hash was bad. This is fatal*/ + fprintf(stderr, "Error: File content hash mismatch for %s (bad server data?)\n", targetfile); + exit(EXIT_FAILURE); + } + +exit: + free_string(&tarfile); + free_string(&targetfile); + if (err) { + unlink_all_staged_content(file); + } + return err; +} diff --git a/src/swupd.h b/src/swupd.h index 333c32b4..ca0086d4 100644 --- a/src/swupd.h +++ b/src/swupd.h @@ -134,10 +134,7 @@ struct file { struct file *peer; /* same file in another manifest */ struct header *header; - - char *staging; /* output name used during download & staging */ - CURL *curl; /* curl handle if downloading */ - FILE *fh; /* file written into during downloading */ + char *staging; }; struct filerecord { @@ -278,13 +275,10 @@ static inline void account_delta_miss(void) extern void print_statistics(int version1, int version2); -extern int download_fullfiles(struct list *files, int num_retries, int timeout); +extern int download_fullfiles(struct list *files, int *num_downloads); extern int download_subscribed_packs(struct list *subs, struct manifest *mom, bool required, bool resume_ok); extern void apply_deltas(struct manifest *current_manifest); -extern void full_download(struct file *file); -extern int start_full_download(void); -extern struct list *end_full_download(void); extern int untar_full_download(void *data); extern int do_staging(struct file *file, struct manifest *manifest); @@ -293,18 +287,19 @@ extern int rename_staged_file_to_final(struct file *file); extern int update_device_latest_version(int version); +/* curl.c */ + extern int swupd_curl_init(void); extern void swupd_curl_deinit(void); extern double swupd_curl_query_content_size(char *url); -extern CURLcode swupd_download_file_start(struct file *file); -extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file); extern int swupd_curl_get_file(const char *url, char *filename); extern int swupd_curl_get_file_full(const char *url, char *filename, struct curl_file_data *file_data, bool resume_ok); -#define SWUPD_CURL_LOW_SPEED_LIMIT 1 -#define SWUPD_CURL_CONNECT_TIMEOUT 30 -#define SWUPD_CURL_RCV_TIMEOUT 120 -extern CURLcode swupd_curl_set_basic_options(CURL *curl, const char *url); +/* download.c */ +typedef bool (*swupd_curl_download_cb)(void *data); +extern void *swupd_curl_parallel_download_start(size_t max_xfer, size_t max_xfer_bottom, swupd_curl_download_cb success_cb); +extern int swupd_curl_parallel_download_enqueue(void *handle, const char *url, const char *filename, const char *hash, void *data); +extern int swupd_curl_parallel_download_end(void *handle, int *num_downloads); extern void free_subscriptions(struct list **subs); extern void read_subscriptions(struct list **subs); diff --git a/src/update.c b/src/update.c index 43dfc910..03678277 100644 --- a/src/update.c +++ b/src/update.c @@ -84,7 +84,7 @@ static int update_loop(struct list *updates, struct manifest *server_manifest) struct file *file; struct list *iter; - ret = download_fullfiles(updates, MAX_TRIES, 10); + ret = download_fullfiles(updates, &nonpack); if (ret) { fprintf(stderr, "ERROR: Could not download all files, aborting update\n"); } diff --git a/src/verify.c b/src/verify.c index 1de1343a..c236a4c9 100644 --- a/src/verify.c +++ b/src/verify.c @@ -404,7 +404,7 @@ static int get_required_files(struct manifest *official_manifest, struct list *s return 0; } - ret = download_fullfiles(official_manifest->files, MAX_TRIES, 10); + ret = download_fullfiles(official_manifest->files, NULL); if (ret) { fprintf(stderr, "Error: Unable to download neccessary files for this OS release\n"); }