clearlinux/swupd-client
2
0
Fork 0
mirror of https://github.com/clearlinux/swupd-client.git synced 2026-09-07 14:11:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

download: Make parallel downloads generic

Browse Source 
Remove all information about the content of the file that the parallel download
element is downloading. Move everything related to fullfile downloads to
fullfile.c and make the parallel downloader ready to be used for other
download cases.

There were some changes in this commit that were really hard to split
in individual commits, like:
 - Create a struct curl_file for curl specific information instead of
   adding extra fields to a generic file struct
 - Rename swupd_download_file_start() to swupd_download_file_create(),
   because that's what this function is doing.
This commit is contained in:
Otavio Pontes
2018-07-13 16:15:41 -07:00
committed by Matthew Johnson
parent f0d3f333ef
commit 0382e8d064
11 changed files with 538 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bundle.c
+1 -1
View File
@@ -777,7 +777,7 @@ download_subscribed_packs:
/* step 2.5: Download missing files */
grabtime_start(&times, "Download missing files");
ret = download_fullfiles(to_install_files, MAX_TRIES, 10);
ret = download_fullfiles(to_install_files, NULL);
if (ret) {
fprintf(stderr, "ERROR: Could not download some files from bundles, aborting bundle installation.\n");
goto out;
src/curl-internal.h
+43
View File
@@ -0,0 +1,43 @@
#ifndef __INCLUDE_GUARD_CURL_INTERNAL_H
#define __INCLUDE_GUARD_CURL_INTERNAL_H
/* Define internal curl functions to be used by curl.c and download.c.
* Not expected to be used by users of swupd_curl_* functions.
*/
#include <curl/curl.h>
#ifdef __cplusplus
extern "C" {
#endif
struct curl_file {
char *path; /* output name used during download */
FILE *fh; /* file written into during downloading */
};
/*
* Create a file in disk to start the download.
*/
extern CURLcode swupd_download_file_create(struct curl_file *file);
/*
* Create a file in disk to start the download, using append mode.
*/
extern CURLcode swupd_download_file_append(struct curl_file *file);
/*
* Close file after the download is finished.
*/
extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct curl_file *file);
/*
* Set swupd default basic options to curl handler.
*/
extern CURLcode swupd_curl_set_basic_options(CURL *curl, const char *url);
#ifdef __cplusplus
}
#endif
#endif
src/curl.c
+17 -12
View File
@@ -44,8 +44,13 @@
#include <unistd.h>
#include "config.h"
#include "curl-internal.h"
#include "swupd.h"
#define SWUPD_CURL_LOW_SPEED_LIMIT 1
#define SWUPD_CURL_CONNECT_TIMEOUT 30
#define SWUPD_CURL_RCV_TIMEOUT 120
static CURL *curl = NULL;
/* alternative CA Path */
@@ -227,34 +232,34 @@ static size_t swupd_download_file_to_memory(void *ptr, size_t size, size_t nmemb
return data_len;
}
CURLcode swupd_download_file_start(struct file *file)
CURLcode swupd_download_file_create(struct curl_file *file)
{
file->fh = fopen(file->staging, "w");
file->fh = fopen(file->path, "w");
if (!file->fh) {
fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
file->staging, strerror(errno));
file->path, strerror(errno));
return CURLE_WRITE_ERROR;
}
return CURLE_OK;
}
CURLcode swupd_download_file_append(struct file *file)
CURLcode swupd_download_file_append(struct curl_file *file)
{
file->fh = fopen(file->staging, "a");
file->fh = fopen(file->path, "a");
if (!file->fh) {
fprintf(stderr, "Cannot open file for append \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
file->staging, strerror(errno));
file->path, strerror(errno));
return CURLE_WRITE_ERROR;
}
return CURLE_OK;
}
CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file)
CURLcode swupd_download_file_complete(CURLcode curl_ret, struct curl_file *file)
{
if (file->fh) {
if (fclose(file->fh)) {
fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
file->staging, strerror(errno));
file->path, strerror(errno));
if (curl_ret == CURLE_OK) {
curl_ret = CURLE_WRITE_ERROR;
}
@@ -284,7 +289,7 @@ int swupd_curl_get_file_full(const char *url, char *filename,
CURLcode curl_ret;
long ret = 0;
int err = -1;
struct file local = { 0 };
struct curl_file local = { 0 };
bool local_download = strncmp(url, "file://", 7) == 0;
if (!curl) {
@@ -297,7 +302,7 @@ restart_download:
// normal file download
struct stat stat;
local.staging = filename;
local.path = filename;
if (resume_ok && resume_download_supported && lstat(filename, &stat) == 0) {
curl_ret = curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)stat.st_size);
@@ -307,7 +312,7 @@ restart_download:
curl_ret = swupd_download_file_append(&local);
} else {
curl_ret = swupd_download_file_start(&local);
curl_ret = swupd_download_file_create(&local);
}
curl_ret = curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)&local);
@@ -439,7 +444,7 @@ exit:
/*
* Download a single file SYNCHRONOUSLY
*
* Returns: Zero on success or a standard < 0 status code on errors.
*/
int swupd_curl_get_file(const char *url, char *filename)
src/download.c
+237 -336
View File
@@ -29,29 +29,28 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "config.h"
#include "curl-internal.h"
#include "hashmap.h"
#include "swupd-build-variant.h"
#include "swupd.h"
/* This file provides a managed download facility for code that needs a set of
* update tar files. Such code starts with one call to "start_full_download()",
* then makes a series of calls to "full_download()" once per desired file, and
* then finishes with one call to "end_full_download()" which will block until
* the previously queued downloads are completed and and untarred.
*/
#define MAX_RETRIES 5
#define RETRY_TIMEOUT 10
static CURLM *mcurl = NULL;
static size_t mcurl_size = 0;
static struct list *failed = NULL;
/* This file provides a managed download facility for parallelizing download
* files. To use this functionality, create a handler using
* swupd_curl_parallel_download_start(), enqueue files using
* swupd_curl_parallel_download_enqueue() and finish the process using
* swupd_curl_parallel_download_end(), which will block until the previously
* queued downloads are completed.
*/
/*
* The following represents a hash data structure used inside download.c
* to track file downloads asynchonously via libcurl. In the past a
* to track file downloads in parallel via libcurl. In the past a
* single linked list was used, but we may have a very large number of
* files and repeatedly scan the list could become expensive. The hashmap
* gives info on what HASH.tar files will be downloaded. A de-duplicated
@@ -74,252 +73,111 @@ static struct list *failed = NULL;
*/
#define HASH_TO_KEY(hash) (HASH_VALUE(hash[0]) << 4 | HASH_VALUE(hash[1]))
static struct hashmap *swupd_curl_hashmap = NULL;
struct swupd_curl_parallel_handle {
int timeout; /* Retry timeout */
size_t mcurl_size, max_xfer, max_xfer_bottom; /* hysteresis parameters */
/* try to insert the file into the hashmap download queue
* returns 1 if no download is needed
* returns 0 if download is needed
* returns -1 if error */
static int swupd_curl_hashmap_insert(struct file *file)
{
char *tar_dotfile;
char *targetfile;
struct stat stat;
CURLM *mcurl; /* Curl handle */
struct list *failed; /* List of failed downloads */
struct hashmap *curl_hashmap; /* Hashmap mentioned above */
swupd_curl_download_cb success_cb; /* Callback to sucess function */
};
if (!hashmap_put(swupd_curl_hashmap, file)) {
// hash already in download queue
return 1;
}
/*
* Struct represinting a single file being downloaded.
*/
struct multi_curl_file {
struct curl_file file; /* Curl file information */
// if valid target file is already here, no need to download
string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash);
char retries; /* Number of retried performed so far */
CURL *curl; /* curl handle if downloading */
char *url; /* The url to be downloaded from */
size_t hash_key; /* hash_key of this file */
const char *hash; /* Unique identifier of this file. */
if (lstat(targetfile, &stat) == 0) {
/* file exists */
if (verify_file(file, targetfile)) {
/* hash matches, no download necessary */
free_string(&targetfile);
return 1;
} else {
/* hash mismatch, remove the staged file to enable re-download */
unlink(targetfile);
}
}
free_string(&targetfile);
// hash not in queue and not present in staged
// clean up in case any prior download failed in a partial state
string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash);
unlink(tar_dotfile);
free_string(&tar_dotfile);
return 0;
}
/* hysteresis thresholds */
static size_t MAX_XFER = 25;
static size_t MAX_XFER_BOTTOM = 15;
void *data; /* user's data */
};
static bool file_hash_cmp(const void *a, const void *b)
{
const struct file *fa = a;
const struct file *fb = b;
const struct multi_curl_file *fa = a;
const struct multi_curl_file *fb = b;
if (!fa->hash || !fb->hash) {
return strcmp(fa->file.path, fb->file.path) == 0;
}
return hash_equal(fa->hash, fb->hash);
}
static size_t file_hash_value(const void *data)
{
return HASH_VALUE(((struct file *)data)->hash[0]);
return ((struct multi_curl_file *)data)->hash_key;
}
int start_full_download(void)
static void free_curl_file(struct swupd_curl_parallel_handle *h, struct multi_curl_file *file)
{
if (swupd_curl_hashmap) {
return -1;
}
failed = NULL;
swupd_curl_hashmap = hashmap_new(SWUPD_CURL_HASH_BUCKETS, file_hash_cmp, file_hash_value);
mcurl = curl_multi_init();
if (mcurl == NULL) {
return -1;
}
curl_multi_setopt(mcurl, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX | CURLPIPE_HTTP1);
fprintf(stderr, "Starting download of remaining update content. This may take a while...\n");
return 0;
}
static void free_curl_list_data(void *data)
{
struct file *file = (struct file *)data;
CURL *curl = file->curl;
(void)swupd_download_file_complete(CURLE_OK, file);
swupd_download_file_complete(CURLE_OK, &file->file);
free_string(&file->url);
free_string(&file->file.path);
if (curl != NULL) {
/* Must remove handle out of multi queue first!*/
curl_multi_remove_handle(mcurl, curl);
curl_multi_remove_handle(h->mcurl, curl);
curl_easy_cleanup(curl);
file->curl = NULL;
}
free(file);
}
/* list the tarfile content, and verify it contains only one line equal to the expected hash.
* loop through all the content to detect the case where archive contains more than one file.
/*
* Start a parallel download element.
*
* Parameters:
* - max_xfer: The maximum number of simultaneos downloads.
* - max_xfer_bottom: Minimum number of simultaneos downloads before starting
* - sucess_cb(): Callback called for each sucessfull download.
*
* Parallel download handler will retry MAX_TRIES times to download each file,
* ading a timeout between each try.
*
* Note: This function is non-blocking.
*/
static int check_tarfile_content(struct file *file, const char *tarfilename)
void *swupd_curl_parallel_download_start(size_t max_xfer, size_t max_xfer_bottom, swupd_curl_download_cb success_cb)
{
int err;
char *tarcommand;
FILE *tar;
int count = 0;
struct swupd_curl_parallel_handle *h = calloc(1, sizeof(struct swupd_curl_parallel_handle));
string_or_die(&tarcommand, TAR_COMMAND " -tf %s/download/%s.tar 2> /dev/null", state_dir, file->hash);
err = access(tarfilename, R_OK);
if (err) {
goto free_tarcommand;
h->mcurl = curl_multi_init();
if (h->mcurl == NULL) {
goto error;
}
tar = popen(tarcommand, "r");
if (tar == NULL) {
err = -1;
goto free_tarcommand;
}
curl_multi_setopt(h->mcurl, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX | CURLPIPE_HTTP1);
while (!feof(tar)) {
char *c;
char buffer[PATH_MAXLEN];
h->max_xfer = max_xfer;
h->max_xfer_bottom = max_xfer_bottom;
h->success_cb = success_cb;
h->curl_hashmap = hashmap_new(SWUPD_CURL_HASH_BUCKETS, file_hash_cmp, file_hash_value);
h->timeout = RETRY_TIMEOUT;
if (fgets(buffer, PATH_MAXLEN, tar) == NULL) {
if (count != 1) {
err = -1;
}
break;
}
fprintf(stderr, "Starting download of remaining update content. This may take a while...\n");
c = strchr(buffer, '\n');
if (c) {
*c = 0;
}
if (c && (c != buffer) && (*(c - 1) == '/')) {
/* strip trailing '/' from directory tar */
*(c - 1) = 0;
}
if (strcmp(buffer, file->hash) != 0) {
err = -1;
break;
}
count++;
}
pclose(tar);
free_tarcommand:
free_string(&tarcommand);
return err;
return h;
error:
free(h);
return NULL;
}
/* This function will break if the same HASH.tar full file is downloaded
* multiple times in parallel. */
int untar_full_download(void *data)
{
struct file *file = data;
char *tarfile;
char *tar_dotfile;
char *targetfile;
struct stat stat;
int err;
string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash);
string_or_die(&tarfile, "%s/download/%s.tar", state_dir, file->hash);
string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash);
/* If valid target file already exists, we're done.
* NOTE: this should NEVER happen given the checking that happens
* ahead of queueing a download. But... */
if (lstat(targetfile, &stat) == 0) {
if (verify_file(file, targetfile)) {
unlink(tar_dotfile);
unlink(tarfile);
free_string(&tar_dotfile);
free_string(&tarfile);
free_string(&targetfile);
return 0;
} else {
unlink(tarfile);
unlink(targetfile);
}
} else if (lstat(tarfile, &stat) == 0) {
/* remove tar file from possible past failure */
unlink(tarfile);
}
err = rename(tar_dotfile, tarfile);
if (err) {
free_string(&tar_dotfile);
goto exit;
}
free_string(&tar_dotfile);
err = check_tarfile_content(file, tarfile);
if (err) {
goto exit;
}
/* modern tar will automatically determine the compression type used */
char *outputdir;
string_or_die(&outputdir, "%s/staged", state_dir);
err = extract_to(tarfile, outputdir);
free_string(&outputdir);
if (err) {
fprintf(stderr, "ignoring tar extract failure for fullfile %s.tar (ret %d)\n",
file->hash, err);
goto exit;
/* TODO: respond to ARCHIVE_RETRY error codes
* libarchive returns ARCHIVE_RETRY when tar extraction fails but the
* operation is retry-able. We need to determine if it is worth our time
* to retry in these situations. */
} else {
/* Only unlink when tar succeeded, so we can examine the tar file
* in the failure case. */
unlink(tarfile);
}
err = lstat(targetfile, &stat);
if (!err && !verify_file(file, targetfile)) {
/* Download was successful but the hash was bad. This is fatal*/
fprintf(stderr, "Error: File content hash mismatch for %s (bad server data?)\n", targetfile);
exit(EXIT_FAILURE);
}
exit:
free_string(&tarfile);
free_string(&targetfile);
if (err) {
unlink_all_staged_content(file);
}
return err;
}
/* Try to process at most COUNT messages from the curl multi-stack, and enforce
* the hysteresis when BOUNDED is true. The COUNT value is a best guess about
* the number of messages ready to process, because it may include file
* descriptors internal to libcurl as well (see curl_multi_wait(3)). */
static int perform_curl_io_and_complete(int count, bool bounded)
// Try to process at most COUNT messages from the curl multi-stack.
static int perform_curl_io_and_complete(struct swupd_curl_parallel_handle *h, int count)
{
CURLMsg *msg;
long ret;
long response;
CURLcode curl_ret;
while (count > 0) {
CURL *handle;
struct file *file;
struct multi_curl_file *file;
char *url = NULL;
bool local_download;
@@ -331,7 +189,7 @@ static int perform_curl_io_and_complete(int count, bool bounded)
* curl_multi_info_read() does not accept NULL for the second
* argument to indicate an unused value. */
int unused;
msg = curl_multi_info_read(mcurl, &unused);
msg = curl_multi_info_read(h->mcurl, &unused);
if (!msg) {
/* Either there were fewer than COUNT messages to
* process, or the multi-stack is now empty. */
@@ -342,8 +200,8 @@ static int perform_curl_io_and_complete(int count, bool bounded)
}
handle = msg->easy_handle;
ret = 404;
curl_ret = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &ret);
response = 404;
curl_ret = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response);
if (curl_ret != CURLE_OK) {
continue;
}
@@ -363,45 +221,21 @@ static int perform_curl_io_and_complete(int count, bool bounded)
/* Get error code from easy handle and augment it if
* completing the download encounters further problems. */
curl_ret = msg->data.result;
curl_ret = swupd_download_file_complete(curl_ret, file);
curl_ret = swupd_download_file_complete(msg->data.result, &file->file);
/* The easy handle may have an error set, even if the server returns
* HTTP 200, so retry the download for this case. */
if (ret == 200 && curl_ret != CURLE_OK) {
fprintf(stderr, "Error for %s download: %s\n", file->hash,
curl_easy_strerror(msg->data.result));
failed = list_prepend_data(failed, file);
} else if (ret == 200) {
/* When both web server and CURL report success, only then
* proceed to uncompress. */
if (untar_full_download(file)) {
fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n",
file->hash, state_dir);
failed = list_prepend_data(failed, file);
}
} else if (ret == 0) {
/* When using the FILE:// protocol, 0 indicates success.
* Otherwise, it means the web server hasn't responded yet.
*/
if (local_download) {
if (untar_full_download(file)) {
fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n",
file->hash, state_dir);
failed = list_prepend_data(failed, file);
}
} else {
fprintf(stderr, "Error for %s download: No response received\n",
file->hash);
failed = list_prepend_data(failed, file);
/* if the server returns HTTP 200 the download is sucessful.
* When using the FILE:// protocol, 0 also indicates success. */
if ((response == 200 && curl_ret == CURLE_OK) || (local_download && response == 0)) {
if (!h->success_cb(file->data)) {
// Retry download if cb return is false. File probably corrupted
h->failed = list_prepend_data(h->failed, file);
}
} else {
fprintf(stderr, "Error for %s download: Received %ld response\n", file->hash, ret);
failed = list_prepend_data(failed, file);
unlink_all_staged_content(file);
h->failed = list_prepend_data(h->failed, file);
fprintf(stderr, "Error for %s download: Response %ld - %s\n",
file->file.path, response, curl_easy_strerror(msg->data.result));
}
free_string(&file->staging);
/* NOTE: Intentionally no removal of file from hashmap. All
* needed files need determined and queued in one complete
@@ -412,19 +246,14 @@ static int perform_curl_io_and_complete(int count, bool bounded)
* and HASH and staged to the _multiple_ filenames with that
* hash. */
curl_multi_remove_handle(mcurl, handle);
curl_multi_remove_handle(h->mcurl, handle);
curl_easy_cleanup(handle);
file->curl = NULL;
/* "bounded" is false when the remainder of the multi-stack is
* to be processed, ignoring the hysteresis bound. */
if (bounded) {
count--;
}
count++;
/* A response has been completely processed by this point, so
* make the stack shorter. */
mcurl_size--;
h->mcurl_size--;
}
return 0;
@@ -437,28 +266,28 @@ static int perform_curl_io_and_complete(int count, bool bounded)
* add new transfer up until the queue reaches the high threshold. At this point
* we don't return to the caller and instead process the queue until its len
* gets below the low threshold */
static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low)
static int poll_fewer_than(struct swupd_curl_parallel_handle *h, size_t xfer_queue_high, size_t xfer_queue_low)
{
CURLMcode curlm_ret;
int running;
curlm_ret = curl_multi_perform(mcurl, &running);
curlm_ret = curl_multi_perform(h->mcurl, &running);
if (curlm_ret != CURLM_OK) {
return -1;
}
// There are not enough transfers in queue yet
if (mcurl_size <= xfer_queue_high) {
if (h->mcurl_size <= xfer_queue_high) {
return 0;
}
/* When the high bound for the hysteresis is reached, process events
* until the low bound is reached */
while (mcurl_size > xfer_queue_low) {
while (h->mcurl_size > xfer_queue_low) {
int numfds = 0;
// Wait for activity on the multi-stack
curlm_ret = curl_multi_wait(mcurl, NULL, 0, 500, &numfds);
curlm_ret = curl_multi_wait(h->mcurl, NULL, 0, 500, &numfds);
if (curlm_ret != CURLM_OK) {
return -1;
}
@@ -466,7 +295,7 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low)
/* Either a timeout was hit, or no events to process.
* Note: curl_multi_wait() never sets numfds < 0 */
if (numfds == 0) {
curlm_ret = curl_multi_perform(mcurl, &running);
curlm_ret = curl_multi_perform(h->mcurl, &running);
if (curlm_ret != CURLM_OK) {
return -1;
}
@@ -479,21 +308,21 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low)
}
// Do more work before processing the queue
curlm_ret = curl_multi_perform(mcurl, &running);
curlm_ret = curl_multi_perform(h->mcurl, &running);
if (curlm_ret != CURLM_OK) {
return -1;
}
// Instead of using "numfds" as a hint for how many transfers
// to process, try to drain the queue to the lower bound.
int remaining = mcurl_size - xfer_queue_low;
int remaining = h->mcurl_size - xfer_queue_low;
if (perform_curl_io_and_complete(remaining, true) != 0) {
if (perform_curl_io_and_complete(h, remaining) != 0) {
return -1;
}
// Do more work, in preparation for the next curl_multi_wait()
curlm_ret = curl_multi_perform(mcurl, &running);
curlm_ret = curl_multi_perform(h->mcurl, &running);
if (curlm_ret != CURLM_OK) {
return -1;
}
@@ -511,38 +340,22 @@ static int poll_fewer_than(size_t xfer_queue_high, size_t xfer_queue_low)
// activity is reported the next BUFFER times. When an error is
// returned, the most recent attempted transfer is added to the failed
// list, later to be retried.
if (xfer_queue_low != xfer_queue_high && mcurl_size > xfer_queue_high + 1 + BUFFER) {
if (xfer_queue_low != xfer_queue_high && h->mcurl_size > xfer_queue_high + 1 + BUFFER) {
return -1;
}
return 0;
}
extern int nonpack;
/* full_download() attempts to enqueue a file for later asynchronous download
- NOTE: See swupd_curl_get_file() for single file synchronous downloads. */
void full_download(struct file *file)
static int process_download(struct swupd_curl_parallel_handle *h, struct multi_curl_file *file)
{
char *url = NULL;
CURL *curl = NULL;
int ret = -EFULLDOWNLOAD;
char *filename = NULL;
CURLMcode curlm_ret = CURLM_OK;
CURLcode curl_ret = CURLE_OK;
file->fh = NULL;
ret = swupd_curl_hashmap_insert(file);
if (ret > 0) { /* no download needed */
/* File already exists - report success */
ret = 0;
goto out_good;
} else if (ret < 0) { /* error */
goto out_bad;
} /* else (ret == 0) download needed */
/* If we get here the pack is missing a file so we have to download it */
nonpack++;
//TODO: Add support to download resume
// clean up in case any prior download failed in a partial state
unlink(file->file.path);
curl = curl_easy_init();
if (curl == NULL) {
@@ -550,76 +363,164 @@ void full_download(struct file *file)
}
file->curl = curl;
ret = poll_fewer_than(MAX_XFER, MAX_XFER_BOTTOM);
if (ret != 0) {
goto out_bad;
}
string_or_die(&url, "%s/%i/files/%s.tar", content_url, file->last_change, file->hash);
string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash);
file->staging = filename;
curl_ret = curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
if (curl_ret != CURLE_OK) {
goto out_bad;
}
curl_ret = curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *)file);
if (curl_ret != CURLE_OK) {
goto out_bad;
}
curl_ret = swupd_download_file_start(file);
curl_ret = swupd_download_file_create(&file->file);
if (curl_ret != CURLE_OK) {
goto out_bad;
}
curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->fh);
curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->file.fh);
if (curl_ret != CURLE_OK) {
goto out_bad;
}
curl_ret = swupd_curl_set_basic_options(curl, url);
curl_ret = swupd_curl_set_basic_options(curl, file->url);
if (curl_ret != CURLE_OK) {
goto out_bad;
}
curlm_ret = curl_multi_add_handle(mcurl, curl);
curlm_ret = curl_multi_add_handle(h->mcurl, curl);
if (curlm_ret != CURLM_OK) {
goto out_bad;
}
/* The current multi-stack size is not exposed by libcurl, so we track
* it with a counter variable. */
mcurl_size++;
h->mcurl_size++;
ret = poll_fewer_than(MAX_XFER + 10, MAX_XFER);
if (ret != 0) {
goto out_bad;
}
ret = 0;
goto out_good;
return poll_fewer_than(h, h->max_xfer, h->max_xfer_bottom);
out_bad:
failed = list_prepend_data(failed, file);
free_curl_list_data(file);
free_string(&filename);
out_good:
free_string(&url);
return -1;
}
struct list *end_full_download(void)
/*
* Enqueue a file to be downloaded. If the number of current downloads is higher
* than max_xfer, this function will be blocked for downloads until the number of
* current downloads reach max_xfer_bottom.
*
* Parameters:
* - handle: Handle created with swupd_curl_parallel_download_start().
* - url: The url to be downloaded.
* - filename: Full path of the filename to save the download content.
* - hash: Optional hex string with hash to be used as unique identifier of this
* file. If NULL, filename will be used as the identifier. String MUST contain
* only characters in '0123456789abcdef'.
* - data: User data to be informed to success_cb().
*
* Note: This function MAY be blocked.
*/
int swupd_curl_parallel_download_enqueue(void *handle, const char *url, const char *filename, const char *hash, void *data)
{
fprintf(stderr, "Finishing download of update content...\n");
struct swupd_curl_parallel_handle *h;
struct multi_curl_file *file;
if (poll_fewer_than(0, 0) == 0) {
// The multi-stack is now emptied.
perform_curl_io_and_complete(1, false);
if (!handle) {
fprintf(stderr, "Invalid parallel download handle\n");
return -1;
}
h = handle;
file = calloc(1, sizeof(struct multi_curl_file));
file->file.path = strdup_or_die(filename);
file->url = strdup_or_die(url);
file->data = data;
if (hash) {
file->hash = hash;
file->hash_key = HASH_TO_KEY(hash);
} else {
file->hash_key = hashmap_hash_from_string(filename);
}
hashmap_free_hash_and_data(swupd_curl_hashmap, free_curl_list_data);
swupd_curl_hashmap = NULL;
if (!hashmap_put(h->curl_hashmap, file)) {
// hash already in download queue
free_curl_file(h, file);
return 0;
}
curl_multi_cleanup(mcurl);
return failed;
return process_download(h, file);
}
/*
* Finish all pending downloads and free memory allocated by parallel download
* handler.
*
* Parameters:
* - handle: Handle created with swupd_curl_parallel_download_start().
* - num_downloads: Optional int pointer to be filled with the number of
* files enqueued for download using this handler. Include failed downloads.
*
* Note: This function MAY be blocked.
*/
int swupd_curl_parallel_download_end(void *handle, int *num_downloads)
{
struct swupd_curl_parallel_handle *h;
struct multi_curl_file *file;
int i, downloads = 0;
struct list *l;
bool retry = true;
if (!handle) {
fprintf(stderr, "Invalid parallel download handle\n");
return -1;
}
h = handle;
fprintf(stderr, "Finishing download of update content...\n");
while (poll_fewer_than(h, 0, 0) == 0 && retry) {
retry = false;
// The multi-stack is now emptied.
perform_curl_io_and_complete(h, h->mcurl_size);
//Retry failed downloads
for (l = h->failed; l;) {
struct multi_curl_file *file = l->data;
if (file->retries < MAX_RETRIES) {
struct list *next;
file->retries++;
//Remove item
if (l == h->failed) {
h->failed = l->next;
}
next = l->next;
list_free_item(l, NULL);
l = next;
fprintf(stderr, "Starting download retry #%d for %s\n", file->retries, file->url);
process_download(h, file);
retry = true;
continue;
}
l = l->next;
}
if (retry) {
sleep(h->timeout);
h->timeout *= 2;
}
}
curl_multi_cleanup(h->mcurl);
list_free_list(h->failed);
HASHMAP_FOREACH(h->curl_hashmap, i, l, file)
{
downloads++;
free_curl_file(h, file);
}
hashmap_free(h->curl_hashmap);
free(h);
if (num_downloads) {
*num_downloads = downloads;
}
return 0;
}
src/fullfile.c
+74 -50
View File
@@ -21,11 +21,53 @@
*/
#define _GNU_SOURCE
#include <sys/stat.h>
#include <unistd.h>
#include "swupd.h"
static struct list *download_loop(struct list *files, bool free_list)
/* hysteresis thresholds */
#define MAX_XFER 25
#define MAX_XFER_BOTTOM 15
static void download_mix_file(struct file *file)
{
char *url, *filename;
string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash);
string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash);
/* Mix content is local, so don't queue files up for curl downloads */
if (link_or_rename(url, filename) == 0) {
untar_full_download(file);
} else {
fprintf(stderr, "Failed to copy local mix file: %s\n", file->staging);
}
free_string(&url);
free_string(&filename);
}
static void download_file(void *download_handle, struct file *file)
{
char *url, *filename;
char *targetfile;
struct stat stat;
string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash);
// Just download if the target file doesn't exist
if (lstat(targetfile, &stat) != 0 || !verify_file(file, targetfile)) {
string_or_die(&filename, "%s/download/.%s.tar", state_dir, file->hash);
string_or_die(&url, "%s/%i/files/%s.tar", content_url, file->last_change, file->hash);
swupd_curl_parallel_download_enqueue(download_handle, url, filename, file->hash, file);
free_string(&url);
free_string(&filename);
}
free_string(&targetfile);
}
static int download_loop(void *download_handle, struct list *files, int *num_downloads)
{
struct list *iter;
unsigned int complete = 0;
@@ -43,70 +85,52 @@ static struct list *download_loop(struct list *files, bool free_list)
continue;
}
/* Mix content is local, so don't queue files up for curl downloads */
if (file->is_mix) {
char *url;
if (file->staging) {
free_string(&file->staging);
}
string_or_die(&url, "%s/%i/files/%s.tar", MIX_STATE_DIR, file->last_change, file->hash);
string_or_die(&file->staging, "%s/download/.%s.tar", state_dir, file->hash);
if (link_or_rename(url, file->staging) != 0) {
fprintf(stderr, "Failed to copy local mix file: %s\n", file->staging);
continue;
}
untar_full_download(file);
free_string(&url);
continue;
download_mix_file(file);
} else {
download_file(download_handle, file);
}
full_download(file);
print_progress(complete, list_length);
}
print_progress(list_length, list_length); /* Force out 100% */
printf("\n");
if (free_list) {
list_free_list(files);
return swupd_curl_parallel_download_end(download_handle, num_downloads);
}
static bool download_successful(void *data)
{
if (!data) {
return false;
}
return end_full_download();
if (untar_full_download(data) != 0) {
fprintf(stderr, "Error for %s tarfile extraction, (check free space for %s?)\n",
((struct file *)data)->hash, state_dir);
}
return true;
}
/*
* Download all fullfiles from the files list.
* Return 0 on success or a negative number if any file was not downloaded successfully
* Download fullfiles from the list of files.
*
* Return 0 on success or a negative number or errors.
*/
int download_fullfiles(struct list *files, int num_retries, int timeout)
int download_fullfiles(struct list *files, int *num_downloads)
{
int ret;
int retries = 0;
void *download_handle;
while (files) {
ret = start_full_download();
if (ret != 0) {
/* If we hit this point, the network is accessible but we were
* unable to download the needed files. This is a terminal error
* and we need good logging */
return -EFULLDOWNLOAD;
}
files = download_loop(files, retries > 0);
/* Set retries only if failed downloads exist, and only retry a fixed
amount of &times */
if (list_head(files)) {
if (retries >= num_retries) {
list_free_list(files);
return -EFULLDOWNLOAD;
}
increment_retries(&retries, &timeout);
fprintf(stderr, "Starting download retry #%d\n", retries);
}
if (!files) {
return 0;
}
return 0;
download_handle = swupd_curl_parallel_download_start(MAX_XFER, MAX_XFER_BOTTOM, download_successful);
if (!download_handle) {
/* If we hit this point, the network is accessible but we were
* unable to download the needed files. This is a terminal error
* and we need good logging */
return -EFULLDOWNLOAD;
}
return download_loop(download_handle, files, num_downloads);
}
src/hashmap.c
+12
View File
@@ -31,6 +31,18 @@
#define HASH_MASK(bits) (HASH_SIZE(bits) - 1)
size_t hashmap_hash_from_string(const char *key)
{
size_t hash = 0;
while (key) {
hash = hash * 29 /* a prime number */ + *key;
key++;
}
return hash;
}
static inline struct list **get_hashmap_list(struct hashmap *hashmap, const void *data)
{
return &hashmap->map[hashmap->hash(data) || HASH_MASK(hashmap->mask_bits)];
src/hashmap.h
+4
View File
@@ -65,6 +65,10 @@ void hashmap_free(struct hashmap *hashmap);
*/
void hashmap_free_hash_and_data(struct hashmap *hashmap, free_data_fn_t free_data);
/*
* Hash function helper to calculate a good hash to be used with strings.
*/
size_t hashmap_hash_from_string(const char *key);
/*
* Loop through all elements in the hashmap.
*
src/helpers.c
+139
View File
@@ -38,6 +38,7 @@
#include "config.h"
#include "signature.h"
#include "swupd-build-variant.h"
#include "swupd.h"
void check_root(void)
@@ -953,3 +954,141 @@ int link_or_rename(const char *orig, const char *dest)
return 0;
}
/* list the tarfile content, and verify it contains only one line equal to the expected hash.
* loop through all the content to detect the case where archive contains more than one file.
*/
static int check_tarfile_content(struct file *file, const char *tarfilename)
{
int err;
char *tarcommand;
FILE *tar;
int count = 0;
string_or_die(&tarcommand, TAR_COMMAND " -tf %s/download/%s.tar 2> /dev/null", state_dir, file->hash);
err = access(tarfilename, R_OK);
if (err) {
goto free_tarcommand;
}
tar = popen(tarcommand, "r");
if (tar == NULL) {
err = -1;
goto free_tarcommand;
}
while (!feof(tar)) {
char *c;
char buffer[PATH_MAXLEN];
if (fgets(buffer, PATH_MAXLEN, tar) == NULL) {
if (count != 1) {
err = -1;
}
break;
}
c = strchr(buffer, '\n');
if (c) {
*c = 0;
}
if (c && (c != buffer) && (*(c - 1) == '/')) {
/* strip trailing '/' from directory tar */
*(c - 1) = 0;
}
if (strcmp(buffer, file->hash) != 0) {
err = -1;
break;
}
count++;
}
pclose(tar);
free_tarcommand:
free_string(&tarcommand);
return err;
}
/* This function will break if the same HASH.tar full file is downloaded
* multiple times in parallel. */
int untar_full_download(void *data)
{
struct file *file = data;
char *tarfile;
char *tar_dotfile;
char *targetfile;
struct stat stat;
int err;
string_or_die(&tar_dotfile, "%s/download/.%s.tar", state_dir, file->hash);
string_or_die(&tarfile, "%s/download/%s.tar", state_dir, file->hash);
string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash);
/* If valid target file already exists, we're done.
* NOTE: this should NEVER happen given the checking that happens
* ahead of queueing a download. But... */
if (lstat(targetfile, &stat) == 0) {
if (verify_file(file, targetfile)) {
unlink(tar_dotfile);
unlink(tarfile);
free_string(&tar_dotfile);
free_string(&tarfile);
free_string(&targetfile);
return 0;
} else {
unlink(tarfile);
unlink(targetfile);
}
} else if (lstat(tarfile, &stat) == 0) {
/* remove tar file from possible past failure */
unlink(tarfile);
}
err = rename(tar_dotfile, tarfile);
if (err) {
free_string(&tar_dotfile);
goto exit;
}
free_string(&tar_dotfile);
err = check_tarfile_content(file, tarfile);
if (err) {
goto exit;
}
/* modern tar will automatically determine the compression type used */
char *outputdir;
string_or_die(&outputdir, "%s/staged", state_dir);
err = extract_to(tarfile, outputdir);
free_string(&outputdir);
if (err) {
fprintf(stderr, "ignoring tar extract failure for fullfile %s.tar (ret %d)\n",
file->hash, err);
goto exit;
/* TODO: respond to ARCHIVE_RETRY error codes
* libarchive returns ARCHIVE_RETRY when tar extraction fails but the
* operation is retry-able. We need to determine if it is worth our time
* to retry in these situations. */
} else {
/* Only unlink when tar succeeded, so we can examine the tar file
* in the failure case. */
unlink(tarfile);
}
err = lstat(targetfile, &stat);
if (!err && !verify_file(file, targetfile)) {
/* Download was successful but the hash was bad. This is fatal*/
fprintf(stderr, "Error: File content hash mismatch for %s (bad server data?)\n", targetfile);
exit(EXIT_FAILURE);
}
exit:
free_string(&tarfile);
free_string(&targetfile);
if (err) {
unlink_all_staged_content(file);
}
return err;
}
src/swupd.h
+9 -14
View File
@@ -134,10 +134,7 @@ struct file {
struct file *peer; /* same file in another manifest */
struct header *header;
char *staging; /* output name used during download & staging */
CURL *curl; /* curl handle if downloading */
FILE *fh; /* file written into during downloading */
char *staging;
};
struct filerecord {
@@ -278,13 +275,10 @@ static inline void account_delta_miss(void)
extern void print_statistics(int version1, int version2);
extern int download_fullfiles(struct list *files, int num_retries, int timeout);
extern int download_fullfiles(struct list *files, int *num_downloads);
extern int download_subscribed_packs(struct list *subs, struct manifest *mom, bool required, bool resume_ok);
extern void apply_deltas(struct manifest *current_manifest);
extern void full_download(struct file *file);
extern int start_full_download(void);
extern struct list *end_full_download(void);
extern int untar_full_download(void *data);
extern int do_staging(struct file *file, struct manifest *manifest);
@@ -293,18 +287,19 @@ extern int rename_staged_file_to_final(struct file *file);
extern int update_device_latest_version(int version);
/* curl.c */
extern int swupd_curl_init(void);
extern void swupd_curl_deinit(void);
extern double swupd_curl_query_content_size(char *url);
extern CURLcode swupd_download_file_start(struct file *file);
extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file);
extern int swupd_curl_get_file(const char *url, char *filename);
extern int swupd_curl_get_file_full(const char *url, char *filename, struct curl_file_data *file_data, bool resume_ok);
#define SWUPD_CURL_LOW_SPEED_LIMIT 1
#define SWUPD_CURL_CONNECT_TIMEOUT 30
#define SWUPD_CURL_RCV_TIMEOUT 120
extern CURLcode swupd_curl_set_basic_options(CURL *curl, const char *url);
/* download.c */
typedef bool (*swupd_curl_download_cb)(void *data);
extern void *swupd_curl_parallel_download_start(size_t max_xfer, size_t max_xfer_bottom, swupd_curl_download_cb success_cb);
extern int swupd_curl_parallel_download_enqueue(void *handle, const char *url, const char *filename, const char *hash, void *data);
extern int swupd_curl_parallel_download_end(void *handle, int *num_downloads);
extern void free_subscriptions(struct list **subs);
extern void read_subscriptions(struct list **subs);
src/update.c
+1 -1
View File
@@ -84,7 +84,7 @@ static int update_loop(struct list *updates, struct manifest *server_manifest)
struct file *file;
struct list *iter;
ret = download_fullfiles(updates, MAX_TRIES, 10);
ret = download_fullfiles(updates, &nonpack);
if (ret) {
fprintf(stderr, "ERROR: Could not download all files, aborting update\n");
}
src/verify.c
+1 -1
View File
@@ -404,7 +404,7 @@ static int get_required_files(struct manifest *official_manifest, struct list *s
return 0;
}
ret = download_fullfiles(official_manifest->files, MAX_TRIES, 10);
ret = download_fullfiles(official_manifest->files, NULL);
if (ret) {
fprintf(stderr, "Error: Unable to download neccessary files for this OS release\n");
}