Compare commits
4 Commits
6d179c3a25
...
d53a93694e
Author | SHA1 | Date |
---|---|---|
Archilocos | d53a93694e | |
Fufu Fang | 81aac8bb57 | |
Mattias Runge-Broberg | 35a213942c | |
liuchenghao | f361527459 |
|
@ -164,10 +164,10 @@ static int Meta_read(Cache *cf)
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sizeof(long) != fread(&cf->time, sizeof(long), 1, fp) ||
|
if (sizeof(long) != fread(&cf->time, sizeof(long), 1, fp) ||
|
||||||
sizeof(off_t) != fread(&cf->content_length, sizeof(off_t), 1, fp) ||
|
sizeof(off_t) != fread(&cf->content_length, sizeof(off_t), 1, fp) ||
|
||||||
sizeof(int) != fread(&cf->blksz, sizeof(int), 1, fp) ||
|
sizeof(int) != fread(&cf->blksz, sizeof(int), 1, fp) ||
|
||||||
sizeof(long) != fread(&cf->segbc, sizeof(long), 1, fp) ||
|
sizeof(long) != fread(&cf->segbc, sizeof(long), 1, fp) ||
|
||||||
ferror(fp)) {
|
ferror(fp)) {
|
||||||
lprintf(error, "error reading core metadata!\n");
|
lprintf(error, "error reading core metadata!\n");
|
||||||
return EIO;
|
return EIO;
|
||||||
}
|
}
|
||||||
|
|
54
src/link.c
54
src/link.c
|
@ -237,11 +237,13 @@ static void LinkTable_uninitialised_fill(LinkTable *linktbl)
|
||||||
}
|
}
|
||||||
/*
|
/*
|
||||||
* Block until the gaps are filled
|
* Block until the gaps are filled
|
||||||
|
* result is an invalid variable
|
||||||
*/
|
*/
|
||||||
int n = curl_multi_perform_once();
|
int result = 0;
|
||||||
|
int n = curl_multi_perform_once(&result);
|
||||||
int i = 0;
|
int i = 0;
|
||||||
int j = 0;
|
int j = 0;
|
||||||
while ((i = curl_multi_perform_once())) {
|
while ((i = curl_multi_perform_once(&result))) {
|
||||||
if (CONFIG.log_type & debug) {
|
if (CONFIG.log_type & debug) {
|
||||||
if (j) {
|
if (j) {
|
||||||
erase_string(stderr, STATUS_LEN, s);
|
erase_string(stderr, STATUS_LEN, s);
|
||||||
|
@ -286,7 +288,7 @@ LinkTable *LinkSystem_init(const char *url)
|
||||||
* This is where the '/' should be
|
* This is where the '/' should be
|
||||||
*/
|
*/
|
||||||
ROOT_LINK_OFFSET = strnlen(url, MAX_PATH_LEN) -
|
ROOT_LINK_OFFSET = strnlen(url, MAX_PATH_LEN) -
|
||||||
((url[url_len] == '/') ? 1 : 0);
|
((url[url_len] == '/') ? 1 : 0);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* --------------------- Enable cache system --------------------
|
* --------------------- Enable cache system --------------------
|
||||||
|
@ -474,7 +476,7 @@ static void LinkTable_fill(LinkTable *linktbl)
|
||||||
spaces in them!). If we only escaped it, and there were already
|
spaces in them!). If we only escaped it, and there were already
|
||||||
encoded characters in it, then that would break the link. */
|
encoded characters in it, then that would break the link. */
|
||||||
char *unescaped_path = curl_easy_unescape(c, this_link->linkpath, 0,
|
char *unescaped_path = curl_easy_unescape(c, this_link->linkpath, 0,
|
||||||
NULL);
|
NULL);
|
||||||
char *escaped_path = curl_easy_escape(c, unescaped_path, 0);
|
char *escaped_path = curl_easy_escape(c, unescaped_path, 0);
|
||||||
curl_free(unescaped_path);
|
curl_free(unescaped_path);
|
||||||
/* Our code does the wrong thing if there's a trailing slash that's been
|
/* Our code does the wrong thing if there's a trailing slash that's been
|
||||||
|
@ -704,7 +706,10 @@ int LinkTable_disk_save(LinkTable *linktbl, const char *dirn)
|
||||||
/* This is necessary to get the compiler on some platforms to stop
|
/* This is necessary to get the compiler on some platforms to stop
|
||||||
complaining about the fact that we're not using the return value of
|
complaining about the fact that we're not using the return value of
|
||||||
fread, when we know we aren't and that's fine. */
|
fread, when we know we aren't and that's fine. */
|
||||||
static inline void ignore_value(int i) { (void) i; }
|
static inline void ignore_value(int i)
|
||||||
|
{
|
||||||
|
(void) i;
|
||||||
|
}
|
||||||
|
|
||||||
LinkTable *LinkTable_disk_open(const char *dirn)
|
LinkTable *LinkTable_disk_open(const char *dirn)
|
||||||
{
|
{
|
||||||
|
@ -799,6 +804,8 @@ LinkTable *path_to_Link_LinkTable_new(const char *path)
|
||||||
if (!next_table) {
|
if (!next_table) {
|
||||||
if (CONFIG.mode == NORMAL) {
|
if (CONFIG.mode == NORMAL) {
|
||||||
next_table = LinkTable_new(tmp_link->f_url);
|
next_table = LinkTable_new(tmp_link->f_url);
|
||||||
|
} else if (CONFIG.mode == SINGLE) {
|
||||||
|
next_table = single_LinkTable_new(tmp_link->f_url);
|
||||||
} else if (CONFIG.mode == SONIC) {
|
} else if (CONFIG.mode == SONIC) {
|
||||||
if (!CONFIG.sonic_id3) {
|
if (!CONFIG.sonic_id3) {
|
||||||
next_table = sonic_LinkTable_new_index(tmp_link->sonic.id);
|
next_table = sonic_LinkTable_new_index(tmp_link->sonic.id);
|
||||||
|
@ -808,7 +815,7 @@ LinkTable *path_to_Link_LinkTable_new(const char *path)
|
||||||
tmp_link->sonic.id);
|
tmp_link->sonic.id);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
lprintf(fatal, "Invalid CONFIG.mode\n");
|
lprintf(fatal, "Invalid CONFIG.mode: %d\n", CONFIG.mode);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (link) {
|
if (link) {
|
||||||
|
@ -944,7 +951,7 @@ TransferStruct Link_download_full(Link *link)
|
||||||
*/
|
*/
|
||||||
long http_resp = 0;
|
long http_resp = 0;
|
||||||
do {
|
do {
|
||||||
transfer_blocking(curl);
|
transfer_blocking(curl, ts.curr_size);
|
||||||
ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||||
if (ret) {
|
if (ret) {
|
||||||
lprintf(error, "%s", curl_easy_strerror(ret));
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
||||||
|
@ -981,7 +988,7 @@ static CURL *Link_download_curl_setup(Link *link, size_t req_size, off_t offset,
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t start = offset;
|
size_t start = offset;
|
||||||
size_t end = start + req_size;
|
size_t end = start + req_size - 1;
|
||||||
|
|
||||||
char range_str[64];
|
char range_str[64];
|
||||||
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
|
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
|
||||||
|
@ -1030,8 +1037,8 @@ range requests\n");
|
||||||
lprintf(error, "%s", curl_easy_strerror(ret));
|
lprintf(error, "%s", curl_easy_strerror(ret));
|
||||||
}
|
}
|
||||||
if ((http_resp != HTTP_OK) &&
|
if ((http_resp != HTTP_OK) &&
|
||||||
(http_resp != HTTP_PARTIAL_CONTENT) &&
|
(http_resp != HTTP_PARTIAL_CONTENT) &&
|
||||||
(http_resp != HTTP_RANGE_NOT_SATISFIABLE)) {
|
(http_resp != HTTP_RANGE_NOT_SATISFIABLE)) {
|
||||||
char *url;
|
char *url;
|
||||||
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
|
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
|
||||||
lprintf(warning, "Could not download %s, HTTP %ld\n", url, http_resp);
|
lprintf(warning, "Could not download %s, HTTP %ld\n", url, http_resp);
|
||||||
|
@ -1061,9 +1068,16 @@ long Link_download(Link *link, char *output_buf, size_t req_size, off_t offset)
|
||||||
header.curr_size = 0;
|
header.curr_size = 0;
|
||||||
header.data = NULL;
|
header.data = NULL;
|
||||||
|
|
||||||
|
if (offset + req_size > link->content_length) {
|
||||||
|
lprintf(error,
|
||||||
|
"requested size too large, req_size: %lu, recv: %ld, content-length: %ld\n",
|
||||||
|
req_size, recv, link->content_length);
|
||||||
|
req_size = link->content_length - offset;
|
||||||
|
}
|
||||||
|
|
||||||
CURL *curl = Link_download_curl_setup(link, req_size, offset, &header, &ts);
|
CURL *curl = Link_download_curl_setup(link, req_size, offset, &header, &ts);
|
||||||
|
|
||||||
transfer_blocking(curl);
|
transfer_blocking(curl, offset);
|
||||||
|
|
||||||
curl_off_t recv = Link_download_cleanup(curl, &header);
|
curl_off_t recv = Link_download_cleanup(curl, &header);
|
||||||
|
|
||||||
|
@ -1096,7 +1110,7 @@ long path_download(const char *path, char *output_buf, size_t req_size,
|
||||||
return Link_download(link, output_buf, req_size, offset);
|
return Link_download(link, output_buf, req_size, offset);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void make_link_relative(const char *page_url, char *link_url)
|
static void make_link_relative(const char *page_url, char *link_url)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
Some servers make the links to subdirectories absolute (in URI terms:
|
Some servers make the links to subdirectories absolute (in URI terms:
|
||||||
|
@ -1139,14 +1153,14 @@ static void make_link_relative(const char *page_url, char *link_url)
|
||||||
page_url++;
|
page_url++;
|
||||||
}
|
}
|
||||||
if (slashes_left_to_find) {
|
if (slashes_left_to_find) {
|
||||||
if (slashes_left_to_find == 1 && ! *page_url)
|
if (slashes_left_to_find == 1 && ! *page_url)
|
||||||
/* We're at the top level of the web site and the user entered the URL
|
/* We're at the top level of the web site and the user entered the URL
|
||||||
without a trailing slash. */
|
without a trailing slash. */
|
||||||
page_url = "/";
|
page_url = "/";
|
||||||
else
|
else
|
||||||
/* Well, that's odd. Let's return rather than trying to dig ourselves
|
/* Well, that's odd. Let's return rather than trying to dig ourselves
|
||||||
deeper into whatever hole we're in. */
|
deeper into whatever hole we're in. */
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
/* The page URL is no longer the full page_url, it's just the part after
|
/* The page URL is no longer the full page_url, it's just the part after
|
||||||
the host name. */
|
the host name. */
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* ----------------- External variables ----------------------
|
* ----------------- External variables ----------------------
|
||||||
|
@ -110,9 +111,10 @@ curl_callback_unlock(CURL *handle, curl_lock_data data, void *userptr)
|
||||||
* \details Adapted from:
|
* \details Adapted from:
|
||||||
* https://curl.haxx.se/libcurl/c/10-at-a-time.html
|
* https://curl.haxx.se/libcurl/c/10-at-a-time.html
|
||||||
*/
|
*/
|
||||||
static void
|
static int
|
||||||
curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
|
curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
|
||||||
{
|
{
|
||||||
|
int result = 0;
|
||||||
(void) n_running_curl;
|
(void) n_running_curl;
|
||||||
(void) n_mesgs;
|
(void) n_mesgs;
|
||||||
static volatile int slept = 0;
|
static volatile int slept = 0;
|
||||||
|
@ -163,6 +165,7 @@ curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
|
||||||
lprintf(error, "%d - %s <%s>\n",
|
lprintf(error, "%d - %s <%s>\n",
|
||||||
curl_msg->data.result,
|
curl_msg->data.result,
|
||||||
curl_easy_strerror(curl_msg->data.result), url);
|
curl_easy_strerror(curl_msg->data.result), url);
|
||||||
|
result = curl_msg->data.result;
|
||||||
}
|
}
|
||||||
curl_multi_remove_handle(curl_multi, curl);
|
curl_multi_remove_handle(curl_multi, curl);
|
||||||
/*
|
/*
|
||||||
|
@ -175,13 +178,50 @@ curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
|
||||||
} else {
|
} else {
|
||||||
lprintf(warning, "curl_msg->msg: %d\n", curl_msg->msg);
|
lprintf(warning, "curl_msg->msg: %d\n", curl_msg->msg);
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int http_error_result(int http_response)
|
||||||
|
{
|
||||||
|
switch(http_response)
|
||||||
|
{
|
||||||
|
case 0: //eg connection down from kick-off ~suggest retrying till some max limit
|
||||||
|
case 200: //yay we at least got to our url
|
||||||
|
case 206: //Partial Content
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 416:
|
||||||
|
//cannot d/l range ~ either cos no server support
|
||||||
|
//or cos we're asking for an invalid range ~ie: we already d/ld the file
|
||||||
|
printf("HTTP416: either the d/l is already complete or the http server cannot d/l a range\n");
|
||||||
|
default:
|
||||||
|
return 0;//suggest quitting on an unhandled error
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int curl_error_result(int curl_result)
|
||||||
|
{
|
||||||
|
switch (curl_result)
|
||||||
|
{
|
||||||
|
case CURLE_OK:
|
||||||
|
case CURLE_COULDNT_CONNECT: //no network connectivity ?
|
||||||
|
case CURLE_OPERATION_TIMEDOUT: //cos of CURLOPT_LOW_SPEED_TIME
|
||||||
|
case CURLE_COULDNT_RESOLVE_HOST: //host/DNS down ?
|
||||||
|
break; //we'll keep trying
|
||||||
|
default://see: http://curl.haxx.se/libcurl/c/libcurl-errors.html
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \details effectively based on
|
* \details effectively based on
|
||||||
* https://curl.haxx.se/libcurl/c/multi-double.html
|
* https://curl.haxx.se/libcurl/c/multi-double.html
|
||||||
*/
|
*/
|
||||||
int curl_multi_perform_once(void)
|
int curl_multi_perform_once(int *result)
|
||||||
{
|
{
|
||||||
lprintf(network_lock_debug,
|
lprintf(network_lock_debug,
|
||||||
"thread %x: locking transfer_lock;\n", pthread_self());
|
"thread %x: locking transfer_lock;\n", pthread_self());
|
||||||
|
@ -207,7 +247,12 @@ int curl_multi_perform_once(void)
|
||||||
int n_mesgs;
|
int n_mesgs;
|
||||||
CURLMsg *curl_msg;
|
CURLMsg *curl_msg;
|
||||||
while ((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
|
while ((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
|
||||||
curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
|
int nResult = curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
|
||||||
|
if (!http_error_result(n_mesgs) || !curl_error_result(nResult)) {
|
||||||
|
*result = 1;
|
||||||
|
}else{
|
||||||
|
*result = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
lprintf(network_lock_debug,
|
lprintf(network_lock_debug,
|
||||||
|
@ -272,7 +317,7 @@ void NetworkSystem_init(void)
|
||||||
crypto_lock_init();
|
crypto_lock_init();
|
||||||
}
|
}
|
||||||
|
|
||||||
void transfer_blocking(CURL *curl)
|
void transfer_blocking(CURL *curl, size_t start)
|
||||||
{
|
{
|
||||||
TransferStruct *ts;
|
TransferStruct *ts;
|
||||||
CURLcode ret = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &ts);
|
CURLcode ret = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &ts);
|
||||||
|
@ -293,8 +338,22 @@ void transfer_blocking(CURL *curl)
|
||||||
"thread %x: unlocking transfer_lock;\n", pthread_self());
|
"thread %x: unlocking transfer_lock;\n", pthread_self());
|
||||||
PTHREAD_MUTEX_UNLOCK(&transfer_lock);
|
PTHREAD_MUTEX_UNLOCK(&transfer_lock);
|
||||||
|
|
||||||
while (ts->transferring) {
|
int result = 0;
|
||||||
curl_multi_perform_once();
|
bool restartDown = false;
|
||||||
|
|
||||||
|
while (ts->transferring && !restartDown) {
|
||||||
|
/*
|
||||||
|
* When the network is abnormal during the file download, start to resume the transfer
|
||||||
|
*/
|
||||||
|
if (0 != result) {
|
||||||
|
curl_multi_remove_handle(curl_multi,curl);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, start);
|
||||||
|
res = curl_multi_add_handle(curl_multi, curl);
|
||||||
|
if (res > 0) {
|
||||||
|
lprintf(error, "%d, %s\n", res, curl_multi_strerror(res));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
curl_multi_perform_once(&result);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,13 +26,13 @@ typedef enum {
|
||||||
extern CURLSH *CURL_SHARE;
|
extern CURLSH *CURL_SHARE;
|
||||||
|
|
||||||
/** \brief perform one transfer cycle */
|
/** \brief perform one transfer cycle */
|
||||||
int curl_multi_perform_once(void);
|
int curl_multi_perform_once(int *result);
|
||||||
|
|
||||||
/** \brief initialise the network module */
|
/** \brief initialise the network module */
|
||||||
void NetworkSystem_init(void);
|
void NetworkSystem_init(void);
|
||||||
|
|
||||||
/** \brief blocking file transfer */
|
/** \brief blocking file transfer */
|
||||||
void transfer_blocking(CURL *curl);
|
void transfer_blocking(CURL *curl, size_t start);
|
||||||
|
|
||||||
/** \brief non blocking file transfer */
|
/** \brief non blocking file transfer */
|
||||||
void transfer_nonblocking(CURL *curl);
|
void transfer_nonblocking(CURL *curl);
|
||||||
|
|
Loading…
Reference in New Issue