From 149319069231ee77c20e859de8ac16b78b5f793f Mon Sep 17 00:00:00 2001 From: Fufu Fang Date: Wed, 4 Sep 2019 18:30:57 +0100 Subject: [PATCH] Improved HTTP temporary failure error handling - Added HTTP response code for Cloudflare timeout - Improved HTTP temporary failure error handling during LinkTable generation - Now checked all HTTP response code in a single function --- src/link.c | 23 +++++++++++------------ src/main.c | 2 +- src/network.c | 24 ++++++++++++++++++------ src/network.h | 25 ++++++++++++++++--------- src/util.h | 2 -- 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/link.c b/src/link.c index ed856ce..cdc20b1 100644 --- a/src/link.c +++ b/src/link.c @@ -200,14 +200,12 @@ void Link_set_file_stat(Link* this_link, CURL *curl) } } else { fprintf(stderr, "Link_set_file_stat(): HTTP %ld", http_resp); - if ((http_resp == HTTP_TOO_MANY_REQUESTS) || - (http_resp == HTTP_UNKNOWN_ERROR)) { + if (HTTP_temp_failure(http_resp)) { fprintf(stderr, ", retrying later.\n"); } else { - fprintf(stderr, "\n"); this_link->type = LINK_INVALID; + fprintf(stderr, ".\n"); } - fprintf(stderr, ".\n"); } } @@ -344,23 +342,24 @@ LinkTable *LinkTable_new(const char *url) buf.memory = NULL; curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf); - /* If we get HTTP 429, wait for 5 seconds before retry */ - volatile long http_resp = 0; + /* If we get temporary HTTP failure, wait for 5 seconds before retry */ + long http_resp = 0; do { transfer_blocking(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp); - if (http_resp == HTTP_TOO_MANY_REQUESTS) { - fprintf(stderr, "LinkTable_new(): URL: %s, HTTP 429, \ -Too Many Requests\n", url); - sleep(HTTP_429_WAIT); + if (HTTP_temp_failure(http_resp)) { + fprintf(stderr, "LinkTable_new(): URL: %s, HTTP %ld, \ +retrying later.\n", url, http_resp); + sleep(HTTP_WAIT_SEC); } else if (http_resp != HTTP_OK) { fprintf(stderr, "LinkTable_new(): cannot retrieve URL: %s, \ HTTP %ld\n", url, http_resp); LinkTable_free(linktbl); curl_easy_cleanup(curl); return NULL; - }; - } while (http_resp != HTTP_OK); + } + } while (HTTP_temp_failure(http_resp)); + curl_easy_getinfo(curl, CURLINFO_FILETIME, &(head_link->time)); curl_easy_cleanup(curl); diff --git a/src/main.c b/src/main.c index a38bc1c..e8843ec 100644 --- a/src/main.c +++ b/src/main.c @@ -199,7 +199,7 @@ parse_arg_list(int argc, char **argv, char ***fuse_argv, int *fuse_argc) NETWORK_CONFIG.user_agent = strdup(optarg); break; case 13: - HTTP_429_WAIT = atoi(optarg); + HTTP_WAIT_SEC = atoi(optarg); break; case 14: NETWORK_CONFIG.cache_dir = strdup(optarg); diff --git a/src/network.c b/src/network.c index d316705..407c0ac 100644 --- a/src/network.c +++ b/src/network.c @@ -10,12 +10,12 @@ #include #define DEFAULT_NETWORK_MAX_CONNS 10 -#define DEFAULT_HTTP_429_WAIT 5 +#define DEFAULT_HTTP_WAIT_SEC 5 /* ----------------- External variables ---------------------- */ CURLSH *CURL_SHARE; NetworkConfigStruct NETWORK_CONFIG; -int HTTP_429_WAIT = DEFAULT_HTTP_429_WAIT; +int HTTP_WAIT_SEC = DEFAULT_HTTP_WAIT_SEC; /* ----------------- Static variable ----------------------- */ /** \brief curl multi interface handle */ @@ -119,12 +119,12 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, /* Wait for 5 seconds if we get HTTP 429 */ long http_resp = 0; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp); - if (http_resp == HTTP_TOO_MANY_REQUESTS) { + if (HTTP_temp_failure(http_resp)) { if (!slept) { fprintf(stderr, - "curl_process_msgs(): HTTP 429, sleeping for %d sec\n", - HTTP_429_WAIT); - sleep(HTTP_429_WAIT); + "curl_process_msgs(): HTTP %ld, sleeping for %d sec\n", + http_resp, HTTP_WAIT_SEC); + sleep(HTTP_WAIT_SEC); slept = 1; } } else { @@ -403,3 +403,15 @@ size_t write_memory_callback(void *contents, size_t size, size_t nmemb, return realsize; } + +int HTTP_temp_failure(HTTPResponseCode http_resp) +{ + switch (http_resp) { + case HTTP_TOO_MANY_REQUESTS: + case HTTP_CLOUDFLARE_UNKNOWN_ERROR: + case HTTP_CLOUDFLARE_TIMEOUT: + return 1; + default: + return 0; + } +} diff --git a/src/network.h b/src/network.h index 361e8f4..d6d6f4c 100644 --- a/src/network.h +++ b/src/network.h @@ -8,14 +8,6 @@ #include "link.h" -typedef enum { - HTTP_OK = 200, - HTTP_PARTIAL_CONTENT = 206, - HTTP_RANGE_NOT_SATISFIABLE = 416, - HTTP_TOO_MANY_REQUESTS = 429, - HTTP_UNKNOWN_ERROR = 520 -} HTTPResponseCode; - typedef enum { FILESTAT = 's', DATA = 'd' @@ -45,8 +37,18 @@ typedef struct { int cache_enabled; } NetworkConfigStruct; +/** \brief HTTP response codes */ +typedef enum { + HTTP_OK = 200, + HTTP_PARTIAL_CONTENT = 206, + HTTP_RANGE_NOT_SATISFIABLE = 416, + HTTP_TOO_MANY_REQUESTS = 429, + HTTP_CLOUDFLARE_UNKNOWN_ERROR = 520, + HTTP_CLOUDFLARE_TIMEOUT = 524 +} HTTPResponseCode; + /** \brief The waiting time after getting HTTP 429 */ -extern int HTTP_429_WAIT; +extern int HTTP_WAIT_SEC; /** \brief CURL configuration */ extern NetworkConfigStruct NETWORK_CONFIG; @@ -73,4 +75,9 @@ void transfer_nonblocking(CURL *curl); size_t write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp); +/** + * \brief check if a HTTP response code corresponds to a temporary failure + */ +int HTTP_temp_failure(HTTPResponseCode http_resp); + #endif diff --git a/src/util.h b/src/util.h index fa4b679..1798c14 100644 --- a/src/util.h +++ b/src/util.h @@ -18,7 +18,6 @@ /** \brief the maximum length of a filename. */ #define MAX_FILENAME_LEN 255 - /** * \brief append a path * \details This function appends a path with the next level, while taking the @@ -48,7 +47,6 @@ void PTHREAD_MUTEX_UNLOCK(pthread_mutex_t *x); */ void exit_failure(void); - /** * \brief erase a string from the terminal */