mirror of
https://github.com/fangfufu/httpdirfs.git
synced 2024-09-27 20:51:55 +02:00
Improved HTTP temporary failure error handling
- Added HTTP response code for Cloudflare timeout - Improved HTTP temporary failure error handling during LinkTable generation - Now checked all HTTP response code in a single function
This commit is contained in:
parent
ff67794b02
commit
1493190692
23
src/link.c
23
src/link.c
@ -200,15 +200,13 @@ void Link_set_file_stat(Link* this_link, CURL *curl)
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
fprintf(stderr, "Link_set_file_stat(): HTTP %ld", http_resp);
|
fprintf(stderr, "Link_set_file_stat(): HTTP %ld", http_resp);
|
||||||
if ((http_resp == HTTP_TOO_MANY_REQUESTS) ||
|
if (HTTP_temp_failure(http_resp)) {
|
||||||
(http_resp == HTTP_UNKNOWN_ERROR)) {
|
|
||||||
fprintf(stderr, ", retrying later.\n");
|
fprintf(stderr, ", retrying later.\n");
|
||||||
} else {
|
} else {
|
||||||
fprintf(stderr, "\n");
|
|
||||||
this_link->type = LINK_INVALID;
|
this_link->type = LINK_INVALID;
|
||||||
}
|
|
||||||
fprintf(stderr, ".\n");
|
fprintf(stderr, ".\n");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -344,23 +342,24 @@ LinkTable *LinkTable_new(const char *url)
|
|||||||
buf.memory = NULL;
|
buf.memory = NULL;
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
||||||
|
|
||||||
/* If we get HTTP 429, wait for 5 seconds before retry */
|
/* If we get temporary HTTP failure, wait for 5 seconds before retry */
|
||||||
volatile long http_resp = 0;
|
long http_resp = 0;
|
||||||
do {
|
do {
|
||||||
transfer_blocking(curl);
|
transfer_blocking(curl);
|
||||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||||
if (http_resp == HTTP_TOO_MANY_REQUESTS) {
|
if (HTTP_temp_failure(http_resp)) {
|
||||||
fprintf(stderr, "LinkTable_new(): URL: %s, HTTP 429, \
|
fprintf(stderr, "LinkTable_new(): URL: %s, HTTP %ld, \
|
||||||
Too Many Requests\n", url);
|
retrying later.\n", url, http_resp);
|
||||||
sleep(HTTP_429_WAIT);
|
sleep(HTTP_WAIT_SEC);
|
||||||
} else if (http_resp != HTTP_OK) {
|
} else if (http_resp != HTTP_OK) {
|
||||||
fprintf(stderr, "LinkTable_new(): cannot retrieve URL: %s, \
|
fprintf(stderr, "LinkTable_new(): cannot retrieve URL: %s, \
|
||||||
HTTP %ld\n", url, http_resp);
|
HTTP %ld\n", url, http_resp);
|
||||||
LinkTable_free(linktbl);
|
LinkTable_free(linktbl);
|
||||||
curl_easy_cleanup(curl);
|
curl_easy_cleanup(curl);
|
||||||
return NULL;
|
return NULL;
|
||||||
};
|
}
|
||||||
} while (http_resp != HTTP_OK);
|
} while (HTTP_temp_failure(http_resp));
|
||||||
|
|
||||||
|
|
||||||
curl_easy_getinfo(curl, CURLINFO_FILETIME, &(head_link->time));
|
curl_easy_getinfo(curl, CURLINFO_FILETIME, &(head_link->time));
|
||||||
curl_easy_cleanup(curl);
|
curl_easy_cleanup(curl);
|
||||||
|
@ -199,7 +199,7 @@ parse_arg_list(int argc, char **argv, char ***fuse_argv, int *fuse_argc)
|
|||||||
NETWORK_CONFIG.user_agent = strdup(optarg);
|
NETWORK_CONFIG.user_agent = strdup(optarg);
|
||||||
break;
|
break;
|
||||||
case 13:
|
case 13:
|
||||||
HTTP_429_WAIT = atoi(optarg);
|
HTTP_WAIT_SEC = atoi(optarg);
|
||||||
break;
|
break;
|
||||||
case 14:
|
case 14:
|
||||||
NETWORK_CONFIG.cache_dir = strdup(optarg);
|
NETWORK_CONFIG.cache_dir = strdup(optarg);
|
||||||
|
@ -10,12 +10,12 @@
|
|||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#define DEFAULT_NETWORK_MAX_CONNS 10
|
#define DEFAULT_NETWORK_MAX_CONNS 10
|
||||||
#define DEFAULT_HTTP_429_WAIT 5
|
#define DEFAULT_HTTP_WAIT_SEC 5
|
||||||
|
|
||||||
/* ----------------- External variables ---------------------- */
|
/* ----------------- External variables ---------------------- */
|
||||||
CURLSH *CURL_SHARE;
|
CURLSH *CURL_SHARE;
|
||||||
NetworkConfigStruct NETWORK_CONFIG;
|
NetworkConfigStruct NETWORK_CONFIG;
|
||||||
int HTTP_429_WAIT = DEFAULT_HTTP_429_WAIT;
|
int HTTP_WAIT_SEC = DEFAULT_HTTP_WAIT_SEC;
|
||||||
|
|
||||||
/* ----------------- Static variable ----------------------- */
|
/* ----------------- Static variable ----------------------- */
|
||||||
/** \brief curl multi interface handle */
|
/** \brief curl multi interface handle */
|
||||||
@ -119,12 +119,12 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl,
|
|||||||
/* Wait for 5 seconds if we get HTTP 429 */
|
/* Wait for 5 seconds if we get HTTP 429 */
|
||||||
long http_resp = 0;
|
long http_resp = 0;
|
||||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||||
if (http_resp == HTTP_TOO_MANY_REQUESTS) {
|
if (HTTP_temp_failure(http_resp)) {
|
||||||
if (!slept) {
|
if (!slept) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
"curl_process_msgs(): HTTP 429, sleeping for %d sec\n",
|
"curl_process_msgs(): HTTP %ld, sleeping for %d sec\n",
|
||||||
HTTP_429_WAIT);
|
http_resp, HTTP_WAIT_SEC);
|
||||||
sleep(HTTP_429_WAIT);
|
sleep(HTTP_WAIT_SEC);
|
||||||
slept = 1;
|
slept = 1;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@ -403,3 +403,15 @@ size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
|
|||||||
|
|
||||||
return realsize;
|
return realsize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int HTTP_temp_failure(HTTPResponseCode http_resp)
|
||||||
|
{
|
||||||
|
switch (http_resp) {
|
||||||
|
case HTTP_TOO_MANY_REQUESTS:
|
||||||
|
case HTTP_CLOUDFLARE_UNKNOWN_ERROR:
|
||||||
|
case HTTP_CLOUDFLARE_TIMEOUT:
|
||||||
|
return 1;
|
||||||
|
default:
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -8,14 +8,6 @@
|
|||||||
|
|
||||||
#include "link.h"
|
#include "link.h"
|
||||||
|
|
||||||
typedef enum {
|
|
||||||
HTTP_OK = 200,
|
|
||||||
HTTP_PARTIAL_CONTENT = 206,
|
|
||||||
HTTP_RANGE_NOT_SATISFIABLE = 416,
|
|
||||||
HTTP_TOO_MANY_REQUESTS = 429,
|
|
||||||
HTTP_UNKNOWN_ERROR = 520
|
|
||||||
} HTTPResponseCode;
|
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
FILESTAT = 's',
|
FILESTAT = 's',
|
||||||
DATA = 'd'
|
DATA = 'd'
|
||||||
@ -45,8 +37,18 @@ typedef struct {
|
|||||||
int cache_enabled;
|
int cache_enabled;
|
||||||
} NetworkConfigStruct;
|
} NetworkConfigStruct;
|
||||||
|
|
||||||
|
/** \brief HTTP response codes */
|
||||||
|
typedef enum {
|
||||||
|
HTTP_OK = 200,
|
||||||
|
HTTP_PARTIAL_CONTENT = 206,
|
||||||
|
HTTP_RANGE_NOT_SATISFIABLE = 416,
|
||||||
|
HTTP_TOO_MANY_REQUESTS = 429,
|
||||||
|
HTTP_CLOUDFLARE_UNKNOWN_ERROR = 520,
|
||||||
|
HTTP_CLOUDFLARE_TIMEOUT = 524
|
||||||
|
} HTTPResponseCode;
|
||||||
|
|
||||||
/** \brief The waiting time after getting HTTP 429 */
|
/** \brief The waiting time after getting HTTP 429 */
|
||||||
extern int HTTP_429_WAIT;
|
extern int HTTP_WAIT_SEC;
|
||||||
|
|
||||||
/** \brief CURL configuration */
|
/** \brief CURL configuration */
|
||||||
extern NetworkConfigStruct NETWORK_CONFIG;
|
extern NetworkConfigStruct NETWORK_CONFIG;
|
||||||
@ -73,4 +75,9 @@ void transfer_nonblocking(CURL *curl);
|
|||||||
size_t
|
size_t
|
||||||
write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp);
|
write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief check if a HTTP response code corresponds to a temporary failure
|
||||||
|
*/
|
||||||
|
int HTTP_temp_failure(HTTPResponseCode http_resp);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -18,7 +18,6 @@
|
|||||||
/** \brief the maximum length of a filename. */
|
/** \brief the maximum length of a filename. */
|
||||||
#define MAX_FILENAME_LEN 255
|
#define MAX_FILENAME_LEN 255
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief append a path
|
* \brief append a path
|
||||||
* \details This function appends a path with the next level, while taking the
|
* \details This function appends a path with the next level, while taking the
|
||||||
@ -48,7 +47,6 @@ void PTHREAD_MUTEX_UNLOCK(pthread_mutex_t *x);
|
|||||||
*/
|
*/
|
||||||
void exit_failure(void);
|
void exit_failure(void);
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief erase a string from the terminal
|
* \brief erase a string from the terminal
|
||||||
*/
|
*/
|
||||||
|
Loading…
Reference in New Issue
Block a user