Improved HTTP temporary failure error handling

- Added HTTP response code for Cloudflare timeout
- Improved HTTP temporary failure error handling during LinkTable generation
- Now checked all HTTP response code in a single function

src/link.c

@@ -200,14 +200,12 @@ void Link_set_file_stat(Link* this_link, CURL *curl)
         }
     } else {
         fprintf(stderr, "Link_set_file_stat(): HTTP %ld", http_resp);
-        if ((http_resp == HTTP_TOO_MANY_REQUESTS) ||
-            (http_resp == HTTP_UNKNOWN_ERROR)) {
+        if (HTTP_temp_failure(http_resp)) {
             fprintf(stderr, ", retrying later.\n");
         } else {
-            fprintf(stderr, "\n");
             this_link->type = LINK_INVALID;
+            fprintf(stderr, ".\n");
         }
-        fprintf(stderr, ".\n");
     }
 }
 
@@ -344,23 +342,24 @@ LinkTable *LinkTable_new(const char *url)
     buf.memory = NULL;
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
 
-    /* If we get HTTP 429, wait for 5 seconds before retry */
-    volatile long http_resp = 0;
+    /* If we get temporary HTTP failure, wait for 5 seconds before retry */
+    long http_resp = 0;
     do {
         transfer_blocking(curl);
         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
-        if (http_resp == HTTP_TOO_MANY_REQUESTS) {
-            fprintf(stderr, "LinkTable_new(): URL: %s, HTTP 429, \
-Too Many Requests\n", url);
-            sleep(HTTP_429_WAIT);
+        if (HTTP_temp_failure(http_resp)) {
+            fprintf(stderr, "LinkTable_new(): URL: %s, HTTP %ld, \
+retrying later.\n", url, http_resp);
+            sleep(HTTP_WAIT_SEC);
         } else if (http_resp != HTTP_OK) {
             fprintf(stderr, "LinkTable_new(): cannot retrieve URL: %s, \
 HTTP %ld\n", url, http_resp);
             LinkTable_free(linktbl);
             curl_easy_cleanup(curl);
             return NULL;
-        };
-    } while (http_resp != HTTP_OK);
+        }
+    } while (HTTP_temp_failure(http_resp));
+
 
     curl_easy_getinfo(curl, CURLINFO_FILETIME, &(head_link->time));
     curl_easy_cleanup(curl);

src/main.c

@@ -199,7 +199,7 @@ parse_arg_list(int argc, char **argv, char ***fuse_argv, int *fuse_argc)
                         NETWORK_CONFIG.user_agent = strdup(optarg);
                         break;
                     case 13:
-                        HTTP_429_WAIT = atoi(optarg);
+                        HTTP_WAIT_SEC = atoi(optarg);
                         break;
                     case 14:
                         NETWORK_CONFIG.cache_dir = strdup(optarg);

src/network.c

@@ -10,12 +10,12 @@
 #include <unistd.h>
 
 #define DEFAULT_NETWORK_MAX_CONNS   10
-#define DEFAULT_HTTP_429_WAIT       5
+#define DEFAULT_HTTP_WAIT_SEC       5
 
 /* ----------------- External variables ---------------------- */
 CURLSH *CURL_SHARE;
 NetworkConfigStruct NETWORK_CONFIG;
-int HTTP_429_WAIT = DEFAULT_HTTP_429_WAIT;
+int HTTP_WAIT_SEC = DEFAULT_HTTP_WAIT_SEC;
 
 /* ----------------- Static variable ----------------------- */
 /** \brief curl multi interface handle */
@@ -119,12 +119,12 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl,
         /* Wait for 5 seconds if we get HTTP 429 */
         long http_resp = 0;
         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
-        if (http_resp == HTTP_TOO_MANY_REQUESTS) {
+        if (HTTP_temp_failure(http_resp)) {
             if (!slept) {
                 fprintf(stderr,
-                        "curl_process_msgs(): HTTP 429, sleeping for %d sec\n",
-                        HTTP_429_WAIT);
-                sleep(HTTP_429_WAIT);
+                        "curl_process_msgs(): HTTP %ld, sleeping for %d sec\n",
+                        http_resp, HTTP_WAIT_SEC);
+                sleep(HTTP_WAIT_SEC);
                 slept = 1;
             }
         } else {
@@ -403,3 +403,15 @@ size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
 
     return realsize;
 }
+
+int HTTP_temp_failure(HTTPResponseCode http_resp)
+{
+    switch (http_resp) {
+        case HTTP_TOO_MANY_REQUESTS:
+        case HTTP_CLOUDFLARE_UNKNOWN_ERROR:
+        case HTTP_CLOUDFLARE_TIMEOUT:
+            return 1;
+        default:
+            return 0;
+    }
+}

src/network.h

@@ -8,14 +8,6 @@
 
 #include "link.h"
 
-typedef enum {
-    HTTP_OK                     = 200,
-    HTTP_PARTIAL_CONTENT        = 206,
-    HTTP_RANGE_NOT_SATISFIABLE  = 416,
-    HTTP_TOO_MANY_REQUESTS      = 429,
-    HTTP_UNKNOWN_ERROR          = 520
-} HTTPResponseCode;
-
 typedef enum {
     FILESTAT = 's',
     DATA = 'd'
@@ -45,8 +37,18 @@ typedef struct {
     int cache_enabled;
 } NetworkConfigStruct;
 
+/** \brief HTTP response codes */
+typedef enum {
+    HTTP_OK                         = 200,
+    HTTP_PARTIAL_CONTENT            = 206,
+    HTTP_RANGE_NOT_SATISFIABLE      = 416,
+    HTTP_TOO_MANY_REQUESTS          = 429,
+    HTTP_CLOUDFLARE_UNKNOWN_ERROR   = 520,
+    HTTP_CLOUDFLARE_TIMEOUT         = 524
+} HTTPResponseCode;
+
 /** \brief The waiting time after getting HTTP 429 */
-extern int HTTP_429_WAIT;
+extern int HTTP_WAIT_SEC;
 
 /** \brief CURL configuration */
 extern NetworkConfigStruct NETWORK_CONFIG;
@@ -73,4 +75,9 @@ void transfer_nonblocking(CURL *curl);
 size_t
 write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp);
 
+/**
+ * \brief check if a HTTP response code corresponds to a temporary failure
+ */
+int HTTP_temp_failure(HTTPResponseCode http_resp);
+
 #endif

src/util.h

@@ -18,7 +18,6 @@
 /** \brief the maximum length of a filename. */
 #define MAX_FILENAME_LEN    255
 
-
 /**
  * \brief append a path
  * \details This function appends a path with the next level, while taking the
@@ -48,7 +47,6 @@ void PTHREAD_MUTEX_UNLOCK(pthread_mutex_t *x);
  */
 void exit_failure(void);
 
-
 /**
  * \brief erase a string from the terminal
  */