added HTTP 429 handling for file stat query

src/link.c

@@ -11,13 +11,6 @@
 #include <string.h>
 #include <unistd.h>
 
-typedef enum {
-    HTTP_OK                     = 200,
-    HTTP_PARTIAL_CONTENT        = 206,
-    HTTP_RANGE_NOT_SATISFIABLE  = 416,
-    HTTP_TOO_MANY_REQUESTS      = 429
-}HTTPResponseCode;
-
 /* ---------------- External variables -----------------------*/
 LinkTable *ROOT_LINK_TBL = NULL;
 int ROOT_LINK_OFFSET = 0;
@@ -278,7 +271,7 @@ LinkTable *LinkTable_new(const char *url)
         if (http_resp == HTTP_TOO_MANY_REQUESTS) {
             fprintf(stderr, "link.c: LinkTable_new(): URL: %s, HTTP 429, \
 Too Many Requests\n", url);
-            sleep(5);
+            sleep(HTTP_429_WAIT);
         } else if (http_resp != HTTP_OK) {
             fprintf(stderr, "link.c: LinkTable_new(): cannot retrieve the base \
 URL, URL: %s, HTTP %ld\n", url, http_resp);

src/link.h

@@ -50,6 +50,11 @@ extern LinkTable *ROOT_LINK_TBL;
  */
 extern int ROOT_LINK_OFFSET;
 
+/**
+ * \brief
+ */
+void Link_get_stat(Link *this_link);
+
 /**
  * \brief set the stats for a file
  */

src/network.c

@@ -98,6 +98,19 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs
         transfer->transferring = 0;
         char *url = NULL;
         curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
+
+        /* Wait for 5 seconds if we get HTTP 429 */
+        long http_resp = 0;
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
+        if (http_resp == HTTP_TOO_MANY_REQUESTS) {
+            fprintf(stderr, "curl_process_msgs(): HTTP 429\n");
+            sleep(HTTP_429_WAIT);
+            /* Re-add the link into the queue, if it is a file stat query */
+            if (transfer->type == FILESTAT) {
+                Link_get_stat(transfer->link);
+            }
+        }
+
         if (curl_msg->data.result) {
             fprintf(stderr, "curl_process_msgs(): %d - %s <%s>\n",
                     curl_msg->data.result,
@@ -107,11 +120,10 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs
         } else {
             /* Transfer successful, query the file size */
             if (transfer->type == FILESTAT) {
-//                 fprintf(stderr, "Link_set_stat(): %d, %d, %s\n",
-//                         n_running_curl, n_mesgs, url);
                 Link_set_stat(transfer->link, curl);
             }
         }
+
         curl_multi_remove_handle(curl_multi, curl);
         /* clean up the handle, if we are querying the file size */
         if (transfer->type == FILESTAT) {

src/network.h

@@ -3,16 +3,25 @@
 
 #include "link.h"
 
-typedef struct {
-    char *memory;
-    size_t size;
-} MemoryStruct;
+#define HTTP_429_WAIT   5
+
+typedef enum {
+    HTTP_OK                     = 200,
+    HTTP_PARTIAL_CONTENT        = 206,
+    HTTP_RANGE_NOT_SATISFIABLE  = 416,
+    HTTP_TOO_MANY_REQUESTS      = 429
+}HTTPResponseCode;
 
 typedef enum {
     FILESTAT = 's',
     DATA = 'd'
 } TransferType;
 
+typedef struct {
+    char *memory;
+    size_t size;
+} MemoryStruct;
+
 typedef struct {
     TransferType type;
     int transferring;