Merge f361527459 into 81aac8bb57

- Fix for not sending ranges which exceed the content-length which will result
in an error.
- Fix for byte range being set to 1 byte too large, it should be the end index,
not the size as described in
https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests

src/cache.c

@@ -164,10 +164,10 @@ static int Meta_read(Cache *cf)
     }
 
     if (sizeof(long) != fread(&cf->time, sizeof(long), 1, fp) ||
-        sizeof(off_t) != fread(&cf->content_length, sizeof(off_t), 1, fp) ||
-        sizeof(int) != fread(&cf->blksz, sizeof(int), 1, fp) ||
-        sizeof(long) != fread(&cf->segbc, sizeof(long), 1, fp) ||
-        ferror(fp)) {
+            sizeof(off_t) != fread(&cf->content_length, sizeof(off_t), 1, fp) ||
+            sizeof(int) != fread(&cf->blksz, sizeof(int), 1, fp) ||
+            sizeof(long) != fread(&cf->segbc, sizeof(long), 1, fp) ||
+            ferror(fp)) {
         lprintf(error, "error reading core metadata!\n");
         return EIO;
     }

src/link.c

@@ -237,11 +237,13 @@ static void LinkTable_uninitialised_fill(LinkTable *linktbl)
         }
         /*
          * Block until the gaps are filled
+         * result is an invalid variable
          */
-        int n = curl_multi_perform_once();
+        int result = 0;
+        int n = curl_multi_perform_once(&result);
         int i = 0;
         int j = 0;
-        while ((i = curl_multi_perform_once())) {
+        while ((i = curl_multi_perform_once(&result))) {
             if (CONFIG.log_type & debug) {
                 if (j) {
                     erase_string(stderr, STATUS_LEN, s);
@@ -286,7 +288,7 @@ LinkTable *LinkSystem_init(const char *url)
      * This is where the '/' should be
      */
     ROOT_LINK_OFFSET = strnlen(url, MAX_PATH_LEN) -
-      ((url[url_len] == '/') ? 1 : 0);
+                       ((url[url_len] == '/') ? 1 : 0);
 
     /*
      * --------------------- Enable cache system --------------------
@@ -474,7 +476,7 @@ static void LinkTable_fill(LinkTable *linktbl)
            spaces in them!). If we only escaped it, and there were already
            encoded characters in it, then that would break the link. */
         char *unescaped_path = curl_easy_unescape(c, this_link->linkpath, 0,
-                                                  NULL);
+                               NULL);
         char *escaped_path = curl_easy_escape(c, unescaped_path, 0);
         curl_free(unescaped_path);
         /* Our code does the wrong thing if there's a trailing slash that's been
@@ -704,7 +706,10 @@ int LinkTable_disk_save(LinkTable *linktbl, const char *dirn)
 /* This is necessary to get the compiler on some platforms to stop
    complaining about the fact that we're not using the return value of
    fread, when we know we aren't and that's fine. */
-static inline void ignore_value(int i) { (void) i; }
+static inline void ignore_value(int i)
+{
+    (void) i;
+}
 
 LinkTable *LinkTable_disk_open(const char *dirn)
 {
@@ -799,6 +804,8 @@ LinkTable *path_to_Link_LinkTable_new(const char *path)
     if (!next_table) {
         if (CONFIG.mode == NORMAL) {
             next_table = LinkTable_new(tmp_link->f_url);
+        } else if (CONFIG.mode == SINGLE) {
+            next_table = single_LinkTable_new(tmp_link->f_url);
         } else if (CONFIG.mode == SONIC) {
             if (!CONFIG.sonic_id3) {
                 next_table = sonic_LinkTable_new_index(tmp_link->sonic.id);
@@ -808,7 +815,7 @@ LinkTable *path_to_Link_LinkTable_new(const char *path)
                                             tmp_link->sonic.id);
             }
         } else {
-            lprintf(fatal, "Invalid CONFIG.mode\n");
+            lprintf(fatal, "Invalid CONFIG.mode: %d\n", CONFIG.mode);
         }
     }
     if (link) {
@@ -944,7 +951,7 @@ TransferStruct Link_download_full(Link *link)
      */
     long http_resp = 0;
     do {
-        transfer_blocking(curl);
+        transfer_blocking(curl, ts.curr_size);
         ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
         if (ret) {
             lprintf(error, "%s", curl_easy_strerror(ret));
@@ -981,7 +988,7 @@ static CURL *Link_download_curl_setup(Link *link, size_t req_size, off_t offset,
     }
 
     size_t start = offset;
-    size_t end = start + req_size;
+    size_t end = start + req_size - 1;
 
     char range_str[64];
     snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
@@ -1030,8 +1037,8 @@ range requests\n");
         lprintf(error, "%s", curl_easy_strerror(ret));
     }
     if ((http_resp != HTTP_OK) &&
-	(http_resp != HTTP_PARTIAL_CONTENT) &&
-	(http_resp != HTTP_RANGE_NOT_SATISFIABLE)) {
+            (http_resp != HTTP_PARTIAL_CONTENT) &&
+            (http_resp != HTTP_RANGE_NOT_SATISFIABLE)) {
         char *url;
         curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
         lprintf(warning, "Could not download %s, HTTP %ld\n", url, http_resp);
@@ -1061,9 +1068,16 @@ long Link_download(Link *link, char *output_buf, size_t req_size, off_t offset)
     header.curr_size = 0;
     header.data = NULL;
 
+    if (offset + req_size > link->content_length) {
+        lprintf(error,
+                "requested size too large, req_size: %lu, recv: %ld, content-length: %ld\n",
+                req_size, recv, link->content_length);
+        req_size = link->content_length - offset;
+    }
+
     CURL *curl = Link_download_curl_setup(link, req_size, offset, &header, &ts);
 
-    transfer_blocking(curl);
+    transfer_blocking(curl, offset);
 
     curl_off_t recv = Link_download_cleanup(curl, &header);
 
@@ -1096,7 +1110,7 @@ long path_download(const char *path, char *output_buf, size_t req_size,
     return Link_download(link, output_buf, req_size, offset);
 }
 
-static void make_link_relative(const char *page_url, char *link_url) 
+static void make_link_relative(const char *page_url, char *link_url)
 {
     /*
       Some servers make the links to subdirectories absolute (in URI terms:
@@ -1139,14 +1153,14 @@ static void make_link_relative(const char *page_url, char *link_url)
         page_url++;
     }
     if (slashes_left_to_find) {
-      if (slashes_left_to_find == 1 && ! *page_url)
-        /* We're at the top level of the web site and the user entered the URL
-           without a trailing slash. */
-        page_url = "/";
-      else
-        /* Well, that's odd. Let's return rather than trying to dig ourselves
-           deeper into whatever hole we're in. */
-        return;
+        if (slashes_left_to_find == 1 && ! *page_url)
+            /* We're at the top level of the web site and the user entered the URL
+               without a trailing slash. */
+            page_url = "/";
+        else
+            /* Well, that's odd. Let's return rather than trying to dig ourselves
+               deeper into whatever hole we're in. */
+            return;
     }
     /* The page URL is no longer the full page_url, it's just the part after
        the host name. */

src/network.c

@@ -10,6 +10,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <unistd.h>
+#include <stdbool.h>
 
 /*
  * ----------------- External variables ----------------------
@@ -110,9 +111,10 @@ curl_callback_unlock(CURL *handle, curl_lock_data data, void *userptr)
  * \details Adapted from:
  * https://curl.haxx.se/libcurl/c/10-at-a-time.html
  */
-static void
+static int
 curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
 {
+    int result = 0;
     (void) n_running_curl;
     (void) n_mesgs;
     static volatile int slept = 0;
@@ -163,6 +165,7 @@ curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
             lprintf(error, "%d - %s <%s>\n",
                     curl_msg->data.result,
                     curl_easy_strerror(curl_msg->data.result), url);
+                    result = curl_msg->data.result;
         }
         curl_multi_remove_handle(curl_multi, curl);
         /*
@@ -175,13 +178,50 @@ curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs)
     } else {
         lprintf(warning, "curl_msg->msg: %d\n", curl_msg->msg);
     }
+    return result;
 }
 
+static int http_error_result(int http_response)
+{
+    switch(http_response)
+    {
+        case 0:   //eg connection down  from kick-off ~suggest retrying till some max limit
+        case 200: //yay we at least got to our url
+        case 206: //Partial Content
+        break;
+
+        case 416:
+        //cannot d/l range ~ either cos no server support
+        //or cos we're asking for an invalid range ~ie: we already d/ld the file
+        printf("HTTP416: either the d/l is already complete or the http server cannot d/l a range\n");
+        default:
+            return 0;//suggest quitting on an unhandled error
+    }
+
+    return 1;
+}
+
+static int curl_error_result(int curl_result)
+{
+    switch (curl_result)
+    {
+        case CURLE_OK:
+        case CURLE_COULDNT_CONNECT:      //no network connectivity ?
+        case CURLE_OPERATION_TIMEDOUT:   //cos of CURLOPT_LOW_SPEED_TIME
+        case CURLE_COULDNT_RESOLVE_HOST: //host/DNS down ?
+            break; //we'll keep trying
+        default://see: http://curl.haxx.se/libcurl/c/libcurl-errors.html
+            return 0;
+    }
+    return 1;
+}
+
+
 /**
  * \details  effectively based on
  * https://curl.haxx.se/libcurl/c/multi-double.html
  */
-int curl_multi_perform_once(void)
+int curl_multi_perform_once(int *result)
 {
     lprintf(network_lock_debug,
             "thread %x: locking transfer_lock;\n", pthread_self());
@@ -207,7 +247,12 @@ int curl_multi_perform_once(void)
     int n_mesgs;
     CURLMsg *curl_msg;
     while ((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
-        curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
+        int nResult = curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
+        if (!http_error_result(n_mesgs) || !curl_error_result(nResult)) {
+            *result = 1;
+        }else{
+            *result = 0;
+        }
     }
 
     lprintf(network_lock_debug,
@@ -272,7 +317,7 @@ void NetworkSystem_init(void)
     crypto_lock_init();
 }
 
-void transfer_blocking(CURL *curl)
+void transfer_blocking(CURL *curl, size_t start)
 {
     TransferStruct *ts;
     CURLcode ret = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &ts);
@@ -293,8 +338,22 @@ void transfer_blocking(CURL *curl)
             "thread %x: unlocking transfer_lock;\n", pthread_self());
     PTHREAD_MUTEX_UNLOCK(&transfer_lock);
 
-    while (ts->transferring) {
-        curl_multi_perform_once();
+    int result = 0;
+    bool restartDown = false;
+
+    while (ts->transferring && !restartDown) {
+        /*
+        * When the network is abnormal during the file download, start to resume the transfer
+        */
+        if (0 != result) {
+            curl_multi_remove_handle(curl_multi,curl);
+            curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, start);
+            res = curl_multi_add_handle(curl_multi, curl);
+            if (res > 0) {
+                lprintf(error, "%d, %s\n", res, curl_multi_strerror(res));
+            }
+        }
+        curl_multi_perform_once(&result);
     }
 }
 

src/network.h

@@ -26,13 +26,13 @@ typedef enum {
 extern CURLSH *CURL_SHARE;
 
 /** \brief perform one transfer cycle */
-int curl_multi_perform_once(void);
+int curl_multi_perform_once(int *result);
 
 /** \brief initialise the network module */
 void NetworkSystem_init(void);
 
 /** \brief blocking file transfer */
-void transfer_blocking(CURL *curl);
+void transfer_blocking(CURL *curl, size_t start);
 
 /** \brief non blocking file transfer */
 void transfer_nonblocking(CURL *curl);