From 16d89b9ad047749e420ee36d0f8b667e0356e593 Mon Sep 17 00:00:00 2001 From: Fufu Fang Date: Sun, 22 Jul 2018 19:30:08 +0100 Subject: [PATCH] we are encountering some sort of thread safety issue --- data.h | 45 ++++++------------------------------- main.c | 35 ++++++++++++++++++++++++----- network.c | 66 +++++++++++++++++++++++++++++++++++++------------------ network.h | 47 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 65 deletions(-) diff --git a/data.h b/data.h index 9dd26fb..44db5b5 100644 --- a/data.h +++ b/data.h @@ -5,47 +5,16 @@ * \brief This header stores all the custom data type definition */ -#include +#include -#define URL_LEN_MAX 2048 -#define LINK_LEN_MAX 255 -#define HTTP_OK 200 +/** \brief use this data type for buffer */ +typedef struct MemoryStruct MemoryStruct; -/** \brief the link type */ -typedef enum { - LINK_HEAD = 'H', - LINK_DIR = 'D', - LINK_FILE = 'F', - LINK_UNKNOWN = 'U', - LINK_INVALID = 'I' -} LinkType; - -/** - * \brief link table type - * \details index 0 contains the Link for the base URL - */ -typedef struct LinkTable LinkTable; - -/** \brief link data type */ -typedef struct Link Link; - -/** \brief root link table */ -extern LinkTable *ROOT_LINK_TBL; - -struct Link { - char p_url[LINK_LEN_MAX]; - char f_url[URL_LEN_MAX]; - LinkType type; - CURL *curl; - char *body; - size_t body_sz; - size_t content_length; - LinkTable *next_table; +struct MemoryStruct { + size_t size; + char *memory; }; -struct LinkTable { - int num; - Link **links; -}; +void MemoryStruct_free(MemoryStruct *ms); #endif diff --git a/main.c b/main.c index ecb57fa..d6edf23 100644 --- a/main.c +++ b/main.c @@ -23,9 +23,13 @@ static struct fuse_operations fs_oper = { .read = fs_read, }; + int main(int argc, char **argv) { - network_init(argv[1]); - return fuse_main(argc - 1, argv + 1, &fs_oper, NULL); + network_init("http://127.0.0.1/~fangfufu/"); + +// return fuse_main(argc - 1, argv + 1, &fs_oper, NULL); + return fuse_main(argc, argv, &fs_oper, NULL); + return 0; } /** \brief return the attributes for a single file indicated by path */ @@ -57,6 +61,7 @@ static int fs_getattr(const char *path, struct stat *stbuf) } } + fflush(stderr); return res; } @@ -80,7 +85,11 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add, } linktbl = link->next_table; if (!linktbl) { - return -ENOENT; + fprintf(stderr, "LinkTable_new(): %s\n", link->f_url); + linktbl = LinkTable_new(link->f_url); + if(!linktbl) { + return -ENOENT; + } } } @@ -92,6 +101,7 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add, dir_add(buf, link->p_url, NULL, 0); } + fflush(stderr); return 0; } @@ -106,6 +116,7 @@ static int fs_open(const char *path, struct fuse_file_info *fi) return -EACCES; } + fflush(stderr); return 0; } @@ -121,8 +132,22 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset, return -ENOENT; } - int bytes_downloaded = Link_download(link, offset, size); - memcpy(buf, link->body, bytes_downloaded); + MemoryStruct ms; + + fprintf(stderr, "fs_read(): Link_download(%s, %ld, %lu)\n", + link->f_url, + offset, + size + ); + fflush(stderr); + size_t bytes_downloaded = Link_download(link, &ms, offset, size); + fprintf(stderr, + "fs_read(): returning from Link_download(), with %lu bytes.\n", + bytes_downloaded); + fflush(stderr); + + memcpy(buf, ms.memory, size); +// free(ms.memory); return bytes_downloaded; } diff --git a/network.c b/network.c index af9b93d..39ad92f 100644 --- a/network.c +++ b/network.c @@ -6,7 +6,8 @@ #include #include -#define NETWORK_MAXIMUM_CONNECTION 10 +#define HTTP_OK 200 +#define HTTP_PARTIAL_CONTENT 206 LinkTable *ROOT_LINK_TBL; @@ -17,7 +18,6 @@ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp); static Link *Link_new(const char *p_url); static void Link_free(Link *link); -static LinkTable *LinkTable_new(const char *url); static void LinkTable_free(LinkTable *linktbl); static void LinkTable_add(LinkTable *linktbl, Link *link); static void LinkTable_fill(LinkTable *linktbl); @@ -64,7 +64,23 @@ static char *url_append(const char *url, const char *sublink) return str; } +/* This is the single thread version */ static void do_transfer(CURL *curl) +{ + fprintf(stderr, "do_transfer(): handle %x\n", curl); + fflush(stderr); + CURLcode res = curl_easy_perform(curl); + if (res) { + fprintf(stderr, "do_transfer() failed: %u, %s\n", res, + curl_easy_strerror(res)); + fflush(stderr); + } + fprintf(stderr, "do_transfer(): DONE\n"); + fflush(stderr); +} + +/* This is the version that uses curl multi handle */ +static void do_transfer_(CURL *curl) { /* Add the transfer handle */ curl_multi_add_handle(curl_multi, curl); @@ -74,6 +90,9 @@ static void do_transfer(CURL *curl) long timeout; fd_set read_fd_set, write_fd_set, exc_fd_set; do { + fprintf(stderr, "do_transfer(): num_transfers: %d\n", + num_transfers); + fflush(stderr); res = curl_multi_perform(curl_multi, &num_transfers); if (res) { fprintf(stderr, @@ -135,29 +154,26 @@ static void do_transfer(CURL *curl) static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { + MemoryStruct *mem = (MemoryStruct *)userp; size_t realsize = size * nmemb; - Link *mem = (Link *)userp; - mem->body = realloc(mem->body, mem->body_sz + realsize + 1); - if(mem->body == NULL) { + mem->memory = malloc(realsize + 1); + if(!mem->memory) { /* out of memory! */ printf("not enough memory (realloc returned NULL)\n"); return 0; } - memcpy(&(mem->body[mem->body_sz]), contents, realsize); - mem->body_sz += realsize; - mem->body[mem->body_sz] = 0; - - return realsize; + memcpy(mem->memory, contents, realsize); + return realsize - 1; } void network_init(const char *url) { curl_global_init(CURL_GLOBAL_ALL); - curl_multi = curl_multi_init(); - curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS, - (long)NETWORK_MAXIMUM_CONNECTION); +// curl_multi = curl_multi_init(); +// curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS, +// (long)NETWORK_MAXIMUM_CONNECTION); ROOT_LINK_TBL = LinkTable_new(url); } @@ -179,7 +195,6 @@ static Link *Link_new(const char *p_url) /* set up some basic curl stuff */ curl_easy_setopt(link->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); - curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)link); curl_easy_setopt(link->curl, CURLOPT_USERAGENT, "mount-http-dir/libcurl"); curl_easy_setopt(link->curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(link->curl, CURLOPT_FOLLOWLOCATION, 1); @@ -195,12 +210,12 @@ static Link *Link_new(const char *p_url) static void Link_free(Link *link) { curl_easy_cleanup(link->curl); - free(link->body); free(link); link = NULL; } -size_t Link_download(Link *link, off_t offset, size_t size) +size_t Link_download(Link *link, MemoryStruct *ms, off_t offset, + size_t size) { size_t start = offset; size_t end = start + size; @@ -209,13 +224,16 @@ size_t Link_download(Link *link, off_t offset, size_t size) snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end); curl_easy_setopt(curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(curl, CURLOPT_URL, link->f_url); curl_easy_setopt(curl, CURLOPT_RANGE, range_str); + curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)ms); + do_transfer(curl); long http_resp; - curl_easy_getinfo(link->curl, CURLINFO_RESPONSE_CODE, &http_resp); - if (http_resp != HTTP_OK) { - fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld", + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp); + if ( (http_resp != HTTP_OK) && ( http_resp != HTTP_PARTIAL_CONTENT) ) { + fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld\n", link->f_url, http_resp); return 0; } @@ -223,10 +241,12 @@ size_t Link_download(Link *link, off_t offset, size_t size) double dl; curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dl); size_t s = dl; + + fflush(stdout); return s; } -static LinkTable *LinkTable_new(const char *url) +LinkTable *LinkTable_new(const char *url) { LinkTable *linktbl = calloc(1, sizeof(LinkTable)); @@ -237,6 +257,9 @@ static LinkTable *LinkTable_new(const char *url) curl_easy_setopt(head_link->curl, CURLOPT_URL, url); /* start downloading the base URL */ + MemoryStruct ms; + curl_easy_setopt(head_link->curl, CURLOPT_WRITEDATA, (void *)&ms); + do_transfer(head_link->curl); /* if downloading base URL failed */ @@ -251,7 +274,8 @@ URL: %s, HTTP %ld\n", url, http_resp); }; /* Otherwise parsed the received data */ - GumboOutput* output = gumbo_parse(head_link->body); + GumboOutput* output = gumbo_parse(ms.memory); + free(ms.memory); HTML_to_LinkTable(output->root, linktbl); gumbo_destroy_output(&kGumboDefaultOptions, output); diff --git a/network.h b/network.h index a9ef4e8..4f2df85 100644 --- a/network.h +++ b/network.h @@ -3,8 +3,49 @@ #include "data.h" +#include #include +#define URL_LEN_MAX 2048 +#define LINK_LEN_MAX 255 +#define NETWORK_MAXIMUM_CONNECTION 10 + +/** \brief the link type */ +typedef enum { + LINK_HEAD = 'H', + LINK_DIR = 'D', + LINK_FILE = 'F', + LINK_UNKNOWN = 'U', + LINK_INVALID = 'I' +} LinkType; + +/** + * \brief link table type + * \details index 0 contains the Link for the base URL + */ +typedef struct LinkTable LinkTable; + +/** \brief link data type */ +typedef struct Link Link; + + +struct Link { + char p_url[LINK_LEN_MAX]; + char f_url[URL_LEN_MAX]; + LinkType type; + CURL *curl; + size_t content_length; + LinkTable *next_table; +}; + +struct LinkTable { + int num; + Link **links; +}; + +/** \brief root link table */ +extern LinkTable *ROOT_LINK_TBL; + /** \brief Initialise the network module */ void network_init(const char *url); @@ -12,7 +53,11 @@ void network_init(const char *url); * \brief download a link */ /* \return the number of bytes downloaded */ -size_t Link_download(Link *link, off_t offset, size_t size); +size_t Link_download(Link *link, MemoryStruct *mem, off_t offset, + size_t size); + +/** \brief create a new LinkTable */ +LinkTable *LinkTable_new(const char *url); /** \brief print a LinkTable */ void LinkTable_print(LinkTable *linktbl);