rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

finally the performance is great

This commit is contained in:
Fufu Fang 2018-07-24 05:58:51 +01:00
parent eae3670992
commit eec104c841
2 changed files with 52 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
main.c
View File

@ -113,10 +113,6 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
} }
linktbl = link->next_table; linktbl = link->next_table;
if (!linktbl) { if (!linktbl) {
#ifdef HTTPDIRFS_DEBUG
fprintf(stderr, "LinkTable_new(): %s\n", link->f_url);
#endif
linktbl = LinkTable_new(link->f_url); linktbl = LinkTable_new(link->f_url);
if(!linktbl) { if(!linktbl) {
return -ENOENT; return -ENOENT;
@ -134,7 +130,6 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
} }
} }
return 0; return 0;
} }
@ -149,7 +144,6 @@ static int fs_open(const char *path, struct fuse_file_info *fi)
return -EACCES; return -EACCES;
} }
return 0; return 0;
} }
@ -158,15 +152,8 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi) struct fuse_file_info *fi)
{ {
(void) fi; (void) fi;
#ifdef HTTPDIRFS_DEBUG
fprintf(stderr, "fs_read(): path: %s, offset: %ld, size: %lu\n",
path, offset, size);
#endif
long received = Link_download(path, buf, size, offset); long received = Link_download(path, buf, size, offset);
#ifdef HTTPDIRFS_DEBUG
fprintf(stderr, "fs_read(): received %ld bytes.\n", received);
#endif
return received; return received;
} }

103
network.c
View File

@ -18,8 +18,8 @@ typedef struct {
} MemoryStruct; } MemoryStruct;
typedef enum { typedef enum {
FILESIZE, FILESIZE = 's',
DATA DATA = 'd'
} TransferType; } TransferType;
typedef struct { typedef struct {
@ -43,7 +43,7 @@ static CURLM *curl_multi;
/* Link related */ /* Link related */
static Link *Link_new(const char *p_url, LinkType type); static Link *Link_new(const char *p_url, LinkType type);
static size_t Link_get_size(Link *this_link); static void Link_get_size(Link *this_link);
static Link *path_to_Link_recursive(char *path, LinkTable *linktbl); static Link *path_to_Link_recursive(char *path, LinkTable *linktbl);
static CURL *Link_to_curl(Link *link); static CURL *Link_to_curl(Link *link);
static LinkType p_url_type(const char *p_url); static LinkType p_url_type(const char *p_url);
@ -56,7 +56,6 @@ static void LinkTable_free(LinkTable *linktbl);
static void HTML_to_LinkTable(GumboNode *node, LinkTable *linktbl); static void HTML_to_LinkTable(GumboNode *node, LinkTable *linktbl);
/* Transfer related */ /* Transfer related */
static void transfer_wrapper(CURL *curl, TransferType type);
static void blocking_transfer(CURL *curl); static void blocking_transfer(CURL *curl);
static void nonblocking_transfer(CURL *curl); static void nonblocking_transfer(CURL *curl);
static int curl_multi_perform_once(); static int curl_multi_perform_once();
@ -64,14 +63,6 @@ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb,
void *userp); void *userp);
/* -------------------------- Functions ---------------------------------- */ /* -------------------------- Functions ---------------------------------- */
static void transfer_wrapper(CURL *curl, TransferType type)
{
if (type == DATA) {
blocking_transfer(curl);
} else if (type == FILESIZE) {
nonblocking_transfer(curl);
}
}
static void nonblocking_transfer(CURL *curl) static void nonblocking_transfer(CURL *curl)
{ {
@ -86,10 +77,13 @@ static void nonblocking_transfer(CURL *curl)
/* This uses the curl multi interface */ /* This uses the curl multi interface */
static void blocking_transfer(CURL *curl) static void blocking_transfer(CURL *curl)
{ {
volatile TransferStruct transfer; TransferStruct *transfer = malloc(sizeof(TransferStruct));
transfer.type = DATA; if (!transfer) {
transfer.transferring = 1; fprintf(stderr, "blocking_transfer(): malloc failed!\n");
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer); }
transfer->type = DATA;
transfer->transferring = 1;
curl_easy_setopt(curl, CURLOPT_PRIVATE, transfer);
CURLMcode res = curl_multi_add_handle(curl_multi, curl); CURLMcode res = curl_multi_add_handle(curl_multi, curl);
if(res > 0) { if(res > 0) {
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n", fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
@ -97,9 +91,10 @@ static void blocking_transfer(CURL *curl)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
while (transfer.transferring) { while (transfer->transferring) {
curl_multi_perform_once(); curl_multi_perform_once();
} }
free(transfer);
} }
/** /**
@ -173,8 +168,34 @@ static int curl_multi_perform_once()
curl_msg->data.result, curl_msg->data.result,
curl_easy_strerror(curl_msg->data.result), curl_easy_strerror(curl_msg->data.result),
url); url);
} else {
/* Transfer successful, query the file size */
if (transfer->type == FILESIZE) {
Link *this_link = transfer->link;
long http_resp;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
if (http_resp == HTTP_OK) {
double cl = 0;
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
if (cl == -1) {
/* Turns out not to be a file after all */
this_link->content_length = 0;
this_link->type = LINK_DIR;
} else {
this_link->content_length = cl;
this_link->type = LINK_FILE;
}
} else {
this_link->type = LINK_INVALID;
}
}
} }
curl_multi_remove_handle(curl_multi, curl); curl_multi_remove_handle(curl_multi, curl);
/* clean up the handle, if we are querying the file size */
if (transfer->type == FILESIZE) {
curl_easy_cleanup(curl);
free(transfer);
}
} else { } else {
fprintf(stderr, "curl_multi_perform_once(): curl_msg->msg: %d\n", fprintf(stderr, "curl_multi_perform_once(): curl_msg->msg: %d\n",
curl_msg->msg); curl_msg->msg);
@ -351,7 +372,7 @@ long Link_download(const char *path, char *output_buf, size_t size,
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str); curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
transfer_wrapper(curl, DATA); blocking_transfer(curl);
long http_resp; long http_resp;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
@ -400,7 +421,7 @@ LinkTable *LinkTable_new(const char *url)
// curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); // curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
transfer_wrapper(curl, DATA); blocking_transfer(curl);
/* if downloading base URL failed */ /* if downloading base URL failed */
long http_resp; long http_resp;
@ -446,42 +467,26 @@ static void LinkTable_add(LinkTable *linktbl, Link *link)
linktbl->links[linktbl->num - 1] = link; linktbl->links[linktbl->num - 1] = link;
} }
size_t Link_get_size(Link *this_link) void Link_get_size(Link *this_link)
{ {
#ifdef HTTPDIRFS_INFO #ifdef HTTPDIRFS_INFO
fprintf(stderr, "Link_get_size(%s);\n", this_link->f_url); fprintf(stderr, "Link_get_size(%s);\n", this_link->f_url);
#endif #endif
double cl = 0;
if (this_link->type == LINK_FILE) { if (this_link->type == LINK_FILE) {
CURL *curl = Link_to_curl(this_link); CURL *curl = Link_to_curl(this_link);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1); curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
TransferStruct transfer; TransferStruct *transfer = malloc(sizeof(TransferStruct));
transfer.link = this_link; if (!transfer) {
transfer.type = FILESIZE; fprintf(stderr, "Link_get_size(): malloc failed!\n");
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
transfer_wrapper(curl, DATA);
long http_resp;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
if (http_resp == HTTP_OK) {
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
if (cl == -1) {
/* Turns out not to be a file after all */
this_link->content_length = 0;
this_link->type = LINK_DIR;
} else {
this_link->content_length = cl;
this_link->type = LINK_FILE;
}
} else {
this_link->type = LINK_INVALID;
} }
curl_easy_cleanup(curl); transfer->link = this_link;
} transfer->type = FILESIZE;
curl_easy_setopt(curl, CURLOPT_PRIVATE, transfer);
return cl; nonblocking_transfer(curl);
}
} }
void LinkTable_fill(LinkTable *linktbl) void LinkTable_fill(LinkTable *linktbl)
@ -495,7 +500,7 @@ void LinkTable_fill(LinkTable *linktbl)
strncpy(this_link->f_url, url, URL_LEN_MAX); strncpy(this_link->f_url, url, URL_LEN_MAX);
free(url); free(url);
if (this_link->type == LINK_FILE && !(this_link->content_length)) { if (this_link->type == LINK_FILE && !(this_link->content_length)) {
this_link->content_length = Link_get_size(this_link); Link_get_size(this_link);
} }
} }
} }
@ -626,10 +631,6 @@ static Link *path_to_Link_recursive(char *path, LinkTable *linktbl)
} }
} }
} }
#ifdef HTTPDIRFS_DEBUG
fprintf(stderr, "path_to_Link(): %s does not exist.\n", path);
#endif
return NULL; return NULL;
} }