mirror of
https://github.com/fangfufu/httpdirfs.git
synced 2024-09-30 14:11:16 +02:00
finally the performance is great
This commit is contained in:
parent
eae3670992
commit
eec104c841
13
main.c
13
main.c
@ -113,10 +113,6 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
|
|||||||
}
|
}
|
||||||
linktbl = link->next_table;
|
linktbl = link->next_table;
|
||||||
if (!linktbl) {
|
if (!linktbl) {
|
||||||
#ifdef HTTPDIRFS_DEBUG
|
|
||||||
fprintf(stderr, "LinkTable_new(): %s\n", link->f_url);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
linktbl = LinkTable_new(link->f_url);
|
linktbl = LinkTable_new(link->f_url);
|
||||||
if(!linktbl) {
|
if(!linktbl) {
|
||||||
return -ENOENT;
|
return -ENOENT;
|
||||||
@ -134,7 +130,6 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -149,7 +144,6 @@ static int fs_open(const char *path, struct fuse_file_info *fi)
|
|||||||
return -EACCES;
|
return -EACCES;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -158,15 +152,8 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset,
|
|||||||
struct fuse_file_info *fi)
|
struct fuse_file_info *fi)
|
||||||
{
|
{
|
||||||
(void) fi;
|
(void) fi;
|
||||||
#ifdef HTTPDIRFS_DEBUG
|
|
||||||
fprintf(stderr, "fs_read(): path: %s, offset: %ld, size: %lu\n",
|
|
||||||
path, offset, size);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
long received = Link_download(path, buf, size, offset);
|
long received = Link_download(path, buf, size, offset);
|
||||||
#ifdef HTTPDIRFS_DEBUG
|
|
||||||
fprintf(stderr, "fs_read(): received %ld bytes.\n", received);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
return received;
|
return received;
|
||||||
}
|
}
|
||||||
|
103
network.c
103
network.c
@ -18,8 +18,8 @@ typedef struct {
|
|||||||
} MemoryStruct;
|
} MemoryStruct;
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
FILESIZE,
|
FILESIZE = 's',
|
||||||
DATA
|
DATA = 'd'
|
||||||
} TransferType;
|
} TransferType;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@ -43,7 +43,7 @@ static CURLM *curl_multi;
|
|||||||
|
|
||||||
/* Link related */
|
/* Link related */
|
||||||
static Link *Link_new(const char *p_url, LinkType type);
|
static Link *Link_new(const char *p_url, LinkType type);
|
||||||
static size_t Link_get_size(Link *this_link);
|
static void Link_get_size(Link *this_link);
|
||||||
static Link *path_to_Link_recursive(char *path, LinkTable *linktbl);
|
static Link *path_to_Link_recursive(char *path, LinkTable *linktbl);
|
||||||
static CURL *Link_to_curl(Link *link);
|
static CURL *Link_to_curl(Link *link);
|
||||||
static LinkType p_url_type(const char *p_url);
|
static LinkType p_url_type(const char *p_url);
|
||||||
@ -56,7 +56,6 @@ static void LinkTable_free(LinkTable *linktbl);
|
|||||||
static void HTML_to_LinkTable(GumboNode *node, LinkTable *linktbl);
|
static void HTML_to_LinkTable(GumboNode *node, LinkTable *linktbl);
|
||||||
|
|
||||||
/* Transfer related */
|
/* Transfer related */
|
||||||
static void transfer_wrapper(CURL *curl, TransferType type);
|
|
||||||
static void blocking_transfer(CURL *curl);
|
static void blocking_transfer(CURL *curl);
|
||||||
static void nonblocking_transfer(CURL *curl);
|
static void nonblocking_transfer(CURL *curl);
|
||||||
static int curl_multi_perform_once();
|
static int curl_multi_perform_once();
|
||||||
@ -64,14 +63,6 @@ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb,
|
|||||||
void *userp);
|
void *userp);
|
||||||
|
|
||||||
/* -------------------------- Functions ---------------------------------- */
|
/* -------------------------- Functions ---------------------------------- */
|
||||||
static void transfer_wrapper(CURL *curl, TransferType type)
|
|
||||||
{
|
|
||||||
if (type == DATA) {
|
|
||||||
blocking_transfer(curl);
|
|
||||||
} else if (type == FILESIZE) {
|
|
||||||
nonblocking_transfer(curl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void nonblocking_transfer(CURL *curl)
|
static void nonblocking_transfer(CURL *curl)
|
||||||
{
|
{
|
||||||
@ -86,10 +77,13 @@ static void nonblocking_transfer(CURL *curl)
|
|||||||
/* This uses the curl multi interface */
|
/* This uses the curl multi interface */
|
||||||
static void blocking_transfer(CURL *curl)
|
static void blocking_transfer(CURL *curl)
|
||||||
{
|
{
|
||||||
volatile TransferStruct transfer;
|
TransferStruct *transfer = malloc(sizeof(TransferStruct));
|
||||||
transfer.type = DATA;
|
if (!transfer) {
|
||||||
transfer.transferring = 1;
|
fprintf(stderr, "blocking_transfer(): malloc failed!\n");
|
||||||
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
|
}
|
||||||
|
transfer->type = DATA;
|
||||||
|
transfer->transferring = 1;
|
||||||
|
curl_easy_setopt(curl, CURLOPT_PRIVATE, transfer);
|
||||||
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
|
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
|
||||||
if(res > 0) {
|
if(res > 0) {
|
||||||
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
|
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
|
||||||
@ -97,9 +91,10 @@ static void blocking_transfer(CURL *curl)
|
|||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
while (transfer.transferring) {
|
while (transfer->transferring) {
|
||||||
curl_multi_perform_once();
|
curl_multi_perform_once();
|
||||||
}
|
}
|
||||||
|
free(transfer);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -173,8 +168,34 @@ static int curl_multi_perform_once()
|
|||||||
curl_msg->data.result,
|
curl_msg->data.result,
|
||||||
curl_easy_strerror(curl_msg->data.result),
|
curl_easy_strerror(curl_msg->data.result),
|
||||||
url);
|
url);
|
||||||
|
} else {
|
||||||
|
/* Transfer successful, query the file size */
|
||||||
|
if (transfer->type == FILESIZE) {
|
||||||
|
Link *this_link = transfer->link;
|
||||||
|
long http_resp;
|
||||||
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||||
|
if (http_resp == HTTP_OK) {
|
||||||
|
double cl = 0;
|
||||||
|
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
|
||||||
|
if (cl == -1) {
|
||||||
|
/* Turns out not to be a file after all */
|
||||||
|
this_link->content_length = 0;
|
||||||
|
this_link->type = LINK_DIR;
|
||||||
|
} else {
|
||||||
|
this_link->content_length = cl;
|
||||||
|
this_link->type = LINK_FILE;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this_link->type = LINK_INVALID;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
curl_multi_remove_handle(curl_multi, curl);
|
curl_multi_remove_handle(curl_multi, curl);
|
||||||
|
/* clean up the handle, if we are querying the file size */
|
||||||
|
if (transfer->type == FILESIZE) {
|
||||||
|
curl_easy_cleanup(curl);
|
||||||
|
free(transfer);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
fprintf(stderr, "curl_multi_perform_once(): curl_msg->msg: %d\n",
|
fprintf(stderr, "curl_multi_perform_once(): curl_msg->msg: %d\n",
|
||||||
curl_msg->msg);
|
curl_msg->msg);
|
||||||
@ -351,7 +372,7 @@ long Link_download(const char *path, char *output_buf, size_t size,
|
|||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
||||||
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
|
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
|
||||||
|
|
||||||
transfer_wrapper(curl, DATA);
|
blocking_transfer(curl);
|
||||||
|
|
||||||
long http_resp;
|
long http_resp;
|
||||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||||
@ -400,7 +421,7 @@ LinkTable *LinkTable_new(const char *url)
|
|||||||
// curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
// curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||||
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
|
||||||
|
|
||||||
transfer_wrapper(curl, DATA);
|
blocking_transfer(curl);
|
||||||
|
|
||||||
/* if downloading base URL failed */
|
/* if downloading base URL failed */
|
||||||
long http_resp;
|
long http_resp;
|
||||||
@ -446,42 +467,26 @@ static void LinkTable_add(LinkTable *linktbl, Link *link)
|
|||||||
linktbl->links[linktbl->num - 1] = link;
|
linktbl->links[linktbl->num - 1] = link;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t Link_get_size(Link *this_link)
|
void Link_get_size(Link *this_link)
|
||||||
{
|
{
|
||||||
#ifdef HTTPDIRFS_INFO
|
#ifdef HTTPDIRFS_INFO
|
||||||
fprintf(stderr, "Link_get_size(%s);\n", this_link->f_url);
|
fprintf(stderr, "Link_get_size(%s);\n", this_link->f_url);
|
||||||
#endif
|
#endif
|
||||||
double cl = 0;
|
|
||||||
|
|
||||||
if (this_link->type == LINK_FILE) {
|
if (this_link->type == LINK_FILE) {
|
||||||
CURL *curl = Link_to_curl(this_link);
|
CURL *curl = Link_to_curl(this_link);
|
||||||
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
|
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
|
||||||
|
|
||||||
TransferStruct transfer;
|
TransferStruct *transfer = malloc(sizeof(TransferStruct));
|
||||||
transfer.link = this_link;
|
if (!transfer) {
|
||||||
transfer.type = FILESIZE;
|
fprintf(stderr, "Link_get_size(): malloc failed!\n");
|
||||||
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
|
|
||||||
|
|
||||||
transfer_wrapper(curl, DATA);
|
|
||||||
long http_resp;
|
|
||||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
|
||||||
if (http_resp == HTTP_OK) {
|
|
||||||
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
|
|
||||||
if (cl == -1) {
|
|
||||||
/* Turns out not to be a file after all */
|
|
||||||
this_link->content_length = 0;
|
|
||||||
this_link->type = LINK_DIR;
|
|
||||||
} else {
|
|
||||||
this_link->content_length = cl;
|
|
||||||
this_link->type = LINK_FILE;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this_link->type = LINK_INVALID;
|
|
||||||
}
|
}
|
||||||
curl_easy_cleanup(curl);
|
transfer->link = this_link;
|
||||||
}
|
transfer->type = FILESIZE;
|
||||||
|
curl_easy_setopt(curl, CURLOPT_PRIVATE, transfer);
|
||||||
|
|
||||||
return cl;
|
nonblocking_transfer(curl);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void LinkTable_fill(LinkTable *linktbl)
|
void LinkTable_fill(LinkTable *linktbl)
|
||||||
@ -495,7 +500,7 @@ void LinkTable_fill(LinkTable *linktbl)
|
|||||||
strncpy(this_link->f_url, url, URL_LEN_MAX);
|
strncpy(this_link->f_url, url, URL_LEN_MAX);
|
||||||
free(url);
|
free(url);
|
||||||
if (this_link->type == LINK_FILE && !(this_link->content_length)) {
|
if (this_link->type == LINK_FILE && !(this_link->content_length)) {
|
||||||
this_link->content_length = Link_get_size(this_link);
|
Link_get_size(this_link);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -626,10 +631,6 @@ static Link *path_to_Link_recursive(char *path, LinkTable *linktbl)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#ifdef HTTPDIRFS_DEBUG
|
|
||||||
fprintf(stderr, "path_to_Link(): %s does not exist.\n", path);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user