rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactoring transfer_blocking

This commit is contained in:
Fufu Fang 2021-09-03 12:40:35 +01:00
parent 177b738522
commit c64a139b46
5 changed files with 53 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -1,7 +1,7 @@
VERSION = 1.2.3
# -fsanitize=undefined -fanalyzer -Wno-analyzer-file-leak \
CFLAGS += -O2 -g -Wall -Wextra -Wshadow \
-fsanitize=undefined -fanalyzer -Wno-analyzer-file-leak\
CFLAGS += -g -O2 -Wall -Wextra -Wshadow \
-rdynamic -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DVERSION=\"$(VERSION)\"\
`pkg-config --cflags-only-I gumbo libcurl fuse uuid expat`
LDFLAGS += `pkg-config --libs-only-L gumbo libcurl fuse uuid expat`

3
src/cache.h
View File

@ -71,6 +71,9 @@ struct Cache {
/** \brief the FUSE filesystem path to the remote file*/
char *fs_path;
/** Transfer struct for the streaming cache */
TransferStruct *ts;
};
/**

47
src/link.c
View File

@ -456,8 +456,8 @@ LinkTable *LinkTable_new(const char *url)
/*
* start downloading the base URL
*/
TransferStruct buf = Link_download_full(linktbl->links[0]);
if (buf.size == 0) {
TransferStruct ts = Link_download_full(linktbl->links[0]);
if (ts.size == 0) {
LinkTable_free(linktbl);
return NULL;
}
@ -465,10 +465,10 @@ LinkTable *LinkTable_new(const char *url)
/*
* Otherwise parsed the received data
*/
GumboOutput *output = gumbo_parse(buf.data);
GumboOutput *output = gumbo_parse(ts.data);
HTML_to_LinkTable(output->root, linktbl);
gumbo_destroy_output(&kGumboDefaultOptions, output);
FREE(buf.data);
FREE(ts.data);
int skip_fill = 0;
char *unescaped_path;
@ -759,11 +759,14 @@ TransferStruct Link_download_full(Link * link)
char *url = link->f_url;
CURL *curl = Link_to_curl(link);
TransferStruct buf;
buf.size = 0;
buf.data = NULL;
TransferStruct ts;
ts.size = 0;
ts.data = NULL;
ts.type = DATA;
ts.transferring = 1;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &buf);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &ts);
curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *) &ts);
/*
* If we get temporary HTTP failure, wait for 5 seconds before retry
@ -780,16 +783,16 @@ TransferStruct Link_download_full(Link * link)
} else if (http_resp != HTTP_OK) {
lprintf(warning,
"cannot retrieve URL: %s, HTTP %ld\n", url, http_resp);
buf.size = 0;
ts.size = 0;
curl_easy_cleanup(curl);
return buf;
return ts;
}
}
while (HTTP_temp_failure(http_resp));
curl_easy_getinfo(curl, CURLINFO_FILETIME, &(link->time));
curl_easy_cleanup(curl);
return buf;
return ts;
}
long
@ -806,19 +809,21 @@ Link_download(Link *link, char *output_buf, size_t req_size,
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
lprintf(debug, "%s: %s\n", link->linkname, range_str);
TransferStruct buf;
buf.size = 0;
buf.data = NULL;
CURL *curl = Link_to_curl(link);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &buf);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
TransferStruct ts;
ts.size = 0;
ts.data = NULL;
ts.type = DATA;
ts.transferring = 1;
TransferStruct header;
header.size = 0;
header.data = NULL;
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &header);
CURL *curl = Link_to_curl(link);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &header);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &ts);
curl_easy_setopt(curl, CURLOPT_PRIVATE, (void *) &ts);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
transfer_blocking(curl);
/*
@ -855,9 +860,9 @@ range requests\n");
lprintf(error, "req_size: %lu, recv: %ld\n", req_size, recv);
}
memmove(output_buf, buf.data, recv);
memmove(output_buf, ts.data, recv);
curl_easy_cleanup(curl);
FREE(buf.data);
FREE(ts.data);
return recv;
}

11
src/link.h
View File

@ -8,6 +8,7 @@
typedef struct Link Link;
typedef struct LinkTable LinkTable;
typedef struct TransferStruct TransferStruct;
#include "sonic.h"
#include "config.h"
@ -26,7 +27,7 @@ typedef enum {
} LinkType;
/**
* \brief specify the type of data transfer
* \brief specify the type of data transfer
*/
typedef enum {
FILESTAT = 's',
@ -34,9 +35,9 @@ typedef enum {
} TransferType;
/**
* \brief For storing transfer data and metadata
* \brief For storing transfer data and metadata
*/
typedef struct {
struct TransferStruct {
/** \brief The array to store the data */
char *data;
/** \brief The size of the array */
@ -46,10 +47,10 @@ typedef struct {
/** \brief The type of transfer being done */
TransferType type;
/** \brief Whether transfer is in progress */
int transferring;
volatile int transferring;
/** \brief The link associated with the transfer */
Link *link;
} TransferStruct;
};
/**
* \brief link table type

35
src/network.c
View File

@ -118,11 +118,11 @@ curl_process_msgs(CURLMsg * curl_msg, int n_running_curl, int n_mesgs)
(void) n_mesgs;
static volatile int slept = 0;
if (curl_msg->msg == CURLMSG_DONE) {
TransferStruct *transfer;
TransferStruct *ts;
CURL *curl = curl_msg->easy_handle;
curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE,
&transfer);
transfer->transferring = 0;
&ts);
ts->transferring = 0;
char *url = NULL;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
@ -147,8 +147,8 @@ curl_process_msgs(CURLMsg * curl_msg, int n_running_curl, int n_mesgs)
/*
* Transfer successful, set the file size
*/
if (transfer->type == FILESTAT) {
Link_set_file_stat(transfer->link, curl);
if (ts->type == FILESTAT) {
Link_set_file_stat(ts->link, curl);
}
} else {
lprintf(error, "%d - %s <%s>\n",
@ -159,9 +159,9 @@ curl_process_msgs(CURLMsg * curl_msg, int n_running_curl, int n_mesgs)
/*
* clean up the handle, if we are querying the file size
*/
if (transfer->type == FILESTAT) {
if (ts->type == FILESTAT) {
curl_easy_cleanup(curl);
FREE(transfer);
FREE(ts);
}
} else {
lprintf(warning, "curl_msg->msg: %d\n", curl_msg->msg);
@ -306,16 +306,10 @@ void NetworkSystem_init(void)
crypto_lock_init();
}
void transfer_blocking(CURL * curl)
void transfer_blocking(CURL *curl)
{
/*
* We don't need to malloc here, as the transfer is finished before
* the variable gets popped from the stack
*/
volatile TransferStruct transfer;
transfer.type = DATA;
transfer.transferring = 1;
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
TransferStruct *ts;
curl_easy_getinfo(curl, CURLINFO_PRIVATE, &ts);
lprintf(network_lock_debug,
"thread %x: locking transfer_lock;\n", pthread_self());
@ -331,12 +325,16 @@ void transfer_blocking(CURL * curl)
lprintf(error, "%d, %s\n", res, curl_multi_strerror(res));
}
while (transfer.transferring) {
while (ts->transferring) {
curl_multi_perform_once();
}
}
void transfer_nonblocking(CURL * curl)
// void transfer_semiblocking(CURL *curl) {
// }
void transfer_nonblocking(CURL *curl)
{
lprintf(network_lock_debug,
"thread %x: locking transfer_lock;\n", pthread_self());
@ -371,7 +369,6 @@ write_memory_callback(void *contents, size_t size, size_t nmemb,
memmove(&mem->data[mem->size], contents, realsize);
mem->size += realsize;
mem->data[mem->size] = 0;
// lprintf(debug, "realsize %d bytes\n", realsize);
return realsize;
}