rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added mutex for curl_multi_add_handle and curl_multi_perform_once, now FUSE can be run in multithreaded mode

This commit is contained in:
Fufu Fang 2018-07-24 15:22:18 +01:00
parent afd0f5fb62
commit 825c2541b7
2 changed files with 90 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -2,22 +2,17 @@
Have you ever wanted to mount those HTTP directory listings as if it was a partition? Look no further, this is your solution. HTTPDirFS stands for Hyper Text Transfer Protocol Directory Filesystem
The performance of the program is excellent, due to the use of curl-multi interface. HTTP connections are reused, and HTTP pipelining is used when available. I haven't benchmarked it, but I feel this is faster than ``rclone mount``.
The performance of the program is excellent, due to the use of curl-multi interface. HTTP connections are reused, and HTTP pipelining is used when available. I haven't benchmarked it, but I feel this is faster than ``rclone mount``. The FUSE component itself also runs in multithreaded mode.
## Usage
./httpdirfs -f -s $URL $YOUR_MOUNT_POINT
./httpdirfs -f $URL $YOUR_MOUNT_POINT
An example URL would be [Debian CD Image Server](https://cdimage.debian.org/debian-cd/). The ``-f`` flag keeps the program in the foreground, which is useful for monitoring which URL the filesystem is visiting.
You have to run the program in single-threaded mode by passing in the ``-s`` flag, otherwise libcurl will crash.
## The Technical Details
I noticed that most HTTP directory listings don't provide the file size for the web page itself. I suppose this makes perfect sense, as they are generated on the fly. Whereas the actual files have got file sizes. So the listing pages can be treated as folders, and the rest are files.
This program downloads the HTML web pages/files using [libcurl](https://curl.haxx.se/libcurl/), then parses the listing pages using [Gumbo](https://github.com/google/gumbo-parser), and presents them using [libfuse](https://github.com/libfuse/libfuse)
## Warning / Help needed
I would love to not having to pass the ``-s`` flag to FUSE. I have no idea how to do so. If you have any ideas, please open an issue request.
## LICENSE
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

150
network.c
View File

@ -34,21 +34,82 @@ LinkTable *ROOT_LINK_TBL;
/* ------------------------ Static variable ------------------------------ */
/** \brief curl shared interface - not actually being used. */
static CURLSH *curl_share;
/** \brief pthread mutex for thread safety */
static pthread_mutex_t pthread_curl_lock;
/** \brief curl multi interface handle */
static CURLM *curl_multi;
/** \brief pthread mutex for transfer functions/ */
static pthread_mutex_t transfer_lock;
/**
* \brief pthread mutex for curl itself
* \note I am sure if it is being used properly
*/
static pthread_mutex_t curl_lock;
/* -------------------------- Functions ---------------------------------- */
static void nonblocking_transfer(CURL *curl)
static void pthread_lock_cb(CURL *handle, curl_lock_data data,
curl_lock_access access, void *userptr)
{
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
if(res > 0) {
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
res, curl_multi_strerror(res));
(void)access; /* unused */
(void)userptr; /* unused */
(void)handle; /* unused */
(void)data; /* unused */
pthread_mutex_lock(&curl_lock);
}
static void pthread_unlock_cb(CURL *handle, curl_lock_data data,
void *userptr)
{
(void)userptr; /* unused */
(void)handle; /* unused */
(void)data; /* unused */
pthread_mutex_unlock(&curl_lock);
}
void network_init(const char *url)
{
/* Global related */
if (curl_global_init(CURL_GLOBAL_ALL)) {
fprintf(stderr, "network_init(): curl_global_init() failed!\n");
exit(EXIT_FAILURE);
}
/* Share related */
curl_share = curl_share_init();
if (!(curl_share)) {
fprintf(stderr, "network_init(): curl_share_init() failed!\n");
exit(EXIT_FAILURE);
}
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
if (pthread_mutex_init(&curl_lock, NULL) != 0)
{
printf(
"network_init(): curl_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
curl_share_setopt(curl_share, CURLSHOPT_LOCKFUNC, pthread_lock_cb);
curl_share_setopt(curl_share, CURLSHOPT_UNLOCKFUNC, pthread_unlock_cb);
/* Multi related */
curl_multi = curl_multi_init();
if (!curl_multi) {
fprintf(stderr, "network_init(): curl_multi_init() failed!\n");
exit(EXIT_FAILURE);
}
curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
CURL_MULTI_MAX_CONNECTION);
/* Initialise transfer lock */
if (pthread_mutex_init(&transfer_lock, NULL) != 0)
{
printf(
"network_init(): transfer_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
/* create the root link table */
ROOT_LINK_TBL = LinkTable_new(url);
}
static void link_set_stat(Link* this_link, CURL *curl)
@ -79,6 +140,7 @@ static void link_set_stat(Link* this_link, CURL *curl)
*/
static int curl_multi_perform_once()
{
pthread_mutex_lock(&transfer_lock);
/* Get curl multi interface to perform pending tasks */
int n_running_curl;
curl_multi_perform(curl_multi, &n_running_curl);
@ -161,9 +223,23 @@ static int curl_multi_perform_once()
curl_msg->msg);
}
}
pthread_mutex_unlock(&transfer_lock);
return n_running_curl;
}
static void nonblocking_transfer(CURL *curl)
{
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
pthread_mutex_unlock(&transfer_lock);
if(res > 0) {
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
res, curl_multi_strerror(res));
exit(EXIT_FAILURE);
}
}
/* This uses the curl multi interface */
static void blocking_transfer(CURL *curl)
{
@ -171,7 +247,11 @@ static void blocking_transfer(CURL *curl)
transfer.type = DATA;
transfer.transferring = 1;
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
pthread_mutex_unlock(&transfer_lock);
if(res > 0) {
fprintf(stderr, "blocking_multi_transfer(): %d, %s\n",
res, curl_multi_strerror(res));
@ -183,60 +263,6 @@ static void blocking_transfer(CURL *curl)
}
}
static void pthread_lock_cb(CURL *handle, curl_lock_data data,
curl_lock_access access, void *userptr)
{
(void)access; /* unused */
(void)userptr; /* unused */
(void)handle; /* unused */
(void)data; /* unused */
pthread_mutex_lock(&pthread_curl_lock);
}
static void pthread_unlock_cb(CURL *handle, curl_lock_data data,
void *userptr)
{
(void)userptr; /* unused */
(void)handle; /* unused */
(void)data; /* unused */
pthread_mutex_unlock(&pthread_curl_lock);
}
void network_init(const char *url)
{
/* Global related */
if (curl_global_init(CURL_GLOBAL_ALL)) {
fprintf(stderr, "network_init(): curl_global_init() failed!\n");
exit(EXIT_FAILURE);
}
/* Share related */
curl_share = curl_share_init();
if (!(curl_share)) {
fprintf(stderr, "network_init(): curl_share_init() failed!\n");
exit(EXIT_FAILURE);
}
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
curl_share_setopt(curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
pthread_mutex_init(&pthread_curl_lock, NULL);
curl_share_setopt(curl_share, CURLSHOPT_LOCKFUNC, pthread_lock_cb);
curl_share_setopt(curl_share, CURLSHOPT_UNLOCKFUNC, pthread_unlock_cb);
/* Multi related */
curl_multi = curl_multi_init();
if (!curl_multi) {
fprintf(stderr, "network_init(): curl_multi_init() failed!\n");
exit(EXIT_FAILURE);
}
curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
CURL_MULTI_MAX_CONNECTION);
/* create the root link table */
ROOT_LINK_TBL = LinkTable_new(url);
}
static char *url_append(const char *url, const char *sublink)
{
int needs_separator = 0;