rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Directory listing performance improvement while file transfers are going on 

- Added a LinkTable generation priority lock
- This allows LinkTable generation to be run exclusively. This
effectively gives LinkTable generation priority over file transfer.
This commit is contained in:
Fufu Fang 2019-08-31 21:21:28 +01:00
parent 1948bbd977
commit afb2a8fe6c
No known key found for this signature in database
GPG Key ID: 0F6BB5EF6F8BB729
6 changed files with 94 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -113,9 +113,10 @@ OpenSSL 1.1 do not.
### Debugging Mutexes
By default the debugging output associated with mutexes are not compiled. To
enable them, compile the program using the following command:
enable them, compile the program with the ``-DCACHE_LOCK_DEBUG``, the
``-DNETWORK_LOCK_DEBUG`` and/or the ``-DLINK_LOCK_DEBUG`` CPPFLAGS, e.g.
make CPPFLAGS=-DLOCK_DEBUG
make CPPFLAGS=-DCACHE_LOCK_DEBUG
## The Technical Details
This program downloads the HTML web pages/files using

20
src/cache.c
View File

@ -755,7 +755,7 @@ cf->content_length: %ld, Data_size(fn): %ld.\n", fn, cf->content_length,
void Cache_close(Cache *cf)
{
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Must wait for the background download thread to stop */
fprintf(stderr, "Cache_close(): locking bgt_lock;\n");
pthread_mutex_lock(&cf->bgt_lock);
@ -815,7 +815,7 @@ static void Seg_set(Cache *cf, off_t offset, int i)
static void *Cache_bgdl(void *arg)
{
Cache *cf = (Cache *) arg;
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: locking rw_lock;\n",
pthread_self());
#endif
@ -834,12 +834,12 @@ static void *Cache_bgdl(void *arg)
"Cache_bgdl(): received %ld, possible network error.\n", recv);
}
free(recv_buf);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: unlocking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_bgdl(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
@ -874,18 +874,18 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
goto bgdl;
} else {
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Wait for the background download thread to finish */
fprintf(stderr, "Cache_read(): thread %lu: locking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking bgt_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&cf->bgt_lock);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
/* Wait for any other download thread to finish*/
fprintf(stderr, "Cache_read(): thread %lu: locking rw_lock;\n",
pthread_self());
@ -895,7 +895,7 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
/* The segment already exists - it was downloaded by other
* download thread. Send it off and unlock the I/O */
send = Data_read(cf, (uint8_t *) output_buf, len, offset);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
@ -931,7 +931,7 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
"Cache_read(): received %ld, possible network error.\n", recv);
}
free(recv_buf);
#ifdef LOCK_DEBUG
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: unlocking rw_lock;\n",
pthread_self());
#endif
@ -945,8 +945,10 @@ long Cache_read(Cache *cf, char *output_buf, off_t len, off_t offset)
cf->next_offset < cf->content_length ){
/* Stop the spawning of multiple background pthreads */
if(!pthread_mutex_trylock(&cf->bgt_lock)) {
#ifdef CACHE_LOCK_DEBUG
fprintf(stderr, "Cache_read(): thread %lu: trylocked bgt_lock;\n",
pthread_self());
#endif
if (pthread_create(&cf->bgt, NULL, Cache_bgdl, cf)) {
fprintf(stderr,
"Cache_read(): Error creating background download thread\n"

41
src/link.c
View File

@ -15,6 +15,23 @@
LinkTable *ROOT_LINK_TBL = NULL;
int ROOT_LINK_OFFSET = 0;
/* ----------------- Static variable ----------------------- */
/**
* \brief LinkTable generation priority lock
* \details This allows LinkTable generation to be run exclusively. This
* effectively gives LinkTable generation priority over file transfer.
*/
static pthread_mutex_t link_lock;
void link_system_init()
{
if (pthread_mutex_init(&link_lock, NULL) != 0) {
fprintf(stderr,
"link_system_init(): link_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
}
static void LinkTable_add(LinkTable *linktbl, Link *link)
{
linktbl->num++;
@ -266,6 +283,12 @@ static void LinkTable_print(LinkTable *linktbl)
LinkTable *LinkTable_new(const char *url)
{
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"LinkTable_new(): thread %lu: locking link_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&link_lock);
LinkTable *linktbl = calloc(1, sizeof(LinkTable));
if (!linktbl) {
fprintf(stderr, "LinkTable_new(): calloc failure!\n");
@ -349,6 +372,12 @@ HTTP %ld\n", url, http_resp);
curl_easy_cleanup(c);
LinkTable_print(linktbl);
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"LinkTable_new(): thread %lu: unlocking link_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&link_lock);
return linktbl;
}
@ -553,6 +582,18 @@ long path_download(const char *path, char *output_buf, size_t size,
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"path_download(): thread %lu: locking link_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&link_lock);
#ifdef LINK_LOCK_DEBUG
fprintf(stderr,
"path_download(): thread %lu: unlocking link_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&link_lock);
transfer_blocking(curl);
long http_resp;

5
src/link.h
View File

@ -50,6 +50,11 @@ extern LinkTable *ROOT_LINK_TBL;
*/
extern int ROOT_LINK_OFFSET;
/**
* \brief initialise link sub-system.
*/
void link_system_init();
/**
* \brief
*/

3
src/main.c
View File

@ -36,6 +36,9 @@ int main(int argc, char **argv)
/*--- FUSE expects the first initialisation to be the program's name ---*/
add_arg(&fuse_argv, &fuse_argc, argv[0]);
/* initialise link subsystem */
link_system_init();
/* initialise network configuration struct */
network_config_init();

34
src/network.c
View File

@ -160,6 +160,11 @@ static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl,
*/
int curl_multi_perform_once()
{
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"curl_multi_perform_once(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
/* Get curl multi interface to perform pending tasks */
int n_running_curl;
@ -217,6 +222,11 @@ int curl_multi_perform_once()
while((curl_msg = curl_multi_info_read(curl_multi, &n_mesgs))) {
curl_process_msgs(curl_msg, n_running_curl, n_mesgs);
}
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"curl_multi_perform_once(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);
return n_running_curl;
}
@ -254,8 +264,7 @@ LinkTable *network_init(const char *url)
curl_share_setopt(CURL_SHARE, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
if (pthread_mutex_init(&curl_lock, NULL) != 0) {
printf(
"network_init(): curl_lock initialisation failed!\n");
fprintf(stderr, "network_init(): curl_lock initialisation failed!\n");
exit(EXIT_FAILURE);
}
curl_share_setopt(CURL_SHARE, CURLSHOPT_LOCKFUNC, curl_callback_lock);
@ -327,9 +336,18 @@ void transfer_blocking(CURL *curl)
transfer.type = DATA;
transfer.transferring = 1;
curl_easy_setopt(curl, CURLOPT_PRIVATE, &transfer);
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_blocking(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_blocking(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);
if(res > 0) {
@ -345,8 +363,18 @@ void transfer_blocking(CURL *curl)
void transfer_nonblocking(CURL *curl)
{
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_nonblocking(): thread %lu: locking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_lock(&transfer_lock);
CURLMcode res = curl_multi_add_handle(curl_multi, curl);
#ifdef NETWORK_LOCK_DEBUG
fprintf(stderr,
"transfer_nonblocking(): thread %lu: unlocking transfer_lock;\n",
pthread_self());
#endif
pthread_mutex_unlock(&transfer_lock);
if(res > 0) {