rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

documented the new feature, and the bug

This commit is contained in:
Fufu Fang 2019-04-22 23:29:59 +01:00
parent e2d2b0dd28
commit 0d8d5ef329
3 changed files with 74 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
README.md
View File

@ -4,6 +4,11 @@ Have you ever wanted to mount those HTTP directory listings as if it was a parti
The performance of the program is excellent, due to the use of curl-multi interface. HTTP connections are reused, and HTTP pipelining is used when available. I haven't benchmarked it, but I feel this is faster than ``rclone mount``. The FUSE component itself also runs in multithreaded mode.
Furthermore, a permanent cache system has been implemented to cache all the files you have downloaded. This is triggered by the ``--cache`` flag
## BUG
The permanent cache system seems to have problem when you have randomly seek across the file during the initial download process. I am not sure what causes the problem. It is probably some sort of concurrency issue. The mutexes I set up doesn't seem to help with the problem.
## Compilation
This program was developed under Debian Stretch. If you are using the same operating system as me, you need ``libgumbo-dev``, ``libfuse-dev``, ``libssl1.0-dev`` and ``libcurl4-openssl-dev``.
@ -29,10 +34,21 @@ An example URL would be [Debian CD Image Server](https://cdimage.debian.org/debi
Other useful options:
-u --user HTTP authentication username
-p --password HTTP authentication password
-P --proxy Proxy for libcurl, for more details refer to
https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html
-u --username HTTP authentication username
-p --password HTTP authentication password
-P --proxy Proxy for libcurl, for more details refer to
https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html
--proxy-username Username for the proxy
--proxy-password Password for the proxy
--cache Set the cache folder
## Permanent cache system
You can now cache all the files you have looked at permanently on your hard
drive by using the ``--cache`` flag. The file it caches persist across sessions For example:
mkdir cache mnt
httpdirfs --cache cache http://cdimage.debian.org/debian-cd/ mnt
Once a segment of the file has been downloaded once, it won't be downloaded again. So the first time you use the file it is slow, the subsequent access is fast. You can also retrieve your partially or fully downloaded file from ``cache/metadata``.
## Configuration file support
There is now rudimentary config file support. The configuration file that the program will read is ``${XDG_CONFIG_HOME}/httpdirfs/config``. If ``${XDG_CONFIG_HOME}`` is not set, it will default to ``${HOME}/.config``. So by default you need to put the configuration file at ``${HOME}/.config/httpdirfs/config``. You will have to create the sub-directory and the configuration file yourself. In the configuration file, please supply one option per line. For example:
@ -43,7 +59,6 @@ There is now rudimentary config file support. The configuration file that the pr
-f
## SSL Support
If you run the program in the foreground, when it starts up, it will output the SSL engine version string. Please verify that your libcurl is linked against OpenSSL, as the pthread mutex functions are designed for OpenSSL.
The SSL engine version string looks something like this:

87
src/cache.c
View File

@ -218,7 +218,7 @@ static int Data_create(Cache *cf)
mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
char *datafn = strndupcat(DATA_DIR, cf->path, MAX_PATH_LEN);
fprintf(stderr, "Data_create(): Creating %s\n", datafn);
// fprintf(stderr, "Data_create(): Creating %s\n", datafn);
fd = open(datafn, O_WRONLY | O_CREAT, mode);
free(datafn);
if (fd == -1) {
@ -244,7 +244,7 @@ static long Data_size(const char *fn)
int s = stat(datafn, &st);
free(datafn);
if (!s) {
fprintf(stderr, "Data_size(): %s is %ld bytes.\n", fn, st.st_size);
// fprintf(stderr, "Data_size(): %s is %ld bytes.\n", fn, st.st_size);
return st.st_size;
}
fprintf(stderr, "Data_size(): stat(): %s\n", strerror(errno));
@ -439,34 +439,7 @@ static int Cache_exist(const char *fn)
free(metafn);
free(datafn);
return !(meta_exists & data_exists);
}
int Cache_create(const char *fn, long len, long time)
{
if (!Cache_exist(fn)) {
return 0;
}
fprintf(stderr, "Cache_create(): Creating cache files for %s.\n", fn);
Cache *cf = Cache_alloc();
cf->path = strndup(fn, MAX_PATH_LEN);
cf->time = time;
cf->content_length = len;
if (Data_create(cf)) {
fprintf(stderr, "Cache_create(): Data_create() failed!\n");
}
if (Meta_create(cf)) {
fprintf(stderr, "Cache_create(): Meta_create() failed!\n");
}
Cache_free(cf);
return Cache_exist(fn);
return meta_exists & data_exists;
}
/**
@ -512,10 +485,47 @@ static int Data_open(Cache *cf)
return 0;
}
int Cache_create(const char *fn, long len, long time)
{
if (Cache_exist(fn)) {
/* We make sure that the cache files are not outdated */
Cache *cf = Cache_open(fn);
if (cf->time == time) {
Cache_close(cf);
return 0;
}
Cache_delete(fn);
}
fprintf(stderr, "Cache_create(): Creating cache files for %s.\n", fn);
Cache *cf = Cache_alloc();
cf->path = strndup(fn, MAX_PATH_LEN);
cf->time = time;
cf->content_length = len;
if (Data_create(cf)) {
fprintf(stderr, "Cache_create(): Data_create() failed!\n");
}
if (Meta_create(cf)) {
fprintf(stderr, "Cache_create(): Meta_create() failed!\n");
}
Cache_free(cf);
/*
* Cache_exist() returns 1, if cache files exist and valid. Whereas this
* function returns 0 on success.
*/
return !Cache_exist(fn);
}
Cache *Cache_open(const char *fn)
{
/* Check if both metadata and data file exist */
if (Cache_exist(fn)) {
if (!Cache_exist(fn)) {
return NULL;
}
@ -523,15 +533,22 @@ Cache *Cache_open(const char *fn)
Cache *cf = Cache_alloc();
cf->path = strndup(fn, MAX_PATH_LEN);
/* Internal inconsistency in metadata file */
/*
* Internal inconsistency in metadata file, note that Meta_read() returns
* -2 on internal metadata inconsistency
*/
if (Meta_read(cf) == -2) {
Cache_free(cf);
Cache_delete(fn);
return NULL;
}
/* Inconsistency between metadata and data file */
if (cf->content_length != Data_size(fn)) {
/*
* Inconsistency between metadata and data file, note that on disk file
* size might be bigger than content_length, due to on-disk filesystem
* allocation policy.
*/
if (cf->content_length > Data_size(fn)) {
fprintf(stderr,
"Cache_open(): Metadata inconsistency: \
cf->content_length: %ld, Data_size(fn): %ld\n",
@ -577,7 +594,6 @@ static int Seg_exist(Cache *cf, off_t offset)
int bit = total_bit % 8;
return cf->seg[byte] & (1 << bit);
}
/**
* \brief Set the existence of a segment
@ -619,7 +635,6 @@ long Cache_read(Cache *cf, char *buf, size_t size, off_t offset)
Seg_set(cf, offset, 1);
}
/* This is how you download from the web server */
pthread_mutex_unlock(&(cf->rw_lock));
return received;
}

6
src/main.c
View File

@ -242,9 +242,9 @@ static void print_http_options()
-u --username HTTP authentication username\n\
-p --password HTTP authentication password\n\
-P --proxy Proxy for libcurl, for more details refer to\n\
https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html\n\
--proxy-username Username for the proxy\n\
--proxy-password Password for the proxy\n\
https://curl.haxx.se/libcurl/c/CURLOPT_PROXY.html\n\
--proxy-username Username for the proxy\n\
--proxy-password Password for the proxy\n\
--cache Set the cache folder\n\
\n\
libfuse options:\n");