rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

first ever published version

This commit is contained in:
Fufu Fang 2018-07-23 01:27:03 +01:00
parent 25648ad947
commit 71339f13b8
6 changed files with 104 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
tmp tmp
*.o *.o
mount-http-dir
*_test.txt *_test.txt
*.kate-swp *.kate-swp
httpdirfs

4
Makefile
View File

@ -5,10 +5,10 @@ OBJ = main.o network.o
%.o: %.c %.o: %.c
$(CC) -c -o $@ $< $(CFLAGS) $(CC) -c -o $@ $< $(CFLAGS)
mount-http-dir: $(OBJ) httpdirfs: $(OBJ)
$(CC) -o $@ $^ $(CFLAGS) $(CC) -o $@ $^ $(CFLAGS)
.PHONY: clean .PHONY: clean
clean: clean:
rm -f *.o mount-http-dir rm -f *.o httpdirfs

8
data.h
View File

@ -8,13 +8,11 @@
#include <stdlib.h> #include <stdlib.h>
/** \brief use this data type for buffer */ /** \brief use this data type for buffer */
typedef struct MemoryStruct MemoryStruct; typedef struct BufferStruct BufferStruct;
struct MemoryStruct { struct BufferStruct {
size_t size; size_t size;
char *memory; char *data;
}; };
void MemoryStruct_free(MemoryStruct *ms);
#endif #endif

40
main.c
View File

@ -9,20 +9,32 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
static char *BASE_URL;
static int fs_getattr(const char *path, struct stat *stbuf); static int fs_getattr(const char *path, struct stat *stbuf);
static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
off_t offset, struct fuse_file_info *fi); off_t offset, struct fuse_file_info *fi);
static int fs_open(const char *path, struct fuse_file_info *fi); static int fs_open(const char *path, struct fuse_file_info *fi);
static int fs_read(const char *path, char *buf, size_t size, off_t offset, static int fs_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi); struct fuse_file_info *fi);
static void *fs_init(struct fuse_conn_info *conn);
static struct fuse_operations fs_oper = { static struct fuse_operations fs_oper = {
.getattr = fs_getattr, .getattr = fs_getattr,
.readdir = fs_readdir, .readdir = fs_readdir,
.open = fs_open, .open = fs_open,
.read = fs_read, .read = fs_read,
.init = fs_init
}; };
static void *fs_init(struct fuse_conn_info *conn)
{
(void) conn;
network_init(BASE_URL);
return NULL;
}
static void fs_usage() static void fs_usage()
{ {
fprintf(stderr, fprintf(stderr,
@ -39,7 +51,7 @@ int main(int argc, char **argv) {
fs_usage(); fs_usage();
} }
network_init(argv[argc-2]); BASE_URL = argv[argc-2];
argv[argc-2] = argv[argc-1]; argv[argc-2] = argv[argc-1];
argv[argc-1] = NULL; argv[argc-1] = NULL;
argc--; argc--;
@ -139,29 +151,13 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi) struct fuse_file_info *fi)
{ {
(void) fi; (void) fi;
Link *link;
link = path_to_Link(path);
if (!link) { fprintf(stderr, "fs_read(): path: %s, offset: %ld, size: %lu\n",
return -ENOENT; path, offset, size);
}
MemoryStruct ms;
fprintf(stderr, "fs_read(): Link_download(%s, %ld, %lu)\n",
link->f_url,
offset,
size
);
fflush(stderr); fflush(stderr);
size_t bytes_downloaded = Link_download(link, &ms, offset, size); long received = Link_download(path, buf, size, offset);
fprintf(stderr, fprintf(stderr, "fs_read(): received %ld bytes.\n", received);
"fs_read(): returning from Link_download(), with %lu bytes.\n",
bytes_downloaded);
fflush(stderr); fflush(stderr);
memcpy(buf, ms.memory, size); return received;
// free(ms.memory);
return bytes_downloaded;
} }

127
network.c
View File

@ -21,7 +21,7 @@ static void HTML_to_LinkTable(GumboNode *node, LinkTable *linktbl);
static int is_valid_link_p_url(const char *n); static int is_valid_link_p_url(const char *n);
static void Link_free(Link *link); static void Link_free(Link *link);
static Link *Link_new(const char *p_url); static Link *Link_new(const char *p_url);
static void Link_curl_init(Link *link); static CURL *Link_to_curl(Link *link);
static void LinkTable_free(LinkTable *linktbl); static void LinkTable_free(LinkTable *linktbl);
static void LinkTable_add(LinkTable *linktbl, Link *link); static void LinkTable_add(LinkTable *linktbl, Link *link);
static void LinkTable_fill(LinkTable *linktbl); static void LinkTable_fill(LinkTable *linktbl);
@ -65,22 +65,26 @@ void network_init(const char *url)
} }
static void Link_curl_init(Link *link) static CURL *Link_to_curl(Link *link)
{ {
link->curl = curl_easy_init(); CURL *curl = curl_easy_init();
/* set up some basic curl stuff */ /* set up some basic curl stuff */
curl_easy_setopt(link->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(link->curl, CURLOPT_USERAGENT, "mount-http-dir/libcurl"); curl_easy_setopt(curl, CURLOPT_USERAGENT, "mount-http-dir/libcurl");
curl_easy_setopt(link->curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(curl, CURLOPT_VERBOSE, 0);
curl_easy_setopt(link->curl, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
/* /*
* only 1 redirection is really needed * only 1 redirection is really needed
* - for following directories without the '/' * - for following directories without the '/'
*/ */
curl_easy_setopt(link->curl, CURLOPT_MAXREDIRS, 3); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3);
curl_easy_setopt(link->curl, CURLOPT_URL, link->f_url); curl_easy_setopt(curl, CURLOPT_URL, link->f_url);
curl_easy_setopt(link->curl, CURLOPT_SHARE, curl_share); fprintf(stderr, "Link_to_curl: %s\n", link->f_url);
fflush(stderr);
curl_easy_setopt(curl, CURLOPT_SHARE, curl_share);
return curl;
} }
@ -196,18 +200,29 @@ static void do_transfer_(CURL *curl)
static size_t static size_t
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
{ {
MemoryStruct *mem = (MemoryStruct *)userp; BufferStruct *buf = (BufferStruct *)userp;
size_t realsize = size * nmemb; size_t recv_size = size * nmemb;
mem->memory = malloc(realsize + 1); if ( !(buf->size) ) {
if(!mem->memory) { buf->data = malloc(recv_size + 1);
/* out of memory! */ buf->data[recv_size] = '\0';
printf("not enough memory (realloc returned NULL)\n"); if(!buf->data) {
return 0; fprintf(stderr,
"WriteMemoryCallback(): cannot malloc.\n");
fflush(stderr);
return 0;
}
memmove(buf->data, contents, recv_size);
return recv_size;
} else {
if(recv_size < buf->size) {
memmove(buf->data, contents, recv_size);
return recv_size;
} else {
memmove(buf->data, contents, buf->size);
return buf->size;
}
} }
memcpy(mem->memory, contents, realsize);
return realsize - 1;
} }
static Link *Link_new(const char *p_url) static Link *Link_new(const char *p_url)
@ -229,39 +244,53 @@ static Link *Link_new(const char *p_url)
static void Link_free(Link *link) static void Link_free(Link *link)
{ {
curl_easy_cleanup(link->curl);
free(link); free(link);
link = NULL; link = NULL;
} }
size_t Link_download(Link *link, MemoryStruct *ms, off_t offset, long Link_download(const char *path, char *output_buf, size_t size,
size_t size) off_t offset)
{ {
Link *link;
link = path_to_Link(path);
if (!link) {
return -ENOENT;
}
size_t start = offset; size_t start = offset;
size_t end = start + size; size_t end = start + size;
Link_curl_init(link); CURL *curl = Link_to_curl(link);
char range_str[64]; char range_str[64];
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end); snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
curl_easy_setopt(link->curl, CURLOPT_RANGE, range_str); BufferStruct buf;
curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)ms); buf.size = size;
buf.data = output_buf;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
do_transfer(link->curl);
do_transfer(curl);
long http_resp; long http_resp;
curl_easy_getinfo(link->curl, CURLINFO_RESPONSE_CODE, &http_resp); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
if ( (http_resp != HTTP_OK) && ( http_resp != HTTP_PARTIAL_CONTENT) ) { if ( (http_resp != HTTP_OK) && ( http_resp != HTTP_PARTIAL_CONTENT) ) {
fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld\n", fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld\n",
link->f_url, http_resp); link->f_url, http_resp);
fflush(stdout); fflush(stdout);
return 0; return -ENOENT;
} }
double dl; double dl;
curl_easy_getinfo(link->curl, CURLINFO_SIZE_DOWNLOAD, &dl); curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dl);
size_t s = dl;
curl_easy_cleanup(link->curl); size_t recv = dl;
return s; if (recv > size) {
recv = size;
}
curl_easy_cleanup(curl);
return recv;
} }
LinkTable *LinkTable_new(const char *url) LinkTable *LinkTable_new(const char *url)
@ -275,15 +304,17 @@ LinkTable *LinkTable_new(const char *url)
strncpy(head_link->f_url, url, URL_LEN_MAX); strncpy(head_link->f_url, url, URL_LEN_MAX);
/* start downloading the base URL */ /* start downloading the base URL */
Link_curl_init(head_link); CURL *curl = Link_to_curl(head_link);
MemoryStruct ms; BufferStruct buf;
curl_easy_setopt(head_link->curl, CURLOPT_WRITEDATA, (void *)&ms); buf.size = 0;
buf.data = NULL;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&buf);
do_transfer(head_link->curl); do_transfer(curl);
/* if downloading base URL failed */ /* if downloading base URL failed */
long http_resp; long http_resp;
curl_easy_getinfo(head_link->curl, CURLINFO_RESPONSE_CODE, &http_resp); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
if (http_resp != HTTP_OK) { if (http_resp != HTTP_OK) {
fprintf(stderr, "link.c: LinkTable_new() cannot retrive the base URL, \ fprintf(stderr, "link.c: LinkTable_new() cannot retrive the base URL, \
URL: %s, HTTP %ld\n", url, http_resp); URL: %s, HTTP %ld\n", url, http_resp);
@ -291,11 +322,11 @@ URL: %s, HTTP %ld\n", url, http_resp);
linktbl = NULL; linktbl = NULL;
return linktbl; return linktbl;
}; };
curl_easy_cleanup(head_link->curl); curl_easy_cleanup(curl);
/* Otherwise parsed the received data */ /* Otherwise parsed the received data */
GumboOutput* output = gumbo_parse(ms.memory); GumboOutput* output = gumbo_parse(buf.data);
free(ms.memory); free(buf.data);
HTML_to_LinkTable(output->root, linktbl); HTML_to_LinkTable(output->root, linktbl);
gumbo_destroy_output(&kGumboDefaultOptions, output); gumbo_destroy_output(&kGumboDefaultOptions, output);
@ -335,18 +366,16 @@ void LinkTable_fill(LinkTable *linktbl)
strncpy(this_link->f_url, url, URL_LEN_MAX); strncpy(this_link->f_url, url, URL_LEN_MAX);
free(url); free(url);
Link_curl_init(this_link); CURL *curl = Link_to_curl(this_link);
curl_easy_setopt(this_link->curl, CURLOPT_NOBODY, 1); curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
do_transfer(this_link->curl); do_transfer(curl);
long http_resp; long http_resp;
curl_easy_getinfo(this_link->curl, CURLINFO_RESPONSE_CODE, curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
&http_resp);
if (http_resp == HTTP_OK) { if (http_resp == HTTP_OK) {
double cl; double cl;
curl_easy_getinfo(this_link->curl, curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl);
if (cl == -1) { if (cl == -1) {
this_link->content_length = 0; this_link->content_length = 0;
this_link->type = LINK_DIR; this_link->type = LINK_DIR;
@ -357,8 +386,8 @@ void LinkTable_fill(LinkTable *linktbl)
} else { } else {
this_link->type = LINK_INVALID; this_link->type = LINK_INVALID;
} }
curl_easy_cleanup(curl);
} }
curl_easy_cleanup(this_link->curl);
} }
} }

5
network.h
View File

@ -33,7 +33,6 @@ struct Link {
char p_url[LINK_LEN_MAX]; char p_url[LINK_LEN_MAX];
char f_url[URL_LEN_MAX]; char f_url[URL_LEN_MAX];
LinkType type; LinkType type;
CURL *curl;
size_t content_length; size_t content_length;
LinkTable *next_table; LinkTable *next_table;
}; };
@ -53,8 +52,8 @@ void network_init(const char *url);
* \brief download a link */ * \brief download a link */
/* \return the number of bytes downloaded /* \return the number of bytes downloaded
*/ */
size_t Link_download(Link *link, MemoryStruct *mem, off_t offset, long Link_download(const char *path, char *output_buf, size_t size,
size_t size); off_t offset);
/** \brief create a new LinkTable */ /** \brief create a new LinkTable */
LinkTable *LinkTable_new(const char *url); LinkTable *LinkTable_new(const char *url);