rdelaage
/
httpdirfs
mirror of https://github.com/fangfufu/httpdirfs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

we are encountering some sort of thread safety issue

This commit is contained in:
Fufu Fang 2018-07-22 19:30:08 +01:00
parent 431cad8ae4
commit 16d89b9ad0
4 changed files with 128 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
data.h
View File

@ -5,47 +5,16 @@
* \brief This header stores all the custom data type definition
*/
#include <curl/curl.h>
#include <stdlib.h>
#define URL_LEN_MAX 2048
#define LINK_LEN_MAX 255
#define HTTP_OK 200
/** \brief use this data type for buffer */
typedef struct MemoryStruct MemoryStruct;
/** \brief the link type */
typedef enum {
LINK_HEAD = 'H',
LINK_DIR = 'D',
LINK_FILE = 'F',
LINK_UNKNOWN = 'U',
LINK_INVALID = 'I'
} LinkType;
/**
* \brief link table type
* \details index 0 contains the Link for the base URL
*/
typedef struct LinkTable LinkTable;
/** \brief link data type */
typedef struct Link Link;
/** \brief root link table */
extern LinkTable *ROOT_LINK_TBL;
struct Link {
char p_url[LINK_LEN_MAX];
char f_url[URL_LEN_MAX];
LinkType type;
CURL *curl;
char *body;
size_t body_sz;
size_t content_length;
LinkTable *next_table;
struct MemoryStruct {
size_t size;
char *memory;
};
struct LinkTable {
int num;
Link **links;
};
void MemoryStruct_free(MemoryStruct *ms);
#endif

35
main.c
View File

@ -23,9 +23,13 @@ static struct fuse_operations fs_oper = {
.read = fs_read,
};
int main(int argc, char **argv) {
network_init(argv[1]);
return fuse_main(argc - 1, argv + 1, &fs_oper, NULL);
network_init("http://127.0.0.1/~fangfufu/");
// return fuse_main(argc - 1, argv + 1, &fs_oper, NULL);
return fuse_main(argc, argv, &fs_oper, NULL);
return 0;
}
/** \brief return the attributes for a single file indicated by path */
@ -57,6 +61,7 @@ static int fs_getattr(const char *path, struct stat *stbuf)
}
}
fflush(stderr);
return res;
}
@ -80,7 +85,11 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
}
linktbl = link->next_table;
if (!linktbl) {
return -ENOENT;
fprintf(stderr, "LinkTable_new(): %s\n", link->f_url);
linktbl = LinkTable_new(link->f_url);
if(!linktbl) {
return -ENOENT;
}
}
}
@ -92,6 +101,7 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
dir_add(buf, link->p_url, NULL, 0);
}
fflush(stderr);
return 0;
}
@ -106,6 +116,7 @@ static int fs_open(const char *path, struct fuse_file_info *fi)
return -EACCES;
}
fflush(stderr);
return 0;
}
@ -121,8 +132,22 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset,
return -ENOENT;
}
int bytes_downloaded = Link_download(link, offset, size);
memcpy(buf, link->body, bytes_downloaded);
MemoryStruct ms;
fprintf(stderr, "fs_read(): Link_download(%s, %ld, %lu)\n",
link->f_url,
offset,
size
);
fflush(stderr);
size_t bytes_downloaded = Link_download(link, &ms, offset, size);
fprintf(stderr,
"fs_read(): returning from Link_download(), with %lu bytes.\n",
bytes_downloaded);
fflush(stderr);
memcpy(buf, ms.memory, size);
// free(ms.memory);
return bytes_downloaded;
}

66
network.c
View File

@ -6,7 +6,8 @@
#include <stdio.h>
#include <stdlib.h>
#define NETWORK_MAXIMUM_CONNECTION 10
#define HTTP_OK 200
#define HTTP_PARTIAL_CONTENT 206
LinkTable *ROOT_LINK_TBL;
@ -17,7 +18,6 @@ static size_t
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp);
static Link *Link_new(const char *p_url);
static void Link_free(Link *link);
static LinkTable *LinkTable_new(const char *url);
static void LinkTable_free(LinkTable *linktbl);
static void LinkTable_add(LinkTable *linktbl, Link *link);
static void LinkTable_fill(LinkTable *linktbl);
@ -64,7 +64,23 @@ static char *url_append(const char *url, const char *sublink)
return str;
}
/* This is the single thread version */
static void do_transfer(CURL *curl)
{
fprintf(stderr, "do_transfer(): handle %x\n", curl);
fflush(stderr);
CURLcode res = curl_easy_perform(curl);
if (res) {
fprintf(stderr, "do_transfer() failed: %u, %s\n", res,
curl_easy_strerror(res));
fflush(stderr);
}
fprintf(stderr, "do_transfer(): DONE\n");
fflush(stderr);
}
/* This is the version that uses curl multi handle */
static void do_transfer_(CURL *curl)
{
/* Add the transfer handle */
curl_multi_add_handle(curl_multi, curl);
@ -74,6 +90,9 @@ static void do_transfer(CURL *curl)
long timeout;
fd_set read_fd_set, write_fd_set, exc_fd_set;
do {
fprintf(stderr, "do_transfer(): num_transfers: %d\n",
num_transfers);
fflush(stderr);
res = curl_multi_perform(curl_multi, &num_transfers);
if (res) {
fprintf(stderr,
@ -135,29 +154,26 @@ static void do_transfer(CURL *curl)
static size_t
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
MemoryStruct *mem = (MemoryStruct *)userp;
size_t realsize = size * nmemb;
Link *mem = (Link *)userp;
mem->body = realloc(mem->body, mem->body_sz + realsize + 1);
if(mem->body == NULL) {
mem->memory = malloc(realsize + 1);
if(!mem->memory) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
memcpy(&(mem->body[mem->body_sz]), contents, realsize);
mem->body_sz += realsize;
mem->body[mem->body_sz] = 0;
return realsize;
memcpy(mem->memory, contents, realsize);
return realsize - 1;
}
void network_init(const char *url)
{
curl_global_init(CURL_GLOBAL_ALL);
curl_multi = curl_multi_init();
curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
(long)NETWORK_MAXIMUM_CONNECTION);
// curl_multi = curl_multi_init();
// curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
// (long)NETWORK_MAXIMUM_CONNECTION);
ROOT_LINK_TBL = LinkTable_new(url);
}
@ -179,7 +195,6 @@ static Link *Link_new(const char *p_url)
/* set up some basic curl stuff */
curl_easy_setopt(link->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)link);
curl_easy_setopt(link->curl, CURLOPT_USERAGENT, "mount-http-dir/libcurl");
curl_easy_setopt(link->curl, CURLOPT_VERBOSE, 0);
curl_easy_setopt(link->curl, CURLOPT_FOLLOWLOCATION, 1);
@ -195,12 +210,12 @@ static Link *Link_new(const char *p_url)
static void Link_free(Link *link)
{
curl_easy_cleanup(link->curl);
free(link->body);
free(link);
link = NULL;
}
size_t Link_download(Link *link, off_t offset, size_t size)
size_t Link_download(Link *link, MemoryStruct *ms, off_t offset,
size_t size)
{
size_t start = offset;
size_t end = start + size;
@ -209,13 +224,16 @@ size_t Link_download(Link *link, off_t offset, size_t size)
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
curl_easy_setopt(curl, CURLOPT_URL, link->f_url);
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)ms);
do_transfer(curl);
long http_resp;
curl_easy_getinfo(link->curl, CURLINFO_RESPONSE_CODE, &http_resp);
if (http_resp != HTTP_OK) {
fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld",
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
if ( (http_resp != HTTP_OK) && ( http_resp != HTTP_PARTIAL_CONTENT) ) {
fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld\n",
link->f_url, http_resp);
return 0;
}
@ -223,10 +241,12 @@ size_t Link_download(Link *link, off_t offset, size_t size)
double dl;
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dl);
size_t s = dl;
fflush(stdout);
return s;
}
static LinkTable *LinkTable_new(const char *url)
LinkTable *LinkTable_new(const char *url)
{
LinkTable *linktbl = calloc(1, sizeof(LinkTable));
@ -237,6 +257,9 @@ static LinkTable *LinkTable_new(const char *url)
curl_easy_setopt(head_link->curl, CURLOPT_URL, url);
/* start downloading the base URL */
MemoryStruct ms;
curl_easy_setopt(head_link->curl, CURLOPT_WRITEDATA, (void *)&ms);
do_transfer(head_link->curl);
/* if downloading base URL failed */
@ -251,7 +274,8 @@ URL: %s, HTTP %ld\n", url, http_resp);
};
/* Otherwise parsed the received data */
GumboOutput* output = gumbo_parse(head_link->body);
GumboOutput* output = gumbo_parse(ms.memory);
free(ms.memory);
HTML_to_LinkTable(output->root, linktbl);
gumbo_destroy_output(&kGumboDefaultOptions, output);

47
network.h
View File

@ -3,8 +3,49 @@
#include "data.h"
#include <curl/curl.h>
#include <gumbo.h>
#define URL_LEN_MAX 2048
#define LINK_LEN_MAX 255
#define NETWORK_MAXIMUM_CONNECTION 10
/** \brief the link type */
typedef enum {
LINK_HEAD = 'H',
LINK_DIR = 'D',
LINK_FILE = 'F',
LINK_UNKNOWN = 'U',
LINK_INVALID = 'I'
} LinkType;
/**
* \brief link table type
* \details index 0 contains the Link for the base URL
*/
typedef struct LinkTable LinkTable;
/** \brief link data type */
typedef struct Link Link;
struct Link {
char p_url[LINK_LEN_MAX];
char f_url[URL_LEN_MAX];
LinkType type;
CURL *curl;
size_t content_length;
LinkTable *next_table;
};
struct LinkTable {
int num;
Link **links;
};
/** \brief root link table */
extern LinkTable *ROOT_LINK_TBL;
/** \brief Initialise the network module */
void network_init(const char *url);
@ -12,7 +53,11 @@ void network_init(const char *url);
* \brief download a link */
/* \return the number of bytes downloaded
*/
size_t Link_download(Link *link, off_t offset, size_t size);
size_t Link_download(Link *link, MemoryStruct *mem, off_t offset,
size_t size);
/** \brief create a new LinkTable */
LinkTable *LinkTable_new(const char *url);
/** \brief print a LinkTable */
void LinkTable_print(LinkTable *linktbl);