we are encountering some sort of thread safety issue
This commit is contained in:
parent
431cad8ae4
commit
16d89b9ad0
45
data.h
45
data.h
|
@ -5,47 +5,16 @@
|
|||
* \brief This header stores all the custom data type definition
|
||||
*/
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define URL_LEN_MAX 2048
|
||||
#define LINK_LEN_MAX 255
|
||||
#define HTTP_OK 200
|
||||
/** \brief use this data type for buffer */
|
||||
typedef struct MemoryStruct MemoryStruct;
|
||||
|
||||
/** \brief the link type */
|
||||
typedef enum {
|
||||
LINK_HEAD = 'H',
|
||||
LINK_DIR = 'D',
|
||||
LINK_FILE = 'F',
|
||||
LINK_UNKNOWN = 'U',
|
||||
LINK_INVALID = 'I'
|
||||
} LinkType;
|
||||
|
||||
/**
|
||||
* \brief link table type
|
||||
* \details index 0 contains the Link for the base URL
|
||||
*/
|
||||
typedef struct LinkTable LinkTable;
|
||||
|
||||
/** \brief link data type */
|
||||
typedef struct Link Link;
|
||||
|
||||
/** \brief root link table */
|
||||
extern LinkTable *ROOT_LINK_TBL;
|
||||
|
||||
struct Link {
|
||||
char p_url[LINK_LEN_MAX];
|
||||
char f_url[URL_LEN_MAX];
|
||||
LinkType type;
|
||||
CURL *curl;
|
||||
char *body;
|
||||
size_t body_sz;
|
||||
size_t content_length;
|
||||
LinkTable *next_table;
|
||||
struct MemoryStruct {
|
||||
size_t size;
|
||||
char *memory;
|
||||
};
|
||||
|
||||
struct LinkTable {
|
||||
int num;
|
||||
Link **links;
|
||||
};
|
||||
void MemoryStruct_free(MemoryStruct *ms);
|
||||
|
||||
#endif
|
||||
|
|
35
main.c
35
main.c
|
@ -23,9 +23,13 @@ static struct fuse_operations fs_oper = {
|
|||
.read = fs_read,
|
||||
};
|
||||
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
network_init(argv[1]);
|
||||
return fuse_main(argc - 1, argv + 1, &fs_oper, NULL);
|
||||
network_init("http://127.0.0.1/~fangfufu/");
|
||||
|
||||
// return fuse_main(argc - 1, argv + 1, &fs_oper, NULL);
|
||||
return fuse_main(argc, argv, &fs_oper, NULL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** \brief return the attributes for a single file indicated by path */
|
||||
|
@ -57,6 +61,7 @@ static int fs_getattr(const char *path, struct stat *stbuf)
|
|||
}
|
||||
}
|
||||
|
||||
fflush(stderr);
|
||||
return res;
|
||||
}
|
||||
|
||||
|
@ -80,7 +85,11 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
|
|||
}
|
||||
linktbl = link->next_table;
|
||||
if (!linktbl) {
|
||||
return -ENOENT;
|
||||
fprintf(stderr, "LinkTable_new(): %s\n", link->f_url);
|
||||
linktbl = LinkTable_new(link->f_url);
|
||||
if(!linktbl) {
|
||||
return -ENOENT;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -92,6 +101,7 @@ static int fs_readdir(const char *path, void *buf, fuse_fill_dir_t dir_add,
|
|||
dir_add(buf, link->p_url, NULL, 0);
|
||||
}
|
||||
|
||||
fflush(stderr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -106,6 +116,7 @@ static int fs_open(const char *path, struct fuse_file_info *fi)
|
|||
return -EACCES;
|
||||
}
|
||||
|
||||
fflush(stderr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -121,8 +132,22 @@ static int fs_read(const char *path, char *buf, size_t size, off_t offset,
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
int bytes_downloaded = Link_download(link, offset, size);
|
||||
memcpy(buf, link->body, bytes_downloaded);
|
||||
MemoryStruct ms;
|
||||
|
||||
fprintf(stderr, "fs_read(): Link_download(%s, %ld, %lu)\n",
|
||||
link->f_url,
|
||||
offset,
|
||||
size
|
||||
);
|
||||
fflush(stderr);
|
||||
size_t bytes_downloaded = Link_download(link, &ms, offset, size);
|
||||
fprintf(stderr,
|
||||
"fs_read(): returning from Link_download(), with %lu bytes.\n",
|
||||
bytes_downloaded);
|
||||
fflush(stderr);
|
||||
|
||||
memcpy(buf, ms.memory, size);
|
||||
// free(ms.memory);
|
||||
|
||||
return bytes_downloaded;
|
||||
}
|
||||
|
|
66
network.c
66
network.c
|
@ -6,7 +6,8 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define NETWORK_MAXIMUM_CONNECTION 10
|
||||
#define HTTP_OK 200
|
||||
#define HTTP_PARTIAL_CONTENT 206
|
||||
|
||||
LinkTable *ROOT_LINK_TBL;
|
||||
|
||||
|
@ -17,7 +18,6 @@ static size_t
|
|||
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp);
|
||||
static Link *Link_new(const char *p_url);
|
||||
static void Link_free(Link *link);
|
||||
static LinkTable *LinkTable_new(const char *url);
|
||||
static void LinkTable_free(LinkTable *linktbl);
|
||||
static void LinkTable_add(LinkTable *linktbl, Link *link);
|
||||
static void LinkTable_fill(LinkTable *linktbl);
|
||||
|
@ -64,7 +64,23 @@ static char *url_append(const char *url, const char *sublink)
|
|||
return str;
|
||||
}
|
||||
|
||||
/* This is the single thread version */
|
||||
static void do_transfer(CURL *curl)
|
||||
{
|
||||
fprintf(stderr, "do_transfer(): handle %x\n", curl);
|
||||
fflush(stderr);
|
||||
CURLcode res = curl_easy_perform(curl);
|
||||
if (res) {
|
||||
fprintf(stderr, "do_transfer() failed: %u, %s\n", res,
|
||||
curl_easy_strerror(res));
|
||||
fflush(stderr);
|
||||
}
|
||||
fprintf(stderr, "do_transfer(): DONE\n");
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
/* This is the version that uses curl multi handle */
|
||||
static void do_transfer_(CURL *curl)
|
||||
{
|
||||
/* Add the transfer handle */
|
||||
curl_multi_add_handle(curl_multi, curl);
|
||||
|
@ -74,6 +90,9 @@ static void do_transfer(CURL *curl)
|
|||
long timeout;
|
||||
fd_set read_fd_set, write_fd_set, exc_fd_set;
|
||||
do {
|
||||
fprintf(stderr, "do_transfer(): num_transfers: %d\n",
|
||||
num_transfers);
|
||||
fflush(stderr);
|
||||
res = curl_multi_perform(curl_multi, &num_transfers);
|
||||
if (res) {
|
||||
fprintf(stderr,
|
||||
|
@ -135,29 +154,26 @@ static void do_transfer(CURL *curl)
|
|||
static size_t
|
||||
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
|
||||
{
|
||||
MemoryStruct *mem = (MemoryStruct *)userp;
|
||||
size_t realsize = size * nmemb;
|
||||
Link *mem = (Link *)userp;
|
||||
|
||||
mem->body = realloc(mem->body, mem->body_sz + realsize + 1);
|
||||
if(mem->body == NULL) {
|
||||
mem->memory = malloc(realsize + 1);
|
||||
if(!mem->memory) {
|
||||
/* out of memory! */
|
||||
printf("not enough memory (realloc returned NULL)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy(&(mem->body[mem->body_sz]), contents, realsize);
|
||||
mem->body_sz += realsize;
|
||||
mem->body[mem->body_sz] = 0;
|
||||
|
||||
return realsize;
|
||||
memcpy(mem->memory, contents, realsize);
|
||||
return realsize - 1;
|
||||
}
|
||||
|
||||
void network_init(const char *url)
|
||||
{
|
||||
curl_global_init(CURL_GLOBAL_ALL);
|
||||
curl_multi = curl_multi_init();
|
||||
curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
|
||||
(long)NETWORK_MAXIMUM_CONNECTION);
|
||||
// curl_multi = curl_multi_init();
|
||||
// curl_multi_setopt(curl_multi, CURLMOPT_MAXCONNECTS,
|
||||
// (long)NETWORK_MAXIMUM_CONNECTION);
|
||||
ROOT_LINK_TBL = LinkTable_new(url);
|
||||
}
|
||||
|
||||
|
@ -179,7 +195,6 @@ static Link *Link_new(const char *p_url)
|
|||
|
||||
/* set up some basic curl stuff */
|
||||
curl_easy_setopt(link->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||
curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)link);
|
||||
curl_easy_setopt(link->curl, CURLOPT_USERAGENT, "mount-http-dir/libcurl");
|
||||
curl_easy_setopt(link->curl, CURLOPT_VERBOSE, 0);
|
||||
curl_easy_setopt(link->curl, CURLOPT_FOLLOWLOCATION, 1);
|
||||
|
@ -195,12 +210,12 @@ static Link *Link_new(const char *p_url)
|
|||
static void Link_free(Link *link)
|
||||
{
|
||||
curl_easy_cleanup(link->curl);
|
||||
free(link->body);
|
||||
free(link);
|
||||
link = NULL;
|
||||
}
|
||||
|
||||
size_t Link_download(Link *link, off_t offset, size_t size)
|
||||
size_t Link_download(Link *link, MemoryStruct *ms, off_t offset,
|
||||
size_t size)
|
||||
{
|
||||
size_t start = offset;
|
||||
size_t end = start + size;
|
||||
|
@ -209,13 +224,16 @@ size_t Link_download(Link *link, off_t offset, size_t size)
|
|||
snprintf(range_str, sizeof(range_str), "%lu-%lu", start, end);
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
|
||||
curl_easy_setopt(curl, CURLOPT_URL, link->f_url);
|
||||
curl_easy_setopt(curl, CURLOPT_RANGE, range_str);
|
||||
curl_easy_setopt(link->curl, CURLOPT_WRITEDATA, (void *)ms);
|
||||
|
||||
do_transfer(curl);
|
||||
|
||||
long http_resp;
|
||||
curl_easy_getinfo(link->curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||
if (http_resp != HTTP_OK) {
|
||||
fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld",
|
||||
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_resp);
|
||||
if ( (http_resp != HTTP_OK) && ( http_resp != HTTP_PARTIAL_CONTENT) ) {
|
||||
fprintf(stderr, "Link_download(): Could not download %s, HTTP %ld\n",
|
||||
link->f_url, http_resp);
|
||||
return 0;
|
||||
}
|
||||
|
@ -223,10 +241,12 @@ size_t Link_download(Link *link, off_t offset, size_t size)
|
|||
double dl;
|
||||
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dl);
|
||||
size_t s = dl;
|
||||
|
||||
fflush(stdout);
|
||||
return s;
|
||||
}
|
||||
|
||||
static LinkTable *LinkTable_new(const char *url)
|
||||
LinkTable *LinkTable_new(const char *url)
|
||||
{
|
||||
LinkTable *linktbl = calloc(1, sizeof(LinkTable));
|
||||
|
||||
|
@ -237,6 +257,9 @@ static LinkTable *LinkTable_new(const char *url)
|
|||
curl_easy_setopt(head_link->curl, CURLOPT_URL, url);
|
||||
|
||||
/* start downloading the base URL */
|
||||
MemoryStruct ms;
|
||||
curl_easy_setopt(head_link->curl, CURLOPT_WRITEDATA, (void *)&ms);
|
||||
|
||||
do_transfer(head_link->curl);
|
||||
|
||||
/* if downloading base URL failed */
|
||||
|
@ -251,7 +274,8 @@ URL: %s, HTTP %ld\n", url, http_resp);
|
|||
};
|
||||
|
||||
/* Otherwise parsed the received data */
|
||||
GumboOutput* output = gumbo_parse(head_link->body);
|
||||
GumboOutput* output = gumbo_parse(ms.memory);
|
||||
free(ms.memory);
|
||||
HTML_to_LinkTable(output->root, linktbl);
|
||||
gumbo_destroy_output(&kGumboDefaultOptions, output);
|
||||
|
||||
|
|
47
network.h
47
network.h
|
@ -3,8 +3,49 @@
|
|||
|
||||
#include "data.h"
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <gumbo.h>
|
||||
|
||||
#define URL_LEN_MAX 2048
|
||||
#define LINK_LEN_MAX 255
|
||||
#define NETWORK_MAXIMUM_CONNECTION 10
|
||||
|
||||
/** \brief the link type */
|
||||
typedef enum {
|
||||
LINK_HEAD = 'H',
|
||||
LINK_DIR = 'D',
|
||||
LINK_FILE = 'F',
|
||||
LINK_UNKNOWN = 'U',
|
||||
LINK_INVALID = 'I'
|
||||
} LinkType;
|
||||
|
||||
/**
|
||||
* \brief link table type
|
||||
* \details index 0 contains the Link for the base URL
|
||||
*/
|
||||
typedef struct LinkTable LinkTable;
|
||||
|
||||
/** \brief link data type */
|
||||
typedef struct Link Link;
|
||||
|
||||
|
||||
struct Link {
|
||||
char p_url[LINK_LEN_MAX];
|
||||
char f_url[URL_LEN_MAX];
|
||||
LinkType type;
|
||||
CURL *curl;
|
||||
size_t content_length;
|
||||
LinkTable *next_table;
|
||||
};
|
||||
|
||||
struct LinkTable {
|
||||
int num;
|
||||
Link **links;
|
||||
};
|
||||
|
||||
/** \brief root link table */
|
||||
extern LinkTable *ROOT_LINK_TBL;
|
||||
|
||||
/** \brief Initialise the network module */
|
||||
void network_init(const char *url);
|
||||
|
||||
|
@ -12,7 +53,11 @@ void network_init(const char *url);
|
|||
* \brief download a link */
|
||||
/* \return the number of bytes downloaded
|
||||
*/
|
||||
size_t Link_download(Link *link, off_t offset, size_t size);
|
||||
size_t Link_download(Link *link, MemoryStruct *mem, off_t offset,
|
||||
size_t size);
|
||||
|
||||
/** \brief create a new LinkTable */
|
||||
LinkTable *LinkTable_new(const char *url);
|
||||
|
||||
/** \brief print a LinkTable */
|
||||
void LinkTable_print(LinkTable *linktbl);
|
||||
|
|
Loading…
Reference in New Issue