From 4acf91a2b8b1eb6e26cf4f77462c20778fc0c6ea Mon Sep 17 00:00:00 2001 From: Fufu Fang Date: Sat, 27 Apr 2019 04:40:26 +0100 Subject: [PATCH] added --max-seg-count command line option --- src/cache.c | 8 +++++--- src/cache.h | 6 ++++++ src/main.c | 32 +++++++++++++++++++++----------- src/network.c | 9 +++++---- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/cache.c b/src/cache.c index 99ae9d9..154ee51 100644 --- a/src/cache.c +++ b/src/cache.c @@ -20,10 +20,11 @@ /** * \brief Maximum segment block count - * \details This is set to 1024*1024*1024 = 1 GB, which allows the user to - * access a 8TB file. + * \details This is set to 128*1024 blocks, which uses 128KB. By default, + * this allows the user to store (128*1024)*(8*1024*1024) = 1TB of data */ -#define MAX_SEGBC 1073741824 + +#define DEFAULT_MAX_SEGBC 128*1024 /** * \brief error associated with metadata @@ -39,6 +40,7 @@ typedef enum { int CACHE_SYSTEM_INIT = 0; int DATA_BLK_SZ = 0; +int MAX_SEGBC = DEFAULT_MAX_SEGBC; /** * \brief The metadata directory diff --git a/src/cache.h b/src/cache.h index 73249c0..7c2f181 100644 --- a/src/cache.h +++ b/src/cache.h @@ -53,6 +53,12 @@ extern int CACHE_SYSTEM_INIT; */ extern int DATA_BLK_SZ; +/** + * \brief The maximum segment count for a single cache file + */ +extern int MAX_SEGBC; + + /** * \brief initialise the cache system directories * \details This function basically sets up the following variables: diff --git a/src/main.c b/src/main.c index b59c66a..40c82eb 100644 --- a/src/main.c +++ b/src/main.c @@ -141,9 +141,10 @@ parse_arg_list(int argc, char **argv, char ***fuse_argv, int *fuse_argc) {"proxy-password", required_argument, NULL, 'L'}, /* 7 */ {"cache", required_argument, NULL, 'L'}, /* 8 */ {"dl-seg-size", required_argument, NULL, 'L'}, /* 9 */ - {"max-conns", required_argument, NULL, 'L'}, /* 10 */ - {"user-agent", required_argument, NULL, 'L'}, /* 11 */ - {"retry-wait", required_argument, NULL, 'L'}, /* 12 */ + {"max-seg-count", required_argument, NULL, 'L'}, /* 10 */ + {"max-conns", required_argument, NULL, 'L'}, /* 11 */ + {"user-agent", required_argument, NULL, 'L'}, /* 12 */ + {"retry-wait", required_argument, NULL, 'L'}, /* 13 */ {0, 0, 0, 0} }; while ((c = @@ -197,12 +198,15 @@ parse_arg_list(int argc, char **argv, char ***fuse_argv, int *fuse_argc) DATA_BLK_SZ = atoi(optarg) * 1024 * 1024; break; case 10: - NETWORK_CONFIG.max_conns = atoi(optarg); + MAX_SEGBC = atoi(optarg); break; case 11: - NETWORK_CONFIG.user_agent = strdup(optarg); + NETWORK_CONFIG.max_conns = atoi(optarg); break; case 12: + NETWORK_CONFIG.user_agent = strdup(optarg); + break; + case 13: HTTP_429_WAIT = atoi(optarg); break; default: @@ -260,12 +264,18 @@ static void print_http_options() --cache Set a cache folder, by default this is disabled\n\ --dl-seg-size The size of each download segment in MB,\n\ default to 8MB.\n\ - --max-conns The maximum number of network connections that\ - libcurl is allowed to make, default to 10.\ - --user-agent The user agent string, default to \"HTTPDirFS\".\ - --retry-wait The waiting interval in seconds before making an\ - HTTP request, after encountering an error, default\ - to 5 seconds.\ + --max-seg-count The maximum number of download segments a file\n\ + can have. By default it is set to 1048576. This\n\ + means the maximum memory usage per file is 1MB\n\ + memory. This allows caching file up to 8TB in\n\ + size, assuming you are using the default segment\n\ + size.\n\ + --max-conns The maximum number of network connections that\n\ + libcurl is allowed to make, default to 10.\n\ + --user-agent The user agent string, default to \"HTTPDirFS\".\n\ + --retry-wait The waiting interval in seconds before making an\n\ + HTTP request, after encountering an error, \n\ + default to 5 seconds.\n\ \n\ libfuse options:\n"); } diff --git a/src/network.c b/src/network.c index e72b721..a849848 100644 --- a/src/network.c +++ b/src/network.c @@ -88,7 +88,8 @@ static void curl_callback_unlock(CURL *handle, curl_lock_data data, * Adapted from: * https://curl.haxx.se/libcurl/c/10-at-a-time.html */ -static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, int n_mesgs) +static void curl_process_msgs(CURLMsg *curl_msg, int n_running_curl, + int n_mesgs) { (void) n_running_curl; (void) n_mesgs; @@ -323,8 +324,8 @@ void transfer_nonblocking(CURL *curl) } } -size_t -write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp) +size_t write_memory_callback(void *contents, size_t size, size_t nmemb, + void *userp) { size_t realsize = size * nmemb; MemoryStruct *mem = (MemoryStruct *)userp; @@ -337,7 +338,7 @@ write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp) return 0; } - memmove(&(mem->memory[mem->size]), contents, realsize); + memmove(&mem->memory[mem->size], contents, realsize); mem->size += realsize; mem->memory[mem->size] = 0;