2019-04-09 13:51:15 +02:00
|
|
|
#include "cache.h"
|
2021-09-01 22:29:13 +02:00
|
|
|
|
2021-08-22 01:51:37 +02:00
|
|
|
#include "config.h"
|
2021-09-01 22:29:13 +02:00
|
|
|
#include "util.h"
|
2021-08-22 03:26:09 +02:00
|
|
|
#include "log.h"
|
2019-04-12 14:52:11 +02:00
|
|
|
|
2019-04-24 00:48:08 +02:00
|
|
|
#include <sys/stat.h>
|
|
|
|
|
2019-04-21 21:48:44 +02:00
|
|
|
#include <dirent.h>
|
2019-04-20 03:04:29 +02:00
|
|
|
#include <errno.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
2019-04-24 00:48:08 +02:00
|
|
|
#include <unistd.h>
|
2019-04-20 03:04:29 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
/*
|
|
|
|
* ---------------- External variables -----------------------
|
|
|
|
*/
|
2019-04-22 12:08:42 +02:00
|
|
|
int CACHE_SYSTEM_INIT = 0;
|
2019-09-04 20:53:11 +02:00
|
|
|
char *META_DIR;
|
2019-04-22 12:08:42 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
/*
|
|
|
|
* ----------------- Static variables -----------------------
|
|
|
|
*/
|
2019-09-01 01:43:50 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
* \brief Cache file locking
|
|
|
|
* \details Ensure cache opening and cache closing is an atomic operation
|
|
|
|
*/
|
|
|
|
static pthread_mutex_t cf_lock;
|
|
|
|
|
2019-04-20 03:04:29 +02:00
|
|
|
/**
|
|
|
|
* \brief The data directory
|
|
|
|
*/
|
2019-04-22 03:33:26 +02:00
|
|
|
static char *DATA_DIR;
|
|
|
|
|
2019-04-30 09:05:46 +02:00
|
|
|
/**
|
2021-08-29 23:46:24 +02:00
|
|
|
* \brief Calculate cache system directory path
|
2019-04-30 09:05:46 +02:00
|
|
|
*/
|
|
|
|
static char *CacheSystem_calc_dir(const char *url)
|
2019-04-12 14:52:11 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *xdg_cache_home = getenv("XDG_CACHE_HOME");
|
|
|
|
if (!xdg_cache_home) {
|
|
|
|
char *home = getenv("HOME");
|
|
|
|
char *xdg_cache_home_default = "/.cache";
|
|
|
|
xdg_cache_home = path_append(home, xdg_cache_home_default);
|
|
|
|
}
|
|
|
|
if (mkdir
|
|
|
|
(xdg_cache_home, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
|
|
|
&& (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
char *cache_dir_root = path_append(xdg_cache_home, "/httpdirfs/");
|
|
|
|
if (mkdir
|
|
|
|
(cache_dir_root, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
|
|
|
&& (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
|
|
|
|
char *fn = path_append(cache_dir_root, "/CACHEDIR.TAG");
|
|
|
|
FILE *fp = fopen(fn, "w");
|
|
|
|
if (fp) {
|
|
|
|
fprintf(fp, "Signature: 8a477f597d28d172789f06886806bc55\n\
|
2019-04-30 09:05:46 +02:00
|
|
|
# This file is a cache directory tag created by httpdirfs.\n\
|
|
|
|
# For information about cache directory tags, see:\n\
|
|
|
|
# http://www.brynosaurus.com/cachedir/\n");
|
2021-08-31 12:18:39 +02:00
|
|
|
} else {
|
|
|
|
lprintf(fatal, "fopen(%s): %s", fn, strerror(errno));
|
|
|
|
}
|
|
|
|
if (ferror(fp)) {
|
|
|
|
lprintf(fatal, "fwrite(): encountered error!\n");
|
|
|
|
}
|
|
|
|
if (fclose(fp)) {
|
|
|
|
lprintf(fatal, "fclose(%s): %s\n", fn, strerror(errno));
|
|
|
|
}
|
|
|
|
CURL *c = curl_easy_init();
|
|
|
|
char *escaped_url = curl_easy_escape(c, url, 0);
|
|
|
|
char *full_path = path_append(cache_dir_root, escaped_url);
|
|
|
|
if (mkdir(full_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
|
|
|
&& (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
FREE(fn);
|
|
|
|
FREE(cache_dir_root);
|
|
|
|
curl_free(escaped_url);
|
|
|
|
curl_easy_cleanup(c);
|
|
|
|
return full_path;
|
2019-04-30 09:05:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void CacheSystem_init(const char *path, int url_supplied)
|
|
|
|
{
|
2021-09-02 17:07:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: initialise cf_lock;\n", pthread_self());
|
|
|
|
if (pthread_mutex_init(&cf_lock, NULL)) {
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(fatal, "cf_lock initialisation failed!\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (url_supplied) {
|
|
|
|
path = CacheSystem_calc_dir(path);
|
|
|
|
}
|
|
|
|
|
2021-08-31 12:50:59 +02:00
|
|
|
lprintf(debug, "%s\n", path);
|
2021-08-31 12:18:39 +02:00
|
|
|
|
|
|
|
META_DIR = path_append(path, "meta/");
|
|
|
|
DATA_DIR = path_append(path, "data/");
|
|
|
|
/*
|
|
|
|
* Check if directories exist, if not, create them
|
|
|
|
*/
|
|
|
|
if (mkdir(META_DIR, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
|
|
|
&& (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (mkdir(DATA_DIR, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
|
|
|
&& (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (CONFIG.mode == SONIC) {
|
|
|
|
char *sonic_path;
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* Create "rest" sub-directory for META_DIR
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
sonic_path = path_append(META_DIR, "rest/");
|
|
|
|
if (mkdir
|
|
|
|
(sonic_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
2021-08-31 12:15:00 +02:00
|
|
|
&& (errno != EEXIST)) {
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
FREE(sonic_path);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* Create "rest" sub-directory for DATA_DIR
|
|
|
|
*/
|
|
|
|
sonic_path = path_append(DATA_DIR, "rest/");
|
|
|
|
if (mkdir
|
|
|
|
(sonic_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
|
2021-08-31 12:15:00 +02:00
|
|
|
&& (errno != EEXIST)) {
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
FREE(sonic_path);
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
CACHE_SYSTEM_INIT = 1;
|
2019-04-21 01:42:32 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief read a metadata file
|
2021-08-30 03:50:03 +02:00
|
|
|
* \return 0 on success, errno on error.
|
2019-04-22 22:20:41 +02:00
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static int Meta_read(Cache * cf)
|
2019-04-12 14:52:11 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
FILE *fp = cf->mfp;
|
|
|
|
rewind(fp);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
int nmemb = 0;
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (!fp) {
|
|
|
|
/*
|
|
|
|
* The metadata file does not exist
|
|
|
|
*/
|
|
|
|
lprintf(error, "fopen(): %s\n", strerror(errno));
|
|
|
|
return EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
fread(&cf->time, sizeof(long), 1, fp);
|
|
|
|
fread(&cf->content_length, sizeof(off_t), 1, fp);
|
|
|
|
fread(&cf->blksz, sizeof(int), 1, fp);
|
|
|
|
fread(&cf->segbc, sizeof(long), 1, fp);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Error checking for fread
|
|
|
|
*/
|
|
|
|
if (ferror(fp)) {
|
|
|
|
lprintf(error, "error reading core metadata!\n");
|
|
|
|
return EIO;
|
|
|
|
}
|
|
|
|
|
2021-08-31 19:49:49 +02:00
|
|
|
/* These things really should not be zero!!! */
|
|
|
|
if (!cf->content_length || !cf->blksz || !cf->segbc) {
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(error,
|
2021-08-31 19:49:49 +02:00
|
|
|
"corruption: content_length: %ld, blksz: %d, segbc: %ld\n",
|
|
|
|
cf->content_length, cf->blksz, cf->segbc);
|
2021-08-31 12:18:39 +02:00
|
|
|
return EBADMSG;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cf->blksz != CONFIG.data_blksz) {
|
|
|
|
lprintf(warning, "Warning: cf->blksz != CONFIG.data_blksz\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cf->segbc > CONFIG.max_segbc) {
|
|
|
|
lprintf(error, "Error: segbc: %ld\n", cf->segbc);
|
|
|
|
return EFBIG;
|
|
|
|
}
|
|
|
|
|
2021-08-31 19:54:58 +02:00
|
|
|
/*
|
|
|
|
* Allocate memory for all segments, and read them in
|
|
|
|
*/
|
|
|
|
cf->seg = CALLOC(cf->segbc, sizeof(Seg));
|
|
|
|
nmemb = fread(cf->seg, sizeof(Seg), cf->segbc, fp);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 19:54:58 +02:00
|
|
|
/*
|
|
|
|
* We shouldn't have gone past the end of the file
|
|
|
|
*/
|
|
|
|
if (feof(fp)) {
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
2021-08-31 19:54:58 +02:00
|
|
|
* reached EOF
|
2021-08-31 12:18:39 +02:00
|
|
|
*/
|
2021-08-31 19:54:58 +02:00
|
|
|
lprintf(error, "attempted to read past the end of the \
|
2021-08-31 12:18:39 +02:00
|
|
|
file!\n");
|
2021-08-31 19:54:58 +02:00
|
|
|
return EBADMSG;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 19:54:58 +02:00
|
|
|
/*
|
|
|
|
* Error checking for fread
|
|
|
|
*/
|
|
|
|
if (ferror(fp)) {
|
|
|
|
lprintf(error, "error reading bitmap!\n");
|
|
|
|
return EIO;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 19:54:58 +02:00
|
|
|
/*
|
|
|
|
* Check for inconsistent metadata file
|
|
|
|
*/
|
|
|
|
if (nmemb != cf->segbc) {
|
|
|
|
lprintf(error, "corrupted metadata!\n");
|
|
|
|
return EBADMSG;
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
return 0;
|
2019-04-12 14:52:11 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief write a metadata file
|
|
|
|
* \return
|
|
|
|
* - -1 on error,
|
|
|
|
* - 0 on success
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static int Meta_write(Cache * cf)
|
2019-04-18 10:48:40 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
FILE *fp = cf->mfp;
|
|
|
|
rewind(fp);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (!fp) {
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* Cannot create the metadata file
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(error, "fopen(): %s\n", strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* These things really should not be zero!!!
|
|
|
|
*/
|
|
|
|
if (!cf->content_length || !cf->blksz || !cf->segbc) {
|
2021-08-31 19:49:49 +02:00
|
|
|
lprintf(error, "content_length: %ld, blksz: %d, segbc: %ld\n",
|
|
|
|
cf->content_length, cf->blksz, cf->segbc);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
fwrite(&cf->time, sizeof(long), 1, fp);
|
|
|
|
fwrite(&cf->content_length, sizeof(off_t), 1, fp);
|
|
|
|
fwrite(&cf->blksz, sizeof(int), 1, fp);
|
|
|
|
fwrite(&cf->segbc, sizeof(long), 1, fp);
|
2021-08-31 19:54:58 +02:00
|
|
|
fwrite(cf->seg, sizeof(Seg), cf->segbc, fp);
|
2021-08-31 12:18:39 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Error checking for fwrite
|
|
|
|
*/
|
|
|
|
if (ferror(fp)) {
|
|
|
|
lprintf(error, "fwrite(): encountered error!\n");
|
|
|
|
return -1;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
return 0;
|
2019-04-20 03:04:29 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief create a data file
|
|
|
|
* \details We use sparse creation here
|
2021-08-30 06:17:15 +02:00
|
|
|
* \return exit on failure
|
2019-04-22 22:20:41 +02:00
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void Data_create(Cache * cf)
|
2019-04-20 03:04:29 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
int fd;
|
|
|
|
int mode;
|
|
|
|
|
|
|
|
mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
|
|
|
|
char *datafn = path_append(DATA_DIR, cf->path);
|
|
|
|
fd = open(datafn, O_WRONLY | O_CREAT, mode);
|
|
|
|
FREE(datafn);
|
|
|
|
if (fd == -1) {
|
|
|
|
lprintf(fatal, "open(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
if (ftruncate(fd, cf->content_length)) {
|
|
|
|
lprintf(warning, "ftruncate(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
if (close(fd)) {
|
|
|
|
lprintf(fatal, "close:(): %s\n", strerror(errno));
|
|
|
|
}
|
2019-04-18 10:48:40 +02:00
|
|
|
}
|
2019-04-12 14:52:11 +02:00
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief obtain the data file size
|
2021-08-30 03:50:03 +02:00
|
|
|
* \return file size on success, -1 on error
|
2019-04-22 22:20:41 +02:00
|
|
|
*/
|
2019-04-22 03:33:26 +02:00
|
|
|
static long Data_size(const char *fn)
|
2019-04-21 00:46:08 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *datafn = path_append(DATA_DIR, fn);
|
|
|
|
struct stat st;
|
|
|
|
int s = stat(datafn, &st);
|
|
|
|
FREE(datafn);
|
|
|
|
if (!s) {
|
|
|
|
return st.st_size;
|
|
|
|
}
|
|
|
|
lprintf(error, "stat(): %s\n", strerror(errno));
|
|
|
|
return -1;
|
2019-04-21 00:46:08 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief read a data file
|
|
|
|
* \param[in] cf the pointer to the cache in-memory data structure
|
|
|
|
* \param[out] buf the output buffer
|
|
|
|
* \param[in] len the length of the segment
|
|
|
|
* \param[in] offset the offset of the segment
|
|
|
|
* \return
|
|
|
|
* - negative values on error,
|
|
|
|
* - otherwise, the number of bytes read.
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static long Data_read(Cache * cf, uint8_t * buf, off_t len, off_t offset)
|
2019-04-12 14:52:11 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
if (len == 0) {
|
|
|
|
lprintf(error, "requested to read 0 byte!\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: locking seek_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf->seek_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
long byte_read = 0;
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* Seek to the right location
|
|
|
|
*/
|
|
|
|
if (fseeko(cf->dfp, offset, SEEK_SET)) {
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* fseeko failed
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(error, "fseeko(): %s\n", strerror(errno));
|
|
|
|
byte_read = -EIO;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Calculate how much to read
|
|
|
|
*/
|
|
|
|
if (offset + len > cf->content_length) {
|
|
|
|
len -= offset + len - cf->content_length;
|
|
|
|
if (len < 0) {
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
byte_read = fread(buf, sizeof(uint8_t), len, cf->dfp);
|
|
|
|
if (byte_read != len) {
|
|
|
|
lprintf(debug,
|
|
|
|
"fread(): requested %ld, returned %ld!\n", len, byte_read);
|
|
|
|
if (feof(cf->dfp)) {
|
|
|
|
/*
|
|
|
|
* reached EOF
|
|
|
|
*/
|
|
|
|
lprintf(error, "fread(): reached the end of the file!\n");
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
if (ferror(cf->dfp)) {
|
|
|
|
/*
|
|
|
|
* filesystem error
|
|
|
|
*/
|
|
|
|
lprintf(error, "fread(): encountered error!\n");
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-30 12:24:32 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
end:
|
2021-08-31 01:13:17 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking seek_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->seek_lock);
|
|
|
|
return byte_read;
|
2019-04-12 14:52:11 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief write to a data file
|
|
|
|
* \param[in] cf the pointer to the cache in-memory data structure
|
|
|
|
* \param[in] buf the input buffer
|
|
|
|
* \param[in] len the length of the segment
|
|
|
|
* \param[in] offset the offset of the segment
|
|
|
|
* \return
|
|
|
|
* - -1 when the data file does not exist
|
|
|
|
* - otherwise, the number of bytes written.
|
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
static long Data_write(Cache * cf, const uint8_t * buf, off_t len,
|
|
|
|
off_t offset)
|
2019-04-12 14:52:11 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
if (len == 0) {
|
|
|
|
/*
|
|
|
|
* We should permit empty files
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: locking seek_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf->seek_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
long byte_written = 0;
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (fseeko(cf->dfp, offset, SEEK_SET)) {
|
|
|
|
/*
|
|
|
|
* fseeko failed
|
|
|
|
*/
|
|
|
|
lprintf(error, "fseeko(): %s\n", strerror(errno));
|
|
|
|
byte_written = -EIO;
|
|
|
|
goto end;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
byte_written = fwrite(buf, sizeof(uint8_t), len, cf->dfp);
|
2021-08-30 12:24:32 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (byte_written != len) {
|
|
|
|
lprintf(error,
|
|
|
|
"fwrite(): requested %ld, returned %ld!\n",
|
|
|
|
len, byte_written);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ferror(cf->dfp)) {
|
|
|
|
/*
|
|
|
|
* filesystem error
|
|
|
|
*/
|
|
|
|
lprintf(error, "fwrite(): encountered error!\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
end:
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking seek_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->seek_lock);
|
|
|
|
return byte_written;
|
2019-04-12 14:52:11 +02:00
|
|
|
}
|
2019-04-21 21:48:44 +02:00
|
|
|
|
2019-04-22 02:50:05 +02:00
|
|
|
int CacheDir_create(const char *dirn)
|
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *metadirn = path_append(META_DIR, dirn);
|
|
|
|
char *datadirn = path_append(DATA_DIR, dirn);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
i = -mkdir(metadirn, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
|
|
|
|
if (i && (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
|
|
|
|
i |= -mkdir(datadirn,
|
|
|
|
S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) << 1;
|
|
|
|
if (i && (errno != EEXIST)) {
|
|
|
|
lprintf(fatal, "mkdir(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
FREE(datadirn);
|
|
|
|
FREE(metadirn);
|
|
|
|
return -i;
|
2019-04-22 02:50:05 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief Allocate a new cache data structure
|
|
|
|
*/
|
2019-04-22 03:33:26 +02:00
|
|
|
static Cache *Cache_alloc()
|
2019-04-21 21:48:44 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
Cache *cf = CALLOC(1, sizeof(Cache));
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_init(&cf->seek_lock, NULL)) {
|
|
|
|
lprintf(fatal, "seek_lock initialisation failed!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_init(&cf->w_lock, NULL)) {
|
|
|
|
lprintf(fatal, "w_lock initialisation failed!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutexattr_init(&cf->bgt_lock_attr)) {
|
|
|
|
lprintf(fatal, "bgt_lock_attr initialisation failed!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutexattr_setpshared(&cf->bgt_lock_attr,
|
|
|
|
PTHREAD_PROCESS_SHARED)) {
|
|
|
|
lprintf(fatal, "could not set bgt_lock_attr!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_init(&cf->bgt_lock, &cf->bgt_lock_attr)) {
|
|
|
|
lprintf(fatal, "bgt_lock initialisation failed!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
return cf;
|
2019-04-21 21:48:44 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief free a cache data structure
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void Cache_free(Cache * cf)
|
2019-04-21 21:48:44 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_destroy(&cf->seek_lock)) {
|
|
|
|
lprintf(fatal, "could not destroy seek_lock!\n");
|
|
|
|
}
|
2019-04-23 20:37:01 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_destroy(&cf->w_lock)) {
|
|
|
|
lprintf(fatal, "could not destroy w_lock!\n");
|
|
|
|
}
|
2019-04-25 09:50:04 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutex_destroy(&cf->bgt_lock)) {
|
|
|
|
lprintf(fatal, "could not destroy bgt_lock!\n");
|
|
|
|
}
|
2019-04-25 00:58:26 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (pthread_mutexattr_destroy(&cf->bgt_lock_attr)) {
|
|
|
|
lprintf(fatal, "could not destroy bgt_lock_attr!\n");
|
|
|
|
}
|
2019-04-25 00:58:26 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (cf->path) {
|
|
|
|
FREE(cf->path);
|
|
|
|
}
|
2019-10-24 03:15:05 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (cf->seg) {
|
|
|
|
FREE(cf->seg);
|
|
|
|
}
|
2019-04-25 00:58:26 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (cf->fs_path) {
|
|
|
|
FREE(cf->fs_path);
|
|
|
|
}
|
2019-10-24 03:15:05 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
FREE(cf);
|
2019-04-21 21:48:44 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief Check if both metadata and data file exist, otherwise perform cleanup.
|
|
|
|
* \details
|
|
|
|
* This function checks if both metadata file and the data file exist. If that
|
|
|
|
* is not the case, clean up is performed - the existing unpaired metadata file
|
|
|
|
* or data file is deleted.
|
|
|
|
* \return
|
|
|
|
* - 0, if both metadata and cache file exist
|
|
|
|
* - -1, otherwise
|
|
|
|
*/
|
2019-04-22 03:33:26 +02:00
|
|
|
static int Cache_exist(const char *fn)
|
2019-04-21 21:48:44 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *metafn = path_append(META_DIR, fn);
|
|
|
|
char *datafn = path_append(DATA_DIR, fn);
|
|
|
|
/*
|
|
|
|
* access() returns 0 on success
|
|
|
|
*/
|
|
|
|
int no_meta = access(metafn, F_OK);
|
|
|
|
int no_data = access(datafn, F_OK);
|
|
|
|
|
|
|
|
if (no_meta ^ no_data) {
|
|
|
|
if (no_meta) {
|
|
|
|
lprintf(warning, "Cache file partially missing.\n");
|
|
|
|
if (unlink(datafn)) {
|
|
|
|
lprintf(error, "unlink(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (no_data) {
|
|
|
|
if (unlink(metafn)) {
|
|
|
|
lprintf(error, "unlink(): %s\n", strerror(errno));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
FREE(metafn);
|
|
|
|
FREE(datafn);
|
|
|
|
|
|
|
|
return no_meta | no_data;
|
2019-04-21 21:48:44 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief delete a cache file set
|
|
|
|
*/
|
2019-04-23 12:38:08 +02:00
|
|
|
void Cache_delete(const char *fn)
|
2019-04-21 21:48:44 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
if (CONFIG.mode == SONIC) {
|
|
|
|
Link *link = path_to_Link(fn);
|
2021-09-01 22:29:13 +02:00
|
|
|
fn = link->sonic.id;
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
char *metafn = path_append(META_DIR, fn);
|
|
|
|
char *datafn = path_append(DATA_DIR, fn);
|
|
|
|
if (!access(metafn, F_OK)) {
|
|
|
|
if (unlink(metafn)) {
|
|
|
|
lprintf(error, "unlink(): %s\n", strerror(errno));
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (!access(datafn, F_OK)) {
|
|
|
|
if (unlink(datafn)) {
|
|
|
|
lprintf(error, "unlink(): %s\n", strerror(errno));
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
FREE(metafn);
|
|
|
|
FREE(datafn);
|
2019-04-21 21:48:44 +02:00
|
|
|
}
|
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief Open the data file of a cache data set
|
|
|
|
* \return
|
|
|
|
* - 0 on success
|
|
|
|
* - -1 on failure, with appropriate errno set.
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static int Data_open(Cache * cf)
|
2019-04-22 22:20:41 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *datafn = path_append(DATA_DIR, cf->path);
|
|
|
|
cf->dfp = fopen(datafn, "r+");
|
|
|
|
if (!cf->dfp) {
|
|
|
|
/*
|
|
|
|
* Failed to open the data file
|
|
|
|
*/
|
|
|
|
lprintf(error, "fopen(%s): %s\n", datafn, strerror(errno));
|
2021-09-01 13:34:53 +02:00
|
|
|
FREE(datafn);
|
2021-08-31 12:18:39 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2021-09-01 13:34:53 +02:00
|
|
|
FREE(datafn);
|
2021-08-31 12:18:39 +02:00
|
|
|
return 0;
|
2019-04-22 22:20:41 +02:00
|
|
|
}
|
|
|
|
|
2019-04-24 04:00:47 +02:00
|
|
|
/**
|
|
|
|
* \brief Open a metafile
|
|
|
|
* \return
|
|
|
|
* - 0 on success
|
|
|
|
* - -1 on failure, with appropriate errno set.
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static int Meta_open(Cache * cf)
|
2019-04-23 00:29:59 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *metafn = path_append(META_DIR, cf->path);
|
|
|
|
cf->mfp = fopen(metafn, "r+");
|
|
|
|
if (!cf->mfp) {
|
|
|
|
/*
|
|
|
|
* Failed to open the data file
|
|
|
|
*/
|
|
|
|
lprintf(error, "fopen(%s): %s\n", metafn, strerror(errno));
|
2021-08-31 12:15:00 +02:00
|
|
|
FREE(metafn);
|
2021-08-31 12:18:39 +02:00
|
|
|
return -1;
|
|
|
|
}
|
2021-09-01 13:34:53 +02:00
|
|
|
FREE(metafn);
|
2021-08-31 12:18:39 +02:00
|
|
|
return 0;
|
2019-04-24 04:00:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* \brief Create a metafile
|
2021-08-30 06:17:15 +02:00
|
|
|
* \return exit on error
|
2019-04-24 04:00:47 +02:00
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void Meta_create(Cache * cf)
|
2019-04-24 04:00:47 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
char *metafn = path_append(META_DIR, cf->path);
|
|
|
|
cf->mfp = fopen(metafn, "w");
|
|
|
|
if (!cf->mfp) {
|
|
|
|
/*
|
|
|
|
* Failed to open the data file
|
|
|
|
*/
|
|
|
|
lprintf(fatal, "fopen(%s): %s\n", metafn, strerror(errno));
|
|
|
|
}
|
2021-09-01 13:19:20 +02:00
|
|
|
if (fclose(cf->mfp)) {
|
|
|
|
lprintf(error,
|
|
|
|
"cannot close metadata after creation: %s.\n",
|
|
|
|
strerror(errno));
|
|
|
|
}
|
2021-09-01 13:34:53 +02:00
|
|
|
FREE(metafn);
|
2019-04-24 04:00:47 +02:00
|
|
|
}
|
2019-04-23 00:29:59 +02:00
|
|
|
|
2019-10-24 03:15:05 +02:00
|
|
|
int Cache_create(const char *path)
|
2019-04-24 04:00:47 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
Link *this_link = path_to_Link(path);
|
|
|
|
|
2021-08-31 14:52:25 +02:00
|
|
|
char *fn = "__UNINITIALISED__";
|
2021-08-31 12:18:39 +02:00
|
|
|
if (CONFIG.mode == NORMAL) {
|
|
|
|
fn = curl_easy_unescape(NULL,
|
|
|
|
this_link->f_url + ROOT_LINK_OFFSET, 0,
|
|
|
|
NULL);
|
2021-08-31 14:52:25 +02:00
|
|
|
} else if (CONFIG.mode == SINGLE) {
|
2021-08-31 19:49:49 +02:00
|
|
|
fn = curl_easy_unescape(NULL, this_link->linkname, 0, NULL);
|
2021-08-31 12:18:39 +02:00
|
|
|
} else if (CONFIG.mode == SONIC) {
|
2021-09-01 22:29:13 +02:00
|
|
|
fn = this_link->sonic.id;
|
2021-08-31 12:18:39 +02:00
|
|
|
} else {
|
|
|
|
lprintf(fatal, "Invalid CONFIG.mode\n");
|
|
|
|
}
|
|
|
|
lprintf(debug, "Creating cache files for %s.\n", fn);
|
|
|
|
|
|
|
|
Cache *cf = Cache_alloc();
|
|
|
|
cf->path = strndup(fn, MAX_PATH_LEN);
|
|
|
|
cf->time = this_link->time;
|
|
|
|
cf->content_length = this_link->content_length;
|
|
|
|
cf->blksz = CONFIG.data_blksz;
|
|
|
|
cf->segbc = (cf->content_length / cf->blksz) + 1;
|
|
|
|
cf->seg = CALLOC(cf->segbc, sizeof(Seg));
|
|
|
|
|
|
|
|
Meta_create(cf);
|
|
|
|
|
|
|
|
if (Meta_open(cf)) {
|
|
|
|
Cache_free(cf);
|
|
|
|
lprintf(error, "cannot open metadata file, %s.\n", fn);
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (Meta_write(cf)) {
|
|
|
|
lprintf(error, "Meta_write() failed!\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (fclose(cf->mfp)) {
|
|
|
|
lprintf(error,
|
|
|
|
"cannot close metadata after write, %s.\n",
|
|
|
|
strerror(errno));
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
Data_create(cf);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
Cache_free(cf);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
int res = Cache_exist(fn);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (CONFIG.mode == NORMAL) {
|
|
|
|
curl_free(fn);
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
return res;
|
2019-04-23 00:29:59 +02:00
|
|
|
}
|
|
|
|
|
2019-04-21 21:48:44 +02:00
|
|
|
Cache *Cache_open(const char *fn)
|
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* Obtain the link structure memory pointer
|
|
|
|
*/
|
|
|
|
Link *link = path_to_Link(fn);
|
|
|
|
if (!link) {
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* There is no associated link to the path
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: locking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-09-01 11:38:45 +02:00
|
|
|
if (link->cache_ptr) {
|
|
|
|
link->cache_ptr->cache_opened++;
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return link->cache_ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if both metadata and data file exist
|
|
|
|
*/
|
2021-08-31 14:52:25 +02:00
|
|
|
if (CONFIG.mode == NORMAL || CONFIG.mode == SINGLE) {
|
2021-08-31 12:18:39 +02:00
|
|
|
if (Cache_exist(fn)) {
|
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
} else if (CONFIG.mode == SONIC) {
|
2021-09-01 22:29:13 +02:00
|
|
|
if (Cache_exist(link->sonic.id)) {
|
2021-08-31 12:18:39 +02:00
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
lprintf(fatal, "Invalid CONFIG.mode\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create the cache in-memory data structure
|
|
|
|
*/
|
|
|
|
Cache *cf = Cache_alloc();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Fill in the fs_path
|
|
|
|
*/
|
|
|
|
cf->fs_path = CALLOC(MAX_PATH_LEN + 1, sizeof(char));
|
|
|
|
strncpy(cf->fs_path, fn, MAX_PATH_LEN);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the path for the local cache file, if we are in sonic mode
|
|
|
|
*/
|
|
|
|
if (CONFIG.mode == SONIC) {
|
2021-09-01 22:29:13 +02:00
|
|
|
fn = link->sonic.id;
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
cf->path = strndup(fn, MAX_PATH_LEN);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Associate the cache structure with a link
|
|
|
|
*/
|
|
|
|
cf->link = link;
|
|
|
|
|
|
|
|
if (Meta_open(cf)) {
|
|
|
|
Cache_free(cf);
|
|
|
|
lprintf(error, "cannot open metadata file %s.\n", fn);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* Corrupt metadata
|
|
|
|
*/
|
|
|
|
if (Meta_read(cf)) {
|
|
|
|
Cache_free(cf);
|
|
|
|
lprintf(error, "metadata error: %s.\n", fn);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Inconsistency between metadata and data file, note that on disk file
|
|
|
|
* size might be bigger than content_length, due to on-disk filesystem
|
|
|
|
* allocation policy.
|
|
|
|
*/
|
|
|
|
if (cf->content_length > Data_size(fn)) {
|
|
|
|
lprintf(error, "metadata inconsistency %s, \
|
2021-08-30 12:24:32 +02:00
|
|
|
cf->content_length: %ld, Data_size(fn): %ld.\n", fn, cf->content_length, Data_size(fn));
|
2021-08-31 12:18:39 +02:00
|
|
|
Cache_free(cf);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check if the cache files are not outdated
|
|
|
|
*/
|
|
|
|
if (cf->time != cf->link->time) {
|
|
|
|
lprintf(warning, "outdated cache file: %s.\n", fn);
|
|
|
|
Cache_free(cf);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return NULL;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (Data_open(cf)) {
|
|
|
|
Cache_free(cf);
|
|
|
|
lprintf(error, "cannot open data file %s.\n", fn);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
2021-08-31 12:18:39 +02:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2021-09-01 11:38:45 +02:00
|
|
|
cf->cache_opened = 1;
|
2021-08-31 12:18:39 +02:00
|
|
|
/*
|
|
|
|
* Yup, we just created a circular loop. ;)
|
|
|
|
*/
|
|
|
|
cf->link->cache_ptr = cf;
|
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return cf;
|
2019-04-21 21:48:44 +02:00
|
|
|
}
|
2019-04-22 22:20:41 +02:00
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
void Cache_close(Cache * cf)
|
2019-04-22 22:20:41 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: locking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-09-01 11:38:45 +02:00
|
|
|
cf->cache_opened--;
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-09-01 11:38:45 +02:00
|
|
|
if (cf->cache_opened > 0) {
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return;
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (Meta_write(cf)) {
|
|
|
|
lprintf(error, "Meta_write() error.");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (fclose(cf->mfp)) {
|
|
|
|
lprintf(error, "cannot close metadata: %s.\n", strerror(errno));
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (fclose(cf->dfp)) {
|
|
|
|
lprintf(error, "cannot close data file %s.\n", strerror(errno));
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-09-02 17:24:55 +02:00
|
|
|
cf->link->cache_ptr = NULL;
|
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking cf_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf_lock);
|
|
|
|
return Cache_free(cf);
|
2019-04-22 22:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* \brief Check if a segment exists.
|
|
|
|
* \return 1 if the segment exists
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static int Seg_exist(Cache * cf, off_t offset)
|
2019-04-22 22:20:41 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
off_t byte = offset / cf->blksz;
|
|
|
|
return cf->seg[byte];
|
2019-04-22 22:20:41 +02:00
|
|
|
}
|
2019-09-02 10:04:20 +02:00
|
|
|
|
2019-04-22 22:20:41 +02:00
|
|
|
/**
|
|
|
|
* \brief Set the existence of a segment
|
2019-04-24 01:33:20 +02:00
|
|
|
* \param[in] cf the cache in-memory data structure
|
2019-04-22 22:20:41 +02:00
|
|
|
* \param[in] offset the starting position of the segment.
|
|
|
|
* \param[in] i 1 for exist, 0 for doesn't exist
|
|
|
|
* \note Call this after downloading a segment.
|
|
|
|
*/
|
2021-08-30 12:24:32 +02:00
|
|
|
static void Seg_set(Cache * cf, off_t offset, int i)
|
2019-04-22 22:20:41 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
off_t byte = offset / cf->blksz;
|
|
|
|
cf->seg[byte] = i;
|
2019-04-22 22:20:41 +02:00
|
|
|
}
|
|
|
|
|
2019-04-25 00:58:26 +02:00
|
|
|
/**
|
|
|
|
* \brief Background download function
|
|
|
|
* \details If we are requesting the data from the second half of the current
|
|
|
|
* segment, we can spawn a pthread using this function to download the next
|
|
|
|
* segment.
|
|
|
|
*/
|
2019-04-25 09:50:04 +02:00
|
|
|
static void *Cache_bgdl(void *arg)
|
2019-04-25 00:58:26 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
Cache *cf = (Cache *) arg;
|
|
|
|
|
|
|
|
lprintf(cache_lock_debug, "thread %x: locking w_lock;\n",
|
|
|
|
pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf->w_lock);
|
|
|
|
|
|
|
|
uint8_t *recv_buf = CALLOC(cf->blksz, sizeof(uint8_t));
|
|
|
|
lprintf(debug, "thread %x spawned.\n ", pthread_self());
|
2021-09-02 16:36:53 +02:00
|
|
|
long recv = Link_download(cf->link, (char *) recv_buf, cf->blksz,
|
2021-08-31 12:18:39 +02:00
|
|
|
cf->next_dl_offset);
|
|
|
|
if (recv < 0) {
|
|
|
|
lprintf(error, "thread %x received %ld bytes, \
|
2021-08-30 04:43:45 +02:00
|
|
|
which does't make sense\n", pthread_self(), recv);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if ((recv == cf->blksz) ||
|
|
|
|
(cf->next_dl_offset ==
|
|
|
|
(cf->content_length / cf->blksz * cf->blksz))) {
|
|
|
|
Data_write(cf, recv_buf, recv, cf->next_dl_offset);
|
|
|
|
Seg_set(cf, cf->next_dl_offset, 1);
|
|
|
|
} else {
|
|
|
|
lprintf(error, "received %ld rather than %ld, possible network \
|
2021-08-30 04:43:45 +02:00
|
|
|
error.\n", recv, cf->blksz);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-30 04:43:45 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
FREE(recv_buf);
|
2021-08-30 04:43:45 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking bgt_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->bgt_lock);
|
2021-08-30 04:43:45 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking w_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
|
2021-08-30 04:43:45 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
pthread_detach(pthread_self());
|
|
|
|
pthread_exit(NULL);
|
2019-04-25 00:58:26 +02:00
|
|
|
}
|
|
|
|
|
2021-08-30 12:24:32 +02:00
|
|
|
long
|
|
|
|
Cache_read(Cache * cf, char *const output_buf, const off_t len,
|
2021-08-31 12:15:00 +02:00
|
|
|
const off_t offset_start)
|
2019-04-22 22:20:41 +02:00
|
|
|
{
|
2021-08-31 12:18:39 +02:00
|
|
|
long send;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The offset of the segment to be downloaded
|
|
|
|
*/
|
|
|
|
off_t dl_offset = (offset_start + len) / cf->blksz * cf->blksz;
|
|
|
|
|
|
|
|
/*
|
2021-08-31 19:49:49 +02:00
|
|
|
* ------------- Check if the segment already exists --------------
|
2021-08-31 12:18:39 +02:00
|
|
|
*/
|
|
|
|
if (Seg_exist(cf, dl_offset)) {
|
|
|
|
send = Data_read(cf, (uint8_t *) output_buf, len, offset_start);
|
|
|
|
goto bgdl;
|
|
|
|
} else {
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* Wait for any other download thread to finish
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %ld: locking w_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_LOCK(&cf->w_lock);
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
if (Seg_exist(cf, dl_offset)) {
|
|
|
|
/*
|
|
|
|
* The segment now exists - it was downloaded by another
|
|
|
|
* download thread. Send it off and unlock the I/O
|
|
|
|
*/
|
|
|
|
send =
|
|
|
|
Data_read(cf, (uint8_t *) output_buf, len, offset_start);
|
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking w_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
|
|
|
|
|
|
|
|
goto bgdl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2021-08-31 19:49:49 +02:00
|
|
|
* ------------------ Download the segment ---------------------
|
2021-08-31 12:18:39 +02:00
|
|
|
*/
|
|
|
|
|
|
|
|
uint8_t *recv_buf = CALLOC(cf->blksz, sizeof(uint8_t));
|
|
|
|
lprintf(debug, "thread %x: spawned.\n ", pthread_self());
|
2021-09-02 16:36:53 +02:00
|
|
|
long recv = Link_download(cf->link, (char *) recv_buf, cf->blksz,
|
2021-08-31 12:18:39 +02:00
|
|
|
dl_offset);
|
|
|
|
if (recv < 0) {
|
|
|
|
lprintf(error, "thread %x received %ld bytes, \
|
2021-08-30 04:43:45 +02:00
|
|
|
which does't make sense\n", pthread_self(), recv);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* check if we have received enough data, write it to the disk
|
|
|
|
*
|
|
|
|
* Condition 1: received the exact amount as the segment size.
|
|
|
|
* Condition 2: offset is the last segment
|
|
|
|
*/
|
|
|
|
if ((recv == cf->blksz) ||
|
|
|
|
(dl_offset == (cf->content_length / cf->blksz * cf->blksz))) {
|
|
|
|
Data_write(cf, recv_buf, recv, dl_offset);
|
|
|
|
Seg_set(cf, dl_offset, 1);
|
|
|
|
} else {
|
|
|
|
lprintf(error, "received %ld rather than %ld, possible network \
|
2021-08-30 04:43:45 +02:00
|
|
|
error.\n", recv, cf->blksz);
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
|
|
|
FREE(recv_buf);
|
|
|
|
send = Data_read(cf, (uint8_t *) output_buf, len, offset_start);
|
|
|
|
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: unlocking w_lock;\n", pthread_self());
|
|
|
|
PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ----------- Download the next segment in background -----------------
|
|
|
|
*/
|
|
|
|
bgdl:
|
|
|
|
{
|
|
|
|
}
|
|
|
|
off_t next_dl_offset = round_div(offset_start, cf->blksz) * cf->blksz;
|
|
|
|
if ((next_dl_offset > dl_offset) && !Seg_exist(cf, next_dl_offset)
|
|
|
|
&& next_dl_offset < cf->content_length) {
|
2021-08-31 12:15:00 +02:00
|
|
|
/*
|
2021-08-31 12:18:39 +02:00
|
|
|
* Stop the spawning of multiple background pthreads
|
2021-08-31 12:15:00 +02:00
|
|
|
*/
|
2021-08-31 12:18:39 +02:00
|
|
|
if (!pthread_mutex_trylock(&cf->bgt_lock)) {
|
|
|
|
lprintf(cache_lock_debug,
|
|
|
|
"thread %x: trylocked bgt_lock;\n", pthread_self());
|
|
|
|
cf->next_dl_offset = next_dl_offset;
|
|
|
|
if (pthread_create(&cf->bgt, NULL, Cache_bgdl, cf)) {
|
|
|
|
lprintf(error,
|
|
|
|
"Error creating background download thread\n");
|
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
}
|
2021-08-31 12:18:39 +02:00
|
|
|
}
|
2021-08-31 12:15:00 +02:00
|
|
|
|
2021-08-31 12:18:39 +02:00
|
|
|
return send;
|
2019-04-22 22:20:41 +02:00
|
|
|
}
|