Changed the way segments work, added more functions
Now segments are fixed size block of 128KiB
This commit is contained in:
parent
3d0269e90f
commit
e5c77f3680
83
src/cache.c
83
src/cache.c
|
@ -10,6 +10,17 @@
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief Data file block size
|
||||||
|
* \details The data file block size is set to 128KiB, for convenience. This is
|
||||||
|
* because the maximum requested block size by FUSE seems to be 128KiB under
|
||||||
|
* Debian Stretch. Note that the minimum requested block size appears to be
|
||||||
|
* 4KiB.
|
||||||
|
*
|
||||||
|
* More information regarding block size can be found at:
|
||||||
|
* https://wiki.vuze.com/w/Torrent_Piece_Size
|
||||||
|
*/
|
||||||
|
#define DATA_BLK_SZ 131072
|
||||||
#define MAX_PATH_LEN 4096
|
#define MAX_PATH_LEN 4096
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -22,6 +33,7 @@ char *META_DIR;
|
||||||
*/
|
*/
|
||||||
char *DATA_DIR;
|
char *DATA_DIR;
|
||||||
|
|
||||||
|
|
||||||
void Cache_init(const char *dir)
|
void Cache_init(const char *dir)
|
||||||
{
|
{
|
||||||
META_DIR = strndupcat(dir, "meta/", MAX_PATH_LEN);
|
META_DIR = strndupcat(dir, "meta/", MAX_PATH_LEN);
|
||||||
|
@ -30,16 +42,11 @@ void Cache_init(const char *dir)
|
||||||
|
|
||||||
Cache *Cache_alloc()
|
Cache *Cache_alloc()
|
||||||
{
|
{
|
||||||
Cache *cf = malloc(sizeof(Cache));
|
Cache *cf = calloc(1, sizeof(Cache));
|
||||||
if (!cf) {
|
if (!cf) {
|
||||||
fprintf(stderr, "Cache_new(): malloc failure!\n");
|
fprintf(stderr, "Cache_new(): calloc failure!\n");
|
||||||
exit(EXIT_FAILURE);
|
exit(EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
cf->filename = NULL;
|
|
||||||
cf->len = 0;
|
|
||||||
cf->time = 0;
|
|
||||||
cf->nseg = 0;
|
|
||||||
cf->seg = NULL;
|
|
||||||
return cf;
|
return cf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -89,7 +96,7 @@ int Cache_exist(const char *fn)
|
||||||
free(metafn);
|
free(metafn);
|
||||||
free(datafn);
|
free(datafn);
|
||||||
|
|
||||||
return 1;
|
return !(meta_exists & data_exists);
|
||||||
}
|
}
|
||||||
|
|
||||||
Cache *Cache_create(const char *fn, long len, long time)
|
Cache *Cache_create(const char *fn, long len, long time)
|
||||||
|
@ -97,8 +104,8 @@ Cache *Cache_create(const char *fn, long len, long time)
|
||||||
Cache *cf = Cache_alloc();
|
Cache *cf = Cache_alloc();
|
||||||
|
|
||||||
cf->filename = strndup(fn, MAX_PATH_LEN);
|
cf->filename = strndup(fn, MAX_PATH_LEN);
|
||||||
cf->len = len;
|
|
||||||
cf->time = time;
|
cf->time = time;
|
||||||
|
cf->len = len;
|
||||||
|
|
||||||
if (Data_create(cf)) {
|
if (Data_create(cf)) {
|
||||||
Cache_free(cf);
|
Cache_free(cf);
|
||||||
|
@ -114,8 +121,34 @@ Cache *Cache_create(const char *fn, long len, long time)
|
||||||
return cf;
|
return cf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Cache_delete(const char *fn)
|
||||||
|
{
|
||||||
|
char *metafn = strndupcat(META_DIR, fn, MAX_PATH_LEN);
|
||||||
|
char *datafn = strndupcat(DATA_DIR, fn, MAX_PATH_LEN);
|
||||||
|
if (!access(metafn, F_OK)) {
|
||||||
|
if(unlink(metafn)) {
|
||||||
|
fprintf(stderr, "Cache_delete(): unlink(): %s\n",
|
||||||
|
strerror(errno));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!access(datafn, F_OK)) {
|
||||||
|
if(unlink(datafn)) {
|
||||||
|
fprintf(stderr, "Cache_delete(): unlink(): %s\n",
|
||||||
|
strerror(errno));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(metafn);
|
||||||
|
free(datafn);
|
||||||
|
}
|
||||||
|
|
||||||
Cache *Cache_open(const char *fn)
|
Cache *Cache_open(const char *fn)
|
||||||
{
|
{
|
||||||
|
/* Check if both metadata and data file exist */
|
||||||
|
if (Cache_validate(fn)) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
/* Create the cache in-memory data structure */
|
/* Create the cache in-memory data structure */
|
||||||
Cache *cf = Cache_alloc();
|
Cache *cf = Cache_alloc();
|
||||||
cf->filename = strndup(fn, MAX_PATH_LEN);
|
cf->filename = strndup(fn, MAX_PATH_LEN);
|
||||||
|
@ -125,17 +158,27 @@ Cache *Cache_open(const char *fn)
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* In consistent metadata / data file */
|
||||||
if (cf->len != Data_size(fn)) {
|
if (cf->len != Data_size(fn)) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
"Cache_open(): metadata is inconsistent with the data file!\n");
|
"Cache_open(): metadata is inconsistent with the data file!\n");
|
||||||
|
Cache_delete(fn);
|
||||||
Cache_free(cf);
|
Cache_free(cf);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
return cf;
|
return cf;
|
||||||
}
|
}
|
||||||
|
|
||||||
int Meta_create(const Cache *cf)
|
int Meta_create(const Cache *cf)
|
||||||
{
|
{
|
||||||
|
cf->blksz = DATA_BLK_SZ;
|
||||||
|
cf->nseg = cf->len / cf->blksz + 1;
|
||||||
|
cf->seg = calloc(nseg, sizeof(Seg));
|
||||||
|
if (!(cf->seg)) {
|
||||||
|
fprintf(stderr, "Meta_create(): calloc failure!\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
return Meta_write(cf);
|
return Meta_write(cf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,11 +197,11 @@ int Meta_read(Cache *cf)
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
fread(&(cf->len), sizeof(long), 1, fp);
|
|
||||||
fread(&(cf->time), sizeof(long), 1, fp);
|
fread(&(cf->time), sizeof(long), 1, fp);
|
||||||
|
fread(&(cf->len), sizeof(long), 1, fp);
|
||||||
|
fread(&(cf->len), sizeof(int), 1, fp);
|
||||||
fread(&(cf->nseg), sizeof(int), 1, fp);
|
fread(&(cf->nseg), sizeof(int), 1, fp);
|
||||||
|
|
||||||
if (cf->nseg) {
|
|
||||||
/* Allocate some memory for the segment */
|
/* Allocate some memory for the segment */
|
||||||
cf->seg = malloc(cf->nseg * sizeof(Seg));
|
cf->seg = malloc(cf->nseg * sizeof(Seg));
|
||||||
if (!(cf->seg)) {
|
if (!(cf->seg)) {
|
||||||
|
@ -167,7 +210,6 @@ int Meta_read(Cache *cf)
|
||||||
}
|
}
|
||||||
/* Read all the segment */
|
/* Read all the segment */
|
||||||
nmemb = fread(cf->seg, sizeof(Seg), cf->nseg, fp);
|
nmemb = fread(cf->seg, sizeof(Seg), cf->nseg, fp);
|
||||||
}
|
|
||||||
|
|
||||||
/* Error checking for fread */
|
/* Error checking for fread */
|
||||||
if (ferror(fp)) {
|
if (ferror(fp)) {
|
||||||
|
@ -204,14 +246,11 @@ int Meta_write(const Cache *cf)
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
fwrite(&(cf->len), sizeof(long), 1, fp);
|
|
||||||
fwrite(&(cf->time), sizeof(long), 1, fp);
|
fwrite(&(cf->time), sizeof(long), 1, fp);
|
||||||
|
fwrite(&(cf->len), sizeof(long), 1, fp);
|
||||||
|
fwrite(&(cf->blksz), sizeof(int), 1, fp);
|
||||||
fwrite(&(cf->nseg), sizeof(int), 1, fp);
|
fwrite(&(cf->nseg), sizeof(int), 1, fp);
|
||||||
|
|
||||||
/* Finally write segments to the file */
|
|
||||||
if (cf->nseg) {
|
|
||||||
fwrite(cf->seg, sizeof(Seg), cf->nseg, fp);
|
fwrite(cf->seg, sizeof(Seg), cf->nseg, fp);
|
||||||
}
|
|
||||||
|
|
||||||
/* Error checking for fwrite */
|
/* Error checking for fwrite */
|
||||||
if (ferror(fp)) {
|
if (ferror(fp)) {
|
||||||
|
@ -239,7 +278,7 @@ int Data_create(Cache *cf)
|
||||||
fprintf(stderr, "Data_create(): open(): %s\n", strerror(errno));
|
fprintf(stderr, "Data_create(): open(): %s\n", strerror(errno));
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (ftruncate(fd, cf->len) == -1) {
|
if (ftruncate(fd, cf->len)) {
|
||||||
fprintf(stderr, "Data_create(): ftruncate(): %s\n", strerror(errno));
|
fprintf(stderr, "Data_create(): ftruncate(): %s\n", strerror(errno));
|
||||||
}
|
}
|
||||||
if (close(fd)) {
|
if (close(fd)) {
|
||||||
|
@ -252,11 +291,11 @@ long Data_size(const char *fn)
|
||||||
{
|
{
|
||||||
char *datafn = strndupcat(DATA_DIR, fn, MAX_PATH_LEN);
|
char *datafn = strndupcat(DATA_DIR, fn, MAX_PATH_LEN);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (stat(datafn, &st) == 0) {
|
int s = stat(datafn, &st);
|
||||||
free(datafn);
|
free(datafn);
|
||||||
|
if (!s) {
|
||||||
return st.st_blksize;
|
return st.st_blksize;
|
||||||
}
|
}
|
||||||
free(datafn);
|
|
||||||
fprintf(stderr, "Data_size(): stat(): %s\n", strerror(errno));
|
fprintf(stderr, "Data_size(): stat(): %s\n", strerror(errno));
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
@ -281,7 +320,7 @@ long Data_read(const Cache *cf, long offset, long len,
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fseek(fp, offset, SEEK_SET) == -1) {
|
if (fseek(fp, offset, SEEK_SET)) {
|
||||||
/* fseek failed */
|
/* fseek failed */
|
||||||
fprintf(stderr, "Data_read(): fseek(): %s\n", strerror(errno));
|
fprintf(stderr, "Data_read(): fseek(): %s\n", strerror(errno));
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
@ -331,7 +370,7 @@ long Data_write(const Cache *cf, long offset, long len,
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fseek(fp, offset, SEEK_SET) == -1) {
|
if (fseek(fp, offset, SEEK_SET)) {
|
||||||
/* fseek failed */
|
/* fseek failed */
|
||||||
fprintf(stderr, "Data_write(): fseek(): %s\n", strerror(errno));
|
fprintf(stderr, "Data_write(): fseek(): %s\n", strerror(errno));
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
44
src/cache.h
44
src/cache.h
|
@ -21,26 +21,22 @@
|
||||||
* - We are using 'long' to store file size, because the offset in fseek() is
|
* - We are using 'long' to store file size, because the offset in fseek() is
|
||||||
* in long, because the way we use the cache system, you cannot seek past
|
* in long, because the way we use the cache system, you cannot seek past
|
||||||
* long. So the biggest file size has to be able to be stored in long. This
|
* long. So the biggest file size has to be able to be stored in long. This
|
||||||
* makes this program architecturally dependent, but this is due to the
|
* makes this program architecturally dependent, i.e. i386 vs amd64
|
||||||
* dependency to fseek().
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief a cache segment
|
* \brief Type definition for a cache segment
|
||||||
*/
|
*/
|
||||||
typedef struct {
|
typedef uint8_t Seg;
|
||||||
long start;
|
|
||||||
long end;
|
|
||||||
} Seg;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief cache in-memory data structure
|
* \brief cache in-memory data structure
|
||||||
* \note fanf2@cam.ac.uk told me to use an array rather than linked list!
|
|
||||||
*/
|
*/
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char *filename; /**< the filename from the http server */
|
char *filename; /**< the filename from the http server */
|
||||||
long len; /**<the size of the file */
|
|
||||||
long time; /**<the modified time of the file */
|
long time; /**<the modified time of the file */
|
||||||
|
long len; /**<the size of the file */
|
||||||
|
int blksz; /**<the block size of the data file */
|
||||||
int nseg; /**<the number of segments */
|
int nseg; /**<the number of segments */
|
||||||
Seg *seg; /**< the detail of each segment */
|
Seg *seg; /**< the detail of each segment */
|
||||||
} Cache;
|
} Cache;
|
||||||
|
@ -63,6 +59,17 @@ long Cache_read(const char *fn, long offset, long len, uint8_t *buf);
|
||||||
*/
|
*/
|
||||||
long Cache_write(const char *fn, long offset, long len,
|
long Cache_write(const char *fn, long offset, long len,
|
||||||
const uint8_t *buf);
|
const uint8_t *buf);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief Initialise cache directory structure
|
||||||
|
*/
|
||||||
|
int Cache_DirInit(const char *fn);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief Create a directory within the cache structure
|
||||||
|
*/
|
||||||
|
int Cache_DirCreate(const char *fn);
|
||||||
|
|
||||||
/****************************** Work in progress *****************************/
|
/****************************** Work in progress *****************************/
|
||||||
|
|
||||||
|
|
||||||
|
@ -88,14 +95,14 @@ Cache *Cache_alloc();
|
||||||
void Cache_free(Cache *cf);
|
void Cache_free(Cache *cf);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief Check if both metadata and the data file exist, and perform cleanup.
|
* \brief Check if both metadata and data file exist, otherwise perform cleanup.
|
||||||
* \details
|
* \details
|
||||||
* This function checks if both metadata file and the data file exist. If that
|
* This function checks if both metadata file and the data file exist. If that
|
||||||
* is not the case, clean up is performed - the existing unpaired metadata file
|
* is not the case, clean up is performed - the existing unpaired metadata file
|
||||||
* or data file is deleted.
|
* or data file is deleted.
|
||||||
* \return
|
* \return
|
||||||
* - 1, if both metadata and cache file exist
|
* - 0, if both metadata and cache file exist
|
||||||
* - 0, otherwise
|
* - -1, otherwise
|
||||||
*/
|
*/
|
||||||
int Cache_exist(const char *fn);
|
int Cache_exist(const char *fn);
|
||||||
|
|
||||||
|
@ -104,14 +111,25 @@ int Cache_exist(const char *fn);
|
||||||
*/
|
*/
|
||||||
Cache *Cache_create(const char *fn, long len, long time);
|
Cache *Cache_create(const char *fn, long len, long time);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief delete a cache file set
|
||||||
|
*/
|
||||||
|
void Cache_delete(const char *fn);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief open a cache file set
|
* \brief open a cache file set
|
||||||
* \warning We assume that the metadata file and the data file both exist
|
|
||||||
*/
|
*/
|
||||||
Cache *Cache_open(const char *fn);
|
Cache *Cache_open(const char *fn);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* \brief create a metadata file
|
* \brief create a metadata file
|
||||||
|
* \details We set the followings here:
|
||||||
|
* - block size
|
||||||
|
* - the number of segments
|
||||||
|
*
|
||||||
|
* The number of segments depends on the block size. The block size is set to
|
||||||
|
* 128KiB for now. In future support for different block size may be
|
||||||
|
* implemented.
|
||||||
*/
|
*/
|
||||||
int Meta_create(const Cache *cf);
|
int Meta_create(const Cache *cf);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue