1 #include "cache.h"
2 #include "config.h"
3 #include "log.h"
4
5 #include <sys/stat.h>
6
7 #include <dirent.h>
8 #include <errno.h>
9 #include <fcntl.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13
14 /*
15 * ---------------- External variables -----------------------
16 */
17 int CACHE_SYSTEM_INIT = 0;
18 char *META_DIR;
19
20 /*
21 * ----------------- Static variables -----------------------
22 */
23
24 /**
25 * \brief Cache file locking
26 * \details Ensure cache opening and cache closing is an atomic operation
27 */
28 static pthread_mutex_t cf_lock;
29
30 /**
31 * \brief The data directory
32 */
33 static char *DATA_DIR;
34
35 /**
36 * \brief Calculate cache system directory path
37 */
CacheSystem_calc_dir(const char * url)38 static char *CacheSystem_calc_dir(const char *url)
39 {
40 char *xdg_cache_home = getenv("XDG_CACHE_HOME");
41 if (!xdg_cache_home) {
42 char *home = getenv("HOME");
43 char *xdg_cache_home_default = "/.cache";
44 xdg_cache_home = path_append(home, xdg_cache_home_default);
45 }
46 if (mkdir
47 (xdg_cache_home, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
48 && (errno != EEXIST)) {
49 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
50 }
51 char *cache_dir_root = path_append(xdg_cache_home, "/httpdirfs/");
52 if (mkdir
53 (cache_dir_root, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
54 && (errno != EEXIST)) {
55 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
56 }
57
58 char *fn = path_append(cache_dir_root, "/CACHEDIR.TAG");
59 FILE *fp = fopen(fn, "w");
60 if (fp) {
61 fprintf(fp, "Signature: 8a477f597d28d172789f06886806bc55\n\
62 # This file is a cache directory tag created by httpdirfs.\n\
63 # For information about cache directory tags, see:\n\
64 # http://www.brynosaurus.com/cachedir/\n");
65 } else {
66 lprintf(fatal, "fopen(%s): %s", fn, strerror(errno));
67 }
68 if (ferror(fp)) {
69 lprintf(fatal, "fwrite(): encountered error!\n");
70 }
71 if (fclose(fp)) {
72 lprintf(fatal, "fclose(%s): %s\n", fn, strerror(errno));
73 }
74 CURL *c = curl_easy_init();
75 char *escaped_url = curl_easy_escape(c, url, 0);
76 char *full_path = path_append(cache_dir_root, escaped_url);
77 if (mkdir(full_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
78 && (errno != EEXIST)) {
79 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
80 }
81 FREE(fn);
82 FREE(cache_dir_root);
83 curl_free(escaped_url);
84 curl_easy_cleanup(c);
85 return full_path;
86 }
87
CacheSystem_init(const char * path,int url_supplied)88 void CacheSystem_init(const char *path, int url_supplied)
89 {
90 if (pthread_mutex_init(&cf_lock, NULL)) {
91 lprintf(fatal, "cf_lock initialisation failed!\n");
92 }
93
94 if (url_supplied) {
95 path = CacheSystem_calc_dir(path);
96 }
97
98 lprintf(debug, "%s\n", path);
99
100 META_DIR = path_append(path, "meta/");
101 DATA_DIR = path_append(path, "data/");
102 /*
103 * Check if directories exist, if not, create them
104 */
105 if (mkdir(META_DIR, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
106 && (errno != EEXIST)) {
107 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
108 }
109
110 if (mkdir(DATA_DIR, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
111 && (errno != EEXIST)) {
112 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
113 }
114
115 if (CONFIG.mode == SONIC) {
116 char *sonic_path;
117 /*
118 * Create "rest" sub-directory for META_DIR
119 */
120 sonic_path = path_append(META_DIR, "rest/");
121 if (mkdir
122 (sonic_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
123 && (errno != EEXIST)) {
124 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
125 }
126 FREE(sonic_path);
127
128 /*
129 * Create "rest" sub-directory for DATA_DIR
130 */
131 sonic_path = path_append(DATA_DIR, "rest/");
132 if (mkdir
133 (sonic_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
134 && (errno != EEXIST)) {
135 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
136 }
137 FREE(sonic_path);
138 }
139
140 CACHE_SYSTEM_INIT = 1;
141 }
142
143 /**
144 * \brief read a metadata file
145 * \return 0 on success, errno on error.
146 */
Meta_read(Cache * cf)147 static int Meta_read(Cache * cf)
148 {
149 FILE *fp = cf->mfp;
150 rewind(fp);
151
152 int nmemb = 0;
153
154 if (!fp) {
155 /*
156 * The metadata file does not exist
157 */
158 lprintf(error, "fopen(): %s\n", strerror(errno));
159 return EIO;
160 }
161
162 fread(&cf->time, sizeof(long), 1, fp);
163 fread(&cf->content_length, sizeof(off_t), 1, fp);
164 fread(&cf->blksz, sizeof(int), 1, fp);
165 fread(&cf->segbc, sizeof(long), 1, fp);
166
167 /*
168 * Error checking for fread
169 */
170 if (ferror(fp)) {
171 lprintf(error, "error reading core metadata!\n");
172 return EIO;
173 }
174
175 /* These things really should not be zero!!! */
176 if (!cf->content_length || !cf->blksz || !cf->segbc) {
177 lprintf(error,
178 "corruption: content_length: %ld, blksz: %d, segbc: %ld\n",
179 cf->content_length, cf->blksz, cf->segbc);
180 return EBADMSG;
181 }
182
183 if (cf->blksz != CONFIG.data_blksz) {
184 lprintf(warning, "Warning: cf->blksz != CONFIG.data_blksz\n");
185 }
186
187 if (cf->segbc > CONFIG.max_segbc) {
188 lprintf(error, "Error: segbc: %ld\n", cf->segbc);
189 return EFBIG;
190 }
191
192 /*
193 * Allocate memory for all segments, and read them in
194 */
195 cf->seg = CALLOC(cf->segbc, sizeof(Seg));
196 nmemb = fread(cf->seg, sizeof(Seg), cf->segbc, fp);
197
198 /*
199 * We shouldn't have gone past the end of the file
200 */
201 if (feof(fp)) {
202 /*
203 * reached EOF
204 */
205 lprintf(error, "attempted to read past the end of the \
206 file!\n");
207 return EBADMSG;
208 }
209
210 /*
211 * Error checking for fread
212 */
213 if (ferror(fp)) {
214 lprintf(error, "error reading bitmap!\n");
215 return EIO;
216 }
217
218 /*
219 * Check for inconsistent metadata file
220 */
221 if (nmemb != cf->segbc) {
222 lprintf(error, "corrupted metadata!\n");
223 return EBADMSG;
224 }
225
226 return 0;
227 }
228
229 /**
230 * \brief write a metadata file
231 * \return
232 * - -1 on error,
233 * - 0 on success
234 */
Meta_write(Cache * cf)235 static int Meta_write(Cache * cf)
236 {
237 FILE *fp = cf->mfp;
238 rewind(fp);
239
240 if (!fp) {
241 /*
242 * Cannot create the metadata file
243 */
244 lprintf(error, "fopen(): %s\n", strerror(errno));
245 return -1;
246 }
247
248 /*
249 * These things really should not be zero!!!
250 */
251 if (!cf->content_length || !cf->blksz || !cf->segbc) {
252 lprintf(error, "content_length: %ld, blksz: %d, segbc: %ld\n",
253 cf->content_length, cf->blksz, cf->segbc);
254 }
255
256 fwrite(&cf->time, sizeof(long), 1, fp);
257 fwrite(&cf->content_length, sizeof(off_t), 1, fp);
258 fwrite(&cf->blksz, sizeof(int), 1, fp);
259 fwrite(&cf->segbc, sizeof(long), 1, fp);
260 fwrite(cf->seg, sizeof(Seg), cf->segbc, fp);
261
262 /*
263 * Error checking for fwrite
264 */
265 if (ferror(fp)) {
266 lprintf(error, "fwrite(): encountered error!\n");
267 return -1;
268 }
269
270 return 0;
271 }
272
273 /**
274 * \brief create a data file
275 * \details We use sparse creation here
276 * \return exit on failure
277 */
Data_create(Cache * cf)278 static void Data_create(Cache * cf)
279 {
280 int fd;
281 int mode;
282
283 mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
284 char *datafn = path_append(DATA_DIR, cf->path);
285 fd = open(datafn, O_WRONLY | O_CREAT, mode);
286 FREE(datafn);
287 if (fd == -1) {
288 lprintf(fatal, "open(): %s\n", strerror(errno));
289 }
290 if (ftruncate(fd, cf->content_length)) {
291 lprintf(warning, "ftruncate(): %s\n", strerror(errno));
292 }
293 if (close(fd)) {
294 lprintf(fatal, "close:(): %s\n", strerror(errno));
295 }
296 }
297
298 /**
299 * \brief obtain the data file size
300 * \return file size on success, -1 on error
301 */
Data_size(const char * fn)302 static long Data_size(const char *fn)
303 {
304 char *datafn = path_append(DATA_DIR, fn);
305 struct stat st;
306 int s = stat(datafn, &st);
307 FREE(datafn);
308 if (!s) {
309 return st.st_size;
310 }
311 lprintf(error, "stat(): %s\n", strerror(errno));
312 return -1;
313 }
314
315 /**
316 * \brief read a data file
317 * \param[in] cf the pointer to the cache in-memory data structure
318 * \param[out] buf the output buffer
319 * \param[in] len the length of the segment
320 * \param[in] offset the offset of the segment
321 * \return
322 * - negative values on error,
323 * - otherwise, the number of bytes read.
324 */
Data_read(Cache * cf,uint8_t * buf,off_t len,off_t offset)325 static long Data_read(Cache * cf, uint8_t * buf, off_t len, off_t offset)
326 {
327 if (len == 0) {
328 lprintf(error, "requested to read 0 byte!\n");
329 return -EINVAL;
330 }
331
332 lprintf(cache_lock_debug,
333 "thread %x: locking seek_lock;\n", pthread_self());
334 PTHREAD_MUTEX_LOCK(&cf->seek_lock);
335
336 long byte_read = 0;
337
338 /*
339 * Seek to the right location
340 */
341 if (fseeko(cf->dfp, offset, SEEK_SET)) {
342 /*
343 * fseeko failed
344 */
345 lprintf(error, "fseeko(): %s\n", strerror(errno));
346 byte_read = -EIO;
347 goto end;
348 }
349
350 /*
351 * Calculate how much to read
352 */
353 if (offset + len > cf->content_length) {
354 len -= offset + len - cf->content_length;
355 if (len < 0) {
356 goto end;
357 }
358 }
359
360 byte_read = fread(buf, sizeof(uint8_t), len, cf->dfp);
361 if (byte_read != len) {
362 lprintf(debug,
363 "fread(): requested %ld, returned %ld!\n", len, byte_read);
364 if (feof(cf->dfp)) {
365 /*
366 * reached EOF
367 */
368 lprintf(error, "fread(): reached the end of the file!\n");
369 }
370 if (ferror(cf->dfp)) {
371 /*
372 * filesystem error
373 */
374 lprintf(error, "fread(): encountered error!\n");
375 }
376 }
377
378 end:
379
380 lprintf(cache_lock_debug,
381 "thread %x: unlocking seek_lock;\n", pthread_self());
382 PTHREAD_MUTEX_UNLOCK(&cf->seek_lock);
383 return byte_read;
384 }
385
386 /**
387 * \brief write to a data file
388 * \param[in] cf the pointer to the cache in-memory data structure
389 * \param[in] buf the input buffer
390 * \param[in] len the length of the segment
391 * \param[in] offset the offset of the segment
392 * \return
393 * - -1 when the data file does not exist
394 * - otherwise, the number of bytes written.
395 */
Data_write(Cache * cf,const uint8_t * buf,off_t len,off_t offset)396 static long Data_write(Cache * cf, const uint8_t * buf, off_t len,
397 off_t offset)
398 {
399 if (len == 0) {
400 /*
401 * We should permit empty files
402 */
403 return 0;
404 }
405
406 lprintf(cache_lock_debug,
407 "thread %x: locking seek_lock;\n", pthread_self());
408 PTHREAD_MUTEX_LOCK(&cf->seek_lock);
409
410 long byte_written = 0;
411
412 if (fseeko(cf->dfp, offset, SEEK_SET)) {
413 /*
414 * fseeko failed
415 */
416 lprintf(error, "fseeko(): %s\n", strerror(errno));
417 byte_written = -EIO;
418 goto end;
419 }
420
421 byte_written = fwrite(buf, sizeof(uint8_t), len, cf->dfp);
422
423 if (byte_written != len) {
424 lprintf(error,
425 "fwrite(): requested %ld, returned %ld!\n",
426 len, byte_written);
427 }
428
429 if (ferror(cf->dfp)) {
430 /*
431 * filesystem error
432 */
433 lprintf(error, "fwrite(): encountered error!\n");
434 }
435
436 end:
437 lprintf(cache_lock_debug,
438 "thread %x: unlocking seek_lock;\n", pthread_self());
439 PTHREAD_MUTEX_UNLOCK(&cf->seek_lock);
440 return byte_written;
441 }
442
CacheDir_create(const char * dirn)443 int CacheDir_create(const char *dirn)
444 {
445 char *metadirn = path_append(META_DIR, dirn);
446 char *datadirn = path_append(DATA_DIR, dirn);
447 int i;
448
449 i = -mkdir(metadirn, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
450 if (i && (errno != EEXIST)) {
451 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
452 }
453
454 i |= -mkdir(datadirn,
455 S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) << 1;
456 if (i && (errno != EEXIST)) {
457 lprintf(fatal, "mkdir(): %s\n", strerror(errno));
458 }
459 FREE(datadirn);
460 FREE(metadirn);
461 return -i;
462 }
463
464 /**
465 * \brief Allocate a new cache data structure
466 */
Cache_alloc()467 static Cache *Cache_alloc()
468 {
469 Cache *cf = CALLOC(1, sizeof(Cache));
470
471 if (pthread_mutex_init(&cf->seek_lock, NULL)) {
472 lprintf(fatal, "seek_lock initialisation failed!\n");
473 }
474
475 if (pthread_mutex_init(&cf->w_lock, NULL)) {
476 lprintf(fatal, "w_lock initialisation failed!\n");
477 }
478
479 if (pthread_mutexattr_init(&cf->bgt_lock_attr)) {
480 lprintf(fatal, "bgt_lock_attr initialisation failed!\n");
481 }
482
483 if (pthread_mutexattr_setpshared(&cf->bgt_lock_attr,
484 PTHREAD_PROCESS_SHARED)) {
485 lprintf(fatal, "could not set bgt_lock_attr!\n");
486 }
487
488 if (pthread_mutex_init(&cf->bgt_lock, &cf->bgt_lock_attr)) {
489 lprintf(fatal, "bgt_lock initialisation failed!\n");
490 }
491
492 return cf;
493 }
494
495 /**
496 * \brief free a cache data structure
497 */
Cache_free(Cache * cf)498 static void Cache_free(Cache * cf)
499 {
500 if (pthread_mutex_destroy(&cf->seek_lock)) {
501 lprintf(fatal, "could not destroy seek_lock!\n");
502 }
503
504 if (pthread_mutex_destroy(&cf->w_lock)) {
505 lprintf(fatal, "could not destroy w_lock!\n");
506 }
507
508 if (pthread_mutex_destroy(&cf->bgt_lock)) {
509 lprintf(fatal, "could not destroy bgt_lock!\n");
510 }
511
512 if (pthread_mutexattr_destroy(&cf->bgt_lock_attr)) {
513 lprintf(fatal, "could not destroy bgt_lock_attr!\n");
514 }
515
516 if (cf->path) {
517 FREE(cf->path);
518 }
519
520 if (cf->seg) {
521 FREE(cf->seg);
522 }
523
524 if (cf->fs_path) {
525 FREE(cf->fs_path);
526 }
527
528 FREE(cf);
529 }
530
531 /**
532 * \brief Check if both metadata and data file exist, otherwise perform cleanup.
533 * \details
534 * This function checks if both metadata file and the data file exist. If that
535 * is not the case, clean up is performed - the existing unpaired metadata file
536 * or data file is deleted.
537 * \return
538 * - 0, if both metadata and cache file exist
539 * - -1, otherwise
540 */
Cache_exist(const char * fn)541 static int Cache_exist(const char *fn)
542 {
543 char *metafn = path_append(META_DIR, fn);
544 char *datafn = path_append(DATA_DIR, fn);
545 /*
546 * access() returns 0 on success
547 */
548 int no_meta = access(metafn, F_OK);
549 int no_data = access(datafn, F_OK);
550
551 if (no_meta ^ no_data) {
552 if (no_meta) {
553 lprintf(warning, "Cache file partially missing.\n");
554 if (unlink(datafn)) {
555 lprintf(error, "unlink(): %s\n", strerror(errno));
556 }
557 }
558 if (no_data) {
559 if (unlink(metafn)) {
560 lprintf(error, "unlink(): %s\n", strerror(errno));
561 }
562 }
563 }
564
565 FREE(metafn);
566 FREE(datafn);
567
568 return no_meta | no_data;
569 }
570
571 /**
572 * \brief delete a cache file set
573 */
Cache_delete(const char * fn)574 void Cache_delete(const char *fn)
575 {
576 if (CONFIG.mode == SONIC) {
577 Link *link = path_to_Link(fn);
578 fn = link->sonic_id;
579 }
580
581 char *metafn = path_append(META_DIR, fn);
582 char *datafn = path_append(DATA_DIR, fn);
583 if (!access(metafn, F_OK)) {
584 if (unlink(metafn)) {
585 lprintf(error, "unlink(): %s\n", strerror(errno));
586 }
587 }
588
589 if (!access(datafn, F_OK)) {
590 if (unlink(datafn)) {
591 lprintf(error, "unlink(): %s\n", strerror(errno));
592 }
593 }
594 FREE(metafn);
595 FREE(datafn);
596 }
597
598 /**
599 * \brief Open the data file of a cache data set
600 * \return
601 * - 0 on success
602 * - -1 on failure, with appropriate errno set.
603 */
Data_open(Cache * cf)604 static int Data_open(Cache * cf)
605 {
606 char *datafn = path_append(DATA_DIR, cf->path);
607 cf->dfp = fopen(datafn, "r+");
608 FREE(datafn);
609 if (!cf->dfp) {
610 /*
611 * Failed to open the data file
612 */
613 lprintf(error, "fopen(%s): %s\n", datafn, strerror(errno));
614 return -1;
615 }
616 return 0;
617 }
618
619 /**
620 * \brief Open a metafile
621 * \return
622 * - 0 on success
623 * - -1 on failure, with appropriate errno set.
624 */
Meta_open(Cache * cf)625 static int Meta_open(Cache * cf)
626 {
627 char *metafn = path_append(META_DIR, cf->path);
628 cf->mfp = fopen(metafn, "r+");
629 if (!cf->mfp) {
630 /*
631 * Failed to open the data file
632 */
633 lprintf(error, "fopen(%s): %s\n", metafn, strerror(errno));
634 FREE(metafn);
635 return -1;
636 }
637 FREE(metafn);
638 return 0;
639 }
640
641 /**
642 * \brief Create a metafile
643 * \return exit on error
644 */
Meta_create(Cache * cf)645 static void Meta_create(Cache * cf)
646 {
647 char *metafn = path_append(META_DIR, cf->path);
648 cf->mfp = fopen(metafn, "w");
649 if (!cf->mfp) {
650 /*
651 * Failed to open the data file
652 */
653 lprintf(fatal, "fopen(%s): %s\n", metafn, strerror(errno));
654 }
655 FREE(metafn);
656 }
657
Cache_create(const char * path)658 int Cache_create(const char *path)
659 {
660 Link *this_link = path_to_Link(path);
661
662 char *fn = "__UNINITIALISED__";
663 if (CONFIG.mode == NORMAL) {
664 fn = curl_easy_unescape(NULL,
665 this_link->f_url + ROOT_LINK_OFFSET, 0,
666 NULL);
667 } else if (CONFIG.mode == SINGLE) {
668 fn = curl_easy_unescape(NULL, this_link->linkname, 0, NULL);
669 } else if (CONFIG.mode == SONIC) {
670 fn = this_link->sonic_id;
671 } else {
672 lprintf(fatal, "Invalid CONFIG.mode\n");
673 }
674 lprintf(debug, "Creating cache files for %s.\n", fn);
675
676 Cache *cf = Cache_alloc();
677 cf->path = strndup(fn, MAX_PATH_LEN);
678 cf->time = this_link->time;
679 cf->content_length = this_link->content_length;
680 cf->blksz = CONFIG.data_blksz;
681 cf->segbc = (cf->content_length / cf->blksz) + 1;
682 cf->seg = CALLOC(cf->segbc, sizeof(Seg));
683
684 Meta_create(cf);
685
686 if (fclose(cf->mfp)) {
687 lprintf(error,
688 "cannot close metadata after creation: %s.\n",
689 strerror(errno));
690 }
691
692 if (Meta_open(cf)) {
693 Cache_free(cf);
694 lprintf(error, "cannot open metadata file, %s.\n", fn);
695 }
696
697 if (Meta_write(cf)) {
698 lprintf(error, "Meta_write() failed!\n");
699 }
700
701 if (fclose(cf->mfp)) {
702 lprintf(error,
703 "cannot close metadata after write, %s.\n",
704 strerror(errno));
705 }
706
707 Data_create(cf);
708
709 Cache_free(cf);
710
711 int res = Cache_exist(fn);
712
713 if (CONFIG.mode == NORMAL) {
714 curl_free(fn);
715 }
716
717 return res;
718 }
719
Cache_open(const char * fn)720 Cache *Cache_open(const char *fn)
721 {
722 /*
723 * Obtain the link structure memory pointer
724 */
725 Link *link = path_to_Link(fn);
726 if (!link) {
727 /*
728 * There is no associated link to the path
729 */
730 return NULL;
731 }
732
733 lprintf(cache_lock_debug,
734 "thread %x: locking cf_lock;\n", pthread_self());
735 PTHREAD_MUTEX_LOCK(&cf_lock);
736
737 if (link->cache_opened) {
738 link->cache_opened++;
739
740 lprintf(cache_lock_debug,
741 "thread %x: unlocking cf_lock;\n", pthread_self());
742 PTHREAD_MUTEX_UNLOCK(&cf_lock);
743 return link->cache_ptr;
744 }
745
746 /*
747 * Check if both metadata and data file exist
748 */
749 if (CONFIG.mode == NORMAL || CONFIG.mode == SINGLE) {
750 if (Cache_exist(fn)) {
751
752 lprintf(cache_lock_debug,
753 "thread %x: unlocking cf_lock;\n", pthread_self());
754 PTHREAD_MUTEX_UNLOCK(&cf_lock);
755 return NULL;
756 }
757 } else if (CONFIG.mode == SONIC) {
758 if (Cache_exist(link->sonic_id)) {
759
760 lprintf(cache_lock_debug,
761 "thread %x: unlocking cf_lock;\n", pthread_self());
762 PTHREAD_MUTEX_UNLOCK(&cf_lock);
763 return NULL;
764 }
765 } else {
766 lprintf(fatal, "Invalid CONFIG.mode\n");
767 }
768
769 /*
770 * Create the cache in-memory data structure
771 */
772 Cache *cf = Cache_alloc();
773
774 /*
775 * Fill in the fs_path
776 */
777 cf->fs_path = CALLOC(MAX_PATH_LEN + 1, sizeof(char));
778 strncpy(cf->fs_path, fn, MAX_PATH_LEN);
779
780 /*
781 * Set the path for the local cache file, if we are in sonic mode
782 */
783 if (CONFIG.mode == SONIC) {
784 fn = link->sonic_id;
785 }
786
787 cf->path = strndup(fn, MAX_PATH_LEN);
788
789 /*
790 * Associate the cache structure with a link
791 */
792 cf->link = link;
793
794 if (Meta_open(cf)) {
795 Cache_free(cf);
796 lprintf(error, "cannot open metadata file %s.\n", fn);
797
798 lprintf(cache_lock_debug,
799 "thread %x: unlocking cf_lock;\n", pthread_self());
800 PTHREAD_MUTEX_UNLOCK(&cf_lock);
801 return NULL;
802 }
803
804 /*
805 * Corrupt metadata
806 */
807 if (Meta_read(cf)) {
808 Cache_free(cf);
809 lprintf(error, "metadata error: %s.\n", fn);
810
811 lprintf(cache_lock_debug,
812 "thread %x: unlocking cf_lock;\n", pthread_self());
813 PTHREAD_MUTEX_UNLOCK(&cf_lock);
814 return NULL;
815 }
816
817 /*
818 * Inconsistency between metadata and data file, note that on disk file
819 * size might be bigger than content_length, due to on-disk filesystem
820 * allocation policy.
821 */
822 if (cf->content_length > Data_size(fn)) {
823 lprintf(error, "metadata inconsistency %s, \
824 cf->content_length: %ld, Data_size(fn): %ld.\n", fn, cf->content_length, Data_size(fn));
825 Cache_free(cf);
826
827 lprintf(cache_lock_debug,
828 "thread %x: unlocking cf_lock;\n", pthread_self());
829 PTHREAD_MUTEX_UNLOCK(&cf_lock);
830 return NULL;
831 }
832
833 /*
834 * Check if the cache files are not outdated
835 */
836 if (cf->time != cf->link->time) {
837 lprintf(warning, "outdated cache file: %s.\n", fn);
838 Cache_free(cf);
839
840 lprintf(cache_lock_debug,
841 "thread %x: unlocking cf_lock;\n", pthread_self());
842 PTHREAD_MUTEX_UNLOCK(&cf_lock);
843 return NULL;
844 }
845
846 if (Data_open(cf)) {
847 Cache_free(cf);
848 lprintf(error, "cannot open data file %s.\n", fn);
849
850 lprintf(cache_lock_debug,
851 "thread %x: unlocking cf_lock;\n", pthread_self());
852 PTHREAD_MUTEX_UNLOCK(&cf_lock);
853 return NULL;
854 }
855
856 cf->link->cache_opened = 1;
857 /*
858 * Yup, we just created a circular loop. ;)
859 */
860 cf->link->cache_ptr = cf;
861
862 lprintf(cache_lock_debug,
863 "thread %x: unlocking cf_lock;\n", pthread_self());
864 PTHREAD_MUTEX_UNLOCK(&cf_lock);
865 return cf;
866 }
867
Cache_close(Cache * cf)868 void Cache_close(Cache * cf)
869 {
870 lprintf(cache_lock_debug,
871 "thread %x: locking cf_lock;\n", pthread_self());
872 PTHREAD_MUTEX_LOCK(&cf_lock);
873
874 cf->link->cache_opened--;
875
876 if (cf->link->cache_opened > 0) {
877
878 lprintf(cache_lock_debug,
879 "thread %x: unlocking cf_lock;\n", pthread_self());
880 PTHREAD_MUTEX_UNLOCK(&cf_lock);
881 return;
882 }
883
884 if (Meta_write(cf)) {
885 lprintf(error, "Meta_write() error.");
886 }
887
888 if (fclose(cf->mfp)) {
889 lprintf(error, "cannot close metadata: %s.\n", strerror(errno));
890 }
891
892 if (fclose(cf->dfp)) {
893 lprintf(error, "cannot close data file %s.\n", strerror(errno));
894 }
895
896 lprintf(cache_lock_debug,
897 "thread %x: unlocking cf_lock;\n", pthread_self());
898 PTHREAD_MUTEX_UNLOCK(&cf_lock);
899 return Cache_free(cf);
900 }
901
902 /**
903 * \brief Check if a segment exists.
904 * \return 1 if the segment exists
905 */
Seg_exist(Cache * cf,off_t offset)906 static int Seg_exist(Cache * cf, off_t offset)
907 {
908 off_t byte = offset / cf->blksz;
909 return cf->seg[byte];
910 }
911
912 /**
913 * \brief Set the existence of a segment
914 * \param[in] cf the cache in-memory data structure
915 * \param[in] offset the starting position of the segment.
916 * \param[in] i 1 for exist, 0 for doesn't exist
917 * \note Call this after downloading a segment.
918 */
Seg_set(Cache * cf,off_t offset,int i)919 static void Seg_set(Cache * cf, off_t offset, int i)
920 {
921 off_t byte = offset / cf->blksz;
922 cf->seg[byte] = i;
923 }
924
925 /**
926 * \brief Background download function
927 * \details If we are requesting the data from the second half of the current
928 * segment, we can spawn a pthread using this function to download the next
929 * segment.
930 */
Cache_bgdl(void * arg)931 static void *Cache_bgdl(void *arg)
932 {
933 Cache *cf = (Cache *) arg;
934
935 lprintf(cache_lock_debug, "thread %x: locking w_lock;\n",
936 pthread_self());
937 PTHREAD_MUTEX_LOCK(&cf->w_lock);
938
939 uint8_t *recv_buf = CALLOC(cf->blksz, sizeof(uint8_t));
940 lprintf(debug, "thread %x spawned.\n ", pthread_self());
941 long recv = path_download(cf->fs_path, (char *) recv_buf, cf->blksz,
942 cf->next_dl_offset);
943 if (recv < 0) {
944 lprintf(error, "thread %x received %ld bytes, \
945 which does't make sense\n", pthread_self(), recv);
946 }
947
948 if ((recv == cf->blksz) ||
949 (cf->next_dl_offset ==
950 (cf->content_length / cf->blksz * cf->blksz))) {
951 Data_write(cf, recv_buf, recv, cf->next_dl_offset);
952 Seg_set(cf, cf->next_dl_offset, 1);
953 } else {
954 lprintf(error, "received %ld rather than %ld, possible network \
955 error.\n", recv, cf->blksz);
956 }
957
958 FREE(recv_buf);
959
960 lprintf(cache_lock_debug,
961 "thread %x: unlocking bgt_lock;\n", pthread_self());
962 PTHREAD_MUTEX_UNLOCK(&cf->bgt_lock);
963
964 lprintf(cache_lock_debug,
965 "thread %x: unlocking w_lock;\n", pthread_self());
966 PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
967
968 pthread_detach(pthread_self());
969 pthread_exit(NULL);
970 }
971
972 long
Cache_read(Cache * cf,char * const output_buf,const off_t len,const off_t offset_start)973 Cache_read(Cache * cf, char *const output_buf, const off_t len,
974 const off_t offset_start)
975 {
976 long send;
977
978 /*
979 * The offset of the segment to be downloaded
980 */
981 off_t dl_offset = (offset_start + len) / cf->blksz * cf->blksz;
982
983 /*
984 * ------------- Check if the segment already exists --------------
985 */
986 if (Seg_exist(cf, dl_offset)) {
987 send = Data_read(cf, (uint8_t *) output_buf, len, offset_start);
988 goto bgdl;
989 } else {
990 /*
991 * Wait for any other download thread to finish
992 */
993
994 lprintf(cache_lock_debug,
995 "thread %ld: locking w_lock;\n", pthread_self());
996 PTHREAD_MUTEX_LOCK(&cf->w_lock);
997
998 if (Seg_exist(cf, dl_offset)) {
999 /*
1000 * The segment now exists - it was downloaded by another
1001 * download thread. Send it off and unlock the I/O
1002 */
1003 send =
1004 Data_read(cf, (uint8_t *) output_buf, len, offset_start);
1005
1006 lprintf(cache_lock_debug,
1007 "thread %x: unlocking w_lock;\n", pthread_self());
1008 PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
1009
1010 goto bgdl;
1011 }
1012 }
1013
1014 /*
1015 * ------------------ Download the segment ---------------------
1016 */
1017
1018 uint8_t *recv_buf = CALLOC(cf->blksz, sizeof(uint8_t));
1019 lprintf(debug, "thread %x: spawned.\n ", pthread_self());
1020 long recv = path_download(cf->fs_path, (char *) recv_buf, cf->blksz,
1021 dl_offset);
1022 if (recv < 0) {
1023 lprintf(error, "thread %x received %ld bytes, \
1024 which does't make sense\n", pthread_self(), recv);
1025 }
1026 /*
1027 * check if we have received enough data, write it to the disk
1028 *
1029 * Condition 1: received the exact amount as the segment size.
1030 * Condition 2: offset is the last segment
1031 */
1032 if ((recv == cf->blksz) ||
1033 (dl_offset == (cf->content_length / cf->blksz * cf->blksz))) {
1034 Data_write(cf, recv_buf, recv, dl_offset);
1035 Seg_set(cf, dl_offset, 1);
1036 } else {
1037 lprintf(error, "received %ld rather than %ld, possible network \
1038 error.\n", recv, cf->blksz);
1039 }
1040 FREE(recv_buf);
1041 send = Data_read(cf, (uint8_t *) output_buf, len, offset_start);
1042
1043 lprintf(cache_lock_debug,
1044 "thread %x: unlocking w_lock;\n", pthread_self());
1045 PTHREAD_MUTEX_UNLOCK(&cf->w_lock);
1046
1047 /*
1048 * ----------- Download the next segment in background -----------------
1049 */
1050 bgdl:
1051 {
1052 }
1053 off_t next_dl_offset = round_div(offset_start, cf->blksz) * cf->blksz;
1054 if ((next_dl_offset > dl_offset) && !Seg_exist(cf, next_dl_offset)
1055 && next_dl_offset < cf->content_length) {
1056 /*
1057 * Stop the spawning of multiple background pthreads
1058 */
1059 if (!pthread_mutex_trylock(&cf->bgt_lock)) {
1060 lprintf(cache_lock_debug,
1061 "thread %x: trylocked bgt_lock;\n", pthread_self());
1062 cf->next_dl_offset = next_dl_offset;
1063 if (pthread_create(&cf->bgt, NULL, Cache_bgdl, cf)) {
1064 lprintf(error,
1065 "Error creating background download thread\n");
1066 }
1067 }
1068 }
1069
1070 return send;
1071 }
1072