1 /*===========================================================================
2 *
3 * PUBLIC DOMAIN NOTICE
4 * National Center for Biotechnology Information
5 *
6 * This software/database is a "United States Government Work" under the
7 * terms of the United States Copyright Act. It was written as part of
8 * the author's official duties as a United States Government employee and
9 * thus cannot be copyrighted. This software/database is freely available
10 * to the public for use. The National Library of Medicine and the U.S.
11 * Government have not placed any restriction on its use or reproduction.
12 *
13 * Although all reasonable efforts have been taken to ensure the accuracy
14 * and reliability of the software and data, the NLM and the U.S.
15 * Government do not and cannot warrant the performance or results that
16 * may be obtained by using this software or data. The NLM and the U.S.
17 * Government disclaim all warranties, express or implied, including
18 * warranties of performance, merchantability or fitness for any particular
19 * purpose.
20 *
21 * Please cite the author in any work or product based on this material.
22 *
23 * ===========================================================================
24 */
25
26 #include <kfs/extern.h>
27
28 struct KCacheTeeFile;
29 #define KFILE_IMPL struct KCacheTeeFile
30 #include <kfs/impl.h>
31
32 #include <klib/rc.h>
33 #include <klib/log.h>
34 #include <klib/out.h>
35 #include <klib/text.h>
36 #include <klib/printf.h>
37 #include <klib/checksum.h>
38 #include <klib/time.h>
39
40 #include <kfs/cacheteefile.h>
41 #include <kfs/defs.h>
42 #include <kproc/queue.h>
43 #include <kproc/timeout.h>
44 #include <atomic32.h>
45
46 #include <sysalloc.h>
47 #include <stdlib.h>
48 #include <string.h>
49 #include <assert.h>
50 #include <endian.h>
51
52 #include <stdio.h>
53
54 #define NO_SCRATCH_BUFFER 1
55 #define USE_BUFFER_POOL NO_SCRATCH_BUFFER && 1
56
57 /* byte-order is an issue for treating these as words */
58 #define USE_32BIT_BITMAP_WORDS 1
59
60
61 /*--------------------------------------------------------------------------
62 layout of local file:
63
64 CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC MMMMMMMMMMMMMM SS BB
65
66 C ... file content in valid/invalid blocks
67 M ... bitmap of blocks bytes = ( ( ( content-size / block-size ) + 1 ) / 8 ) + 1 )
68 S ... size of content ( uint64_t ) 8 bytes
69 B ... used blocksize ( uint32_t ) 4 bytes
70 */
71
72
73 /*--------------------------------------------------------------------------
74 * KCacheTeeFile
75 */
76
77 #define CACHE_TEE_DEFAULT_BLOCKSIZE ( 32 * 1024 * 4 )
78
79 #define CACHE_STAT 0
80
81 #if( CACHE_STAT > 0 )
82 typedef struct CacheStatistic
83 {
84 uint64_t requests;
85 uint64_t requested_bytes;
86 uint64_t delivered_bytes;
87 uint64_t requests_below_32k;
88 uint64_t requests_consecutive;
89 uint64_t requests_in_first32k;
90 uint64_t prev_pos;
91 uint64_t requests_same_pos;
92 uint64_t requests_same_pos_and_len;
93 size_t prev_len;
94 size_t min_len;
95 size_t max_len;
96
97 } CacheStatistic;
98
99
init_cache_stat(CacheStatistic * stat)100 static void init_cache_stat( CacheStatistic * stat )
101 {
102 stat -> requests = 0;
103 stat -> requested_bytes = 0;
104 stat -> delivered_bytes = 0;
105 stat -> requests_below_32k = 0;
106 stat -> requests_consecutive = 0;
107 stat -> requests_in_first32k = 0;
108 stat -> prev_pos = 0xFFFFFFFFFFFFFFFF;
109 stat -> requests_same_pos = 0;
110 stat -> requests_same_pos_and_len = 0;
111 stat -> prev_len = 0;
112 stat -> min_len = -1;
113 stat -> max_len = 0;
114 }
115
write_cache_stat(CacheStatistic * stat,uint64_t pos,size_t requested,size_t delivered)116 static void write_cache_stat( CacheStatistic * stat, uint64_t pos, size_t requested, size_t delivered )
117 {
118 ( stat -> requests )++;
119 ( stat -> requested_bytes )+= requested;
120 ( stat -> delivered_bytes )+= delivered;
121
122 if ( requested < 0x08000 )
123 ( stat -> requests_below_32k )++;
124
125 if ( delivered < stat -> min_len )
126 stat -> min_len = delivered;
127
128 if ( delivered > stat -> max_len )
129 stat -> max_len = delivered;
130
131 if ( pos + requested < 0x08000 )
132 ( stat -> requests_in_first32k )++;
133
134 if ( stat -> prev_pos != pos )
135 {
136 if ( stat -> prev_pos + stat -> prev_len == pos )
137 {
138 ( stat -> requests_consecutive )++;
139 }
140
141 stat -> prev_pos = pos;
142 stat -> prev_len = requested;
143 }
144 else
145 {
146 if ( stat -> prev_len != requested )
147 {
148 ( stat -> requests_same_pos_and_len )++;
149 stat -> prev_len = requested;
150 }
151 ( stat -> requests_same_pos )++;
152 }
153 }
154
report_cache_stat(CacheStatistic * stat)155 static void report_cache_stat( CacheStatistic * stat )
156 {
157 fprintf( stderr, "\n" );
158 fprintf( stderr, "cache-stat.requests ................... %lu\n", stat -> requests );
159 fprintf( stderr, "cache-stat.requested_bytes ............ %lu\n", stat -> requested_bytes );
160 fprintf( stderr, "cache-stat.delivered_bytes ............ %lu\n", stat -> delivered_bytes );
161 fprintf( stderr, "cache-stat.requests_below_32k ......... %lu\n", stat -> requests_below_32k );
162 fprintf( stderr, "cache-stat.requests_consecutive........ %lu\n", stat -> requests_consecutive );
163 fprintf( stderr, "cache-stat.requests_in_first32k........ %lu\n", stat -> requests_in_first32k );
164
165 fprintf( stderr, "cache-stat.requests_same_pos .......... %lu\n", stat -> requests_same_pos );
166 fprintf( stderr, "cache-stat.requests_same_pos_and_len .. %lu\n", stat -> requests_same_pos );
167 fprintf( stderr, "cache-stat.min_len .................... %u\n", stat -> min_len );
168 fprintf( stderr, "cache-stat.max_len .................... %u\n", stat -> max_len );
169
170 fprintf( stderr, "\n" );
171 }
172
173 #endif
174
175
176 typedef struct KCacheTeeFile
177 {
178 KFile dad;
179 const KFile * remote; /* the remote file we are wrapping (can be a local one too, we make no assumptions about that) */
180 KFile * local; /* the local cache of the remote one */
181 KDirectory * dir; /* we have to store a KDirectory because we need it at closing the file, where we test for promoting the cache */
182
183 uint64_t remote_size; /* the size of the wrapped file */
184 uint64_t local_size; /* the size of the local cache file ( remote_size + bitmap + tail ) */
185 uint64_t block_count; /* how many blocks do we need to cache the remote file ( last block may be shorter ) */
186
187 #if USE_32BIT_BITMAP_WORDS
188 atomic32_t * bitmap;
189 #else
190 uint8_t volatile * bitmap; /* the bitmap of cached blocks */
191 #endif
192
193 uint64_t bitmap_bytes; /* how many bytes do we need to store the bitmap */
194
195 #if USE_BUFFER_POOL
196 KQueue * buffer_pool;
197 #endif
198 #if ! NO_SCRATCH_BUFFER
199 uint8_t * scratch_buffer;
200 uint64_t first_block_in_scratch; /* what is the block-id of the first block in the scratch-buffer */
201 uint64_t scratch_size; /* how many bytes are allocated for the scratch-buffer */
202 uint64_t valid_scratch_bytes; /* how many bytes store valid data in the scratch-buffer */
203 #endif
204
205 uint32_t block_size; /* how big is a block ( aka 1 bit in the bitmap )*/
206
207 #if( CACHE_STAT > 0 )
208 CacheStatistic stat; /* optional cache statistic */
209 #endif
210
211 bool local_read_only;
212 bool promote;
213 char local_path [ 1 ]; /* stores the path to the local cache, for eventual promoting at close */
214 } KCacheTeeFile;
215
216
217 #if USE_32BIT_BITMAP_WORDS
218
219 #if __BYTE_ORDER == __LITTLE_ENDIAN
220 #define SWAP_FN(val) (val)
221 #else
222 #define SWAP_FN(val) \
223 (((val)>>24)&0xff) | /* move byte 3 to byte 0 */ \
224 (((val)<<8)&0xff0000) | /* move byte 1 to byte 2 */ \
225 (((val)>>8)&0xff00) | /* move byte 2 to byte 1 */ \
226 (((val)<<24)&0xff000000) /* byte 0 to byte 3 */
227 #endif
228 #define GEN_BIT_NR_MASK_ROW(i) SWAP_FN( 1U << ( (i) * 4 ) ), SWAP_FN( 1U << ( (i) * 4 + 1 ) ), SWAP_FN( 1U << ( (i) * 4 + 2 ) ), SWAP_FN( 1U << ( (i) * 4 + 3 ) )
229
230 const uint32_t BitNr2Mask[ 32 ] =
231 {
232 GEN_BIT_NR_MASK_ROW(0),
233 GEN_BIT_NR_MASK_ROW(1),
234 GEN_BIT_NR_MASK_ROW(2),
235 GEN_BIT_NR_MASK_ROW(3),
236 GEN_BIT_NR_MASK_ROW(4),
237 GEN_BIT_NR_MASK_ROW(5),
238 GEN_BIT_NR_MASK_ROW(6),
239 GEN_BIT_NR_MASK_ROW(7)
240 };
241 #undef SWAP_FN
242 #undef GEN_BIT_NR_MASK_ROW
243
244 #define IS_CACHE_BIT( CacheFile, Block_Nr ) \
245 ( ( atomic32_read ( & ( CacheFile )->bitmap[ (Block_Nr) >> 5 ] ) & BitNr2Mask[ (Block_Nr) & 31 ] ) > 0 )
246
247 #define IS_BITMAP_BIT( BitMap, Block_Nr ) \
248 ( ( atomic32_read ( & ( BitMap )[ (Block_Nr) >> 5 ] ) & BitNr2Mask[ (Block_Nr) & 31 ] ) > 0 )
249
250 #else
251
252 const uint8_t BitNr2Mask[ 8 ] =
253 {
254 /* 0 1 2 3 4 5 6 7 */
255 1, 2, 4, 8, 16, 32, 64, 128
256 };
257
258 #define IS_CACHE_BIT( CacheFile, Block_Nr ) ( ( CacheFile->bitmap[ (Block_Nr) >> 3 ] & BitNr2Mask[ (Block_Nr) & 7 ] ) > 0 )
259 #define IS_BITMAP_BIT( BitMap, Block_Nr ) ( ( BitMap[ (Block_Nr) >> 3 ] & BitNr2Mask[ (Block_Nr) & 7 ] ) > 0 )
260
261
262 #endif
263
264 #define BITS_2_BYTES( BitCount ) ( ( ( BitCount ) + 7 ) >> 3 )
265 #define SIZE_2_BLOCK_COUNT( Number_Of_Bytes, Block_Size ) ( ( ( Number_Of_Bytes ) + ( Block_Size ) - 1 ) / ( Block_Size ) )
266
calculate_local_size_from_remote_size(KCacheTeeFile * self)267 static rc_t calculate_local_size_from_remote_size( KCacheTeeFile *self )
268 {
269 rc_t rc = 0;
270 if ( self->block_size > 0 )
271 {
272 self->block_count = SIZE_2_BLOCK_COUNT( self->remote_size, self->block_size );
273 self->bitmap_bytes = BITS_2_BYTES( self->block_count );
274 self->local_size = ( self->remote_size +
275 self->bitmap_bytes +
276 sizeof self->remote_size +
277 sizeof self->block_size );
278 }
279 else
280 {
281 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
282 LOGERR( klogErr, rc, "div by zero attempt in calculating local size" );
283 }
284 return rc;
285 }
286
287 /* factored out because it is used in:
288 create_bitmap(), IsCacheFileComplete(), GetCacheCompleteness() and Has_Cache_Zero_Blocks()
289 */
290 #if USE_32BIT_BITMAP_WORDS
create_bitmap_buffer(atomic32_t ** bitmap,uint64_t bitmap_bytes)291 static rc_t create_bitmap_buffer( atomic32_t ** bitmap, uint64_t bitmap_bytes )
292 {
293 rc_t rc = 0;
294 *bitmap = calloc ( sizeof **bitmap, ( bitmap_bytes + sizeof ** bitmap - 1 ) / sizeof ** bitmap );
295 if ( *bitmap == NULL )
296 {
297 rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
298 LOGERR( klogErr, rc, "init local bitmap-area" );
299 }
300 return rc;
301 }
302 #else
create_bitmap_buffer(uint8_t volatile ** bitmap,uint64_t bitmap_bytes)303 static rc_t create_bitmap_buffer( uint8_t volatile ** bitmap, uint64_t bitmap_bytes )
304 {
305 rc_t rc = 0;
306 *bitmap = calloc ( sizeof **bitmap, bitmap_bytes );
307 if ( *bitmap == NULL )
308 {
309 rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
310 LOGERR( klogErr, rc, "init local bitmap-area" );
311 }
312 return rc;
313 }
314 #endif
315
316 /* factored out because it is used in:
317 init_new_local_file() and verify_existing_local_file()
318 */
create_bitmap(KCacheTeeFile * self)319 static rc_t create_bitmap( KCacheTeeFile *self )
320 {
321 return create_bitmap_buffer( &self->bitmap, self->bitmap_bytes );
322 }
323
324
325 #if USE_32BIT_BITMAP_WORDS
is_bitmap_full(const atomic32_t * bitmap,uint64_t bitmap_bytes,uint64_t block_count)326 static bool is_bitmap_full( const atomic32_t * bitmap, uint64_t bitmap_bytes, uint64_t block_count )
327 {
328 uint64_t bitmap_word;
329 const uint64_t bitmap_words_minus_one = ( ( bitmap_bytes + sizeof * bitmap - 1 ) >> 2 ) - 1;
330 for( bitmap_word = 0; bitmap_word < bitmap_words_minus_one; ++ bitmap_word )
331 {
332 if ( ~ atomic32_read ( & bitmap [ bitmap_word ] ) != 0 )
333 return false;
334 }
335
336 {
337 uint64_t block_id = ( bitmap_word << 5 );
338 while ( block_id < block_count )
339 {
340 bool block_cached = IS_BITMAP_BIT( bitmap, block_id );
341 if ( !block_cached )
342 return false;
343 ++block_id;
344 }
345 }
346 return true;
347 }
348 #else
is_bitmap_full(const uint8_t volatile * bitmap,uint64_t bitmap_bytes,uint64_t block_count)349 static bool is_bitmap_full( const uint8_t volatile * bitmap, uint64_t bitmap_bytes, uint64_t block_count )
350 {
351 uint64_t bitmap_byte = 0;
352 while( bitmap_byte < ( bitmap_bytes - 1 ) )
353 {
354 if ( bitmap[ bitmap_byte++ ] != 0xFF )
355 return false;
356 }
357
358 {
359 uint64_t block_id = ( bitmap_byte << 3 );
360 while ( block_id < block_count )
361 {
362 bool block_cached = IS_BITMAP_BIT( bitmap, block_id );
363 if ( !block_cached )
364 return false;
365 ++block_id;
366 }
367 }
368 return true;
369 }
370 #endif
371
372
init_new_local_file(KCacheTeeFile * cf)373 static rc_t init_new_local_file( KCacheTeeFile * cf )
374 {
375 rc_t rc = create_bitmap( cf );
376 if ( rc == 0 )
377 {
378 size_t written;
379 uint64_t pos = cf -> remote_size;
380
381 /* write the bitmap ( zero'd out ) into the local file */
382 rc = KFileWriteAll ( cf -> local, pos,
383 ( const void * ) cf -> bitmap, cf -> bitmap_bytes, &written );
384 if ( rc == 0 && written != cf -> bitmap_bytes )
385 {
386 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
387 LOGERR( klogErr, rc, "no full initialization of local file bitmap" );
388 }
389
390 /* write the remote-file-size into the local file ( uint64_t = 8 bytes )*/
391 if ( rc == 0 )
392 {
393 pos += cf -> bitmap_bytes;
394 rc = KFileWriteAll ( cf -> local, pos,
395 &cf -> remote_size, sizeof( cf -> remote_size ), &written );
396 if ( rc == 0 && written != sizeof( cf -> remote_size ) )
397 {
398 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
399 LOGERR( klogErr, rc, "cannot write remote-filesize into local file" );
400 }
401 }
402
403 /* write the block-size into the local file ( size_t = 4 bytes )*/
404 if ( rc == 0 )
405 {
406 pos += sizeof( cf -> remote_size );
407 rc = KFileWriteAll ( cf -> local, pos,
408 &cf -> block_size, sizeof( cf -> block_size ), &written );
409 if ( rc == 0 && written != sizeof( cf -> block_size ) )
410 {
411 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
412 LOGERR( klogErr, rc, "cannot write block-size into local file" );
413 }
414 }
415
416 }
417 return rc;
418 }
419
420
try_read_uint32_t(const struct KFile * self,uint64_t pos,uint32_t * value)421 static rc_t try_read_uint32_t( const struct KFile * self, uint64_t pos, uint32_t * value )
422 {
423 size_t num_read;
424 rc_t rc = KFileRead( self, pos, value, sizeof *value, &num_read );
425 if ( rc == 0 )
426 {
427 if ( num_read != sizeof *value )
428 rc = SILENT_RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
429 }
430 return rc;
431 }
432
try_read_uint64_t(const struct KFile * self,uint64_t pos,uint64_t * value)433 static rc_t try_read_uint64_t( const struct KFile * self, uint64_t pos, uint64_t * value )
434 {
435 size_t num_read;
436 rc_t rc = KFileRead( self, pos, value, sizeof *value, &num_read );
437 if ( rc == 0 )
438 {
439 if ( num_read != sizeof *value )
440 rc = SILENT_RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
441 }
442 return rc;
443 }
444
445
read_block_size(const struct KFile * self,uint64_t local_size,uint32_t * block_size)446 static rc_t read_block_size( const struct KFile * self, uint64_t local_size, uint32_t *block_size )
447 {
448 if ( local_size >= sizeof *block_size )
449 {
450 uint64_t pos = local_size - ( sizeof *block_size );
451 int num_try = 3;
452 rc_t rc;
453
454 while ( true )
455 {
456 rc = try_read_uint32_t( self, pos, block_size );
457
458 if ( rc == 0 && *block_size != 0 )
459 // we are done
460 return 0;
461
462 if ( --num_try == 0 )
463 break;
464
465 KSleep( 1 );
466 }
467
468 if ( rc != 0 )
469 return rc;
470 }
471
472 return RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
473 }
474
475
read_content_size(const struct KFile * self,uint64_t local_size,uint64_t * content_size)476 static rc_t read_content_size( const struct KFile * self, uint64_t local_size, uint64_t *content_size )
477 {
478 if ( local_size >= sizeof( *content_size ) + 4 )
479 {
480 uint64_t pos = ( local_size - 4 ) - sizeof( *content_size );
481 int num_try = 3;
482 rc_t rc;
483
484 while ( true )
485 {
486 rc = try_read_uint64_t( self, pos, content_size );
487 if ( rc == 0 && *content_size != 0 )
488 {
489 if ( *content_size < local_size )
490 return 0;
491 else
492 return RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
493 }
494
495 if ( --num_try == 0 )
496 break;
497
498 KSleep( 1 );
499 }
500
501 if ( rc != 0 )
502 return rc;
503 }
504
505 return RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
506 }
507
508
verify_file_structure(const uint64_t local_size,const uint32_t block_size,const uint64_t content_size,uint64_t * block_count,uint64_t * bitmap_bytes)509 static rc_t verify_file_structure( const uint64_t local_size, const uint32_t block_size, const uint64_t content_size,
510 uint64_t * block_count, /* size_t */ uint64_t * bitmap_bytes )
511 {
512 rc_t rc = 0;
513 uint64_t expected_size;
514
515 *block_count = SIZE_2_BLOCK_COUNT( content_size, block_size );
516 *bitmap_bytes = BITS_2_BYTES( *block_count );
517
518 /* check if the values 'content-size' and 'block_size' result in the currect real file size */
519 expected_size = content_size + *bitmap_bytes + sizeof ( local_size ) + sizeof ( block_size );
520 if ( expected_size != local_size )
521 rc = RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
522 return rc;
523 }
524
525
read_bitmap(KCacheTeeFile * cf)526 static rc_t read_bitmap( KCacheTeeFile * cf )
527 {
528 /* read the bitmap from the local file */
529 size_t num_read;
530 rc_t rc = KFileReadAll ( cf -> local, cf -> remote_size, ( void * ) cf -> bitmap, cf -> bitmap_bytes, &num_read );
531 if ( rc != 0 )
532 {
533 LOGERR( klogErr, rc, "cannot read bitmap from local file" );
534 }
535 else if ( num_read != cf -> bitmap_bytes )
536 {
537 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
538 PLOGERR( klogErr, ( klogErr, rc, "cannot read $(ls) bitmap-bytes from local file, read $(rs) instead",
539 "ls=%lu,rs=%lu", cf -> bitmap_bytes, num_read ));
540 }
541 return rc;
542 }
543
544
verify_existing_local_file(KCacheTeeFile * cf,bool * fully_in_cache)545 static rc_t verify_existing_local_file( KCacheTeeFile * cf, bool * fully_in_cache )
546 {
547 uint64_t bitmap_bytes, content_size, block_count;
548 uint32_t block_size;
549 bool cached = false;
550
551 rc_t rc = read_block_size ( cf -> local, cf -> local_size, &block_size );
552 /* read content-size, compare to the content-size of the remote file */
553 if ( rc == 0 )
554 rc = read_content_size ( cf -> local, cf -> local_size, &content_size );
555
556 if ( rc == 0 && content_size != cf -> remote_size )
557 {
558 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
559 PLOGERR( klogErr, ( klogErr, rc, "content-size in local file $(ls) does not match size of remote file $(rs)",
560 "ls=%lu,rs=%lu", content_size, cf -> remote_size ) );
561 }
562
563 /* check if the local file has the right size ( content + bitmap + content_size + block_size )*/
564 if ( rc == 0 )
565 rc = verify_file_structure ( cf -> local_size, block_size, content_size, &block_count, &bitmap_bytes );
566
567 /* check if the requested block-size equals the stored block-size */
568 if ( rc == 0 )
569 {
570 if ( cf -> block_size == 0 )
571 cf -> block_size = CACHE_TEE_DEFAULT_BLOCKSIZE;
572 if ( cf -> block_size != block_size )
573 {
574 PLOGMSG( klogWarn, ( klogWarn, "block-size in local file $(ls) does not match requested value $(rv)",
575 "ls=%u,rv=%u", block_size, cf -> block_size ) );
576 }
577 /* use the block-size as stored in the file */
578 cf -> block_size = block_size;
579 cf -> block_count = block_count;
580 cf -> bitmap_bytes = bitmap_bytes;
581 rc = create_bitmap( cf );
582 }
583
584 /* read the bitmap into the memory ... */
585 if ( rc == 0 )
586 rc = read_bitmap( cf );
587
588 if ( rc == 0 )
589 cached = is_bitmap_full( cf -> bitmap, cf -> bitmap_bytes, cf -> block_count );
590
591 if ( rc == 0 && fully_in_cache != NULL )
592 *fully_in_cache = cached;
593
594 return rc;
595 }
596
597
IsCacheFileComplete(const struct KFile * self,bool * is_complete)598 LIB_EXPORT rc_t CC IsCacheFileComplete( const struct KFile * self, bool * is_complete )
599 {
600 rc_t rc;
601 if ( self == NULL )
602 rc = RC( rcFS, rcFile, rcValidating, rcSelf, rcNull );
603 else if ( is_complete == NULL )
604 rc = RC( rcFS, rcFile, rcValidating, rcParam, rcNull );
605 else
606 {
607 uint64_t local_size;
608 *is_complete = false;
609 rc = KFileSize( self, &local_size );
610 if ( rc != 0 )
611 {
612 LOGERR( klogErr, rc, "cannot detect filesize when checking if cached file is complete" );
613 }
614 /* TODO: need to do a better constant tha a magic number */
615 else if (local_size < 13)
616 {
617 rc = RC ( rcFS, rcFile, rcValidating, rcSize, rcInsufficient );
618 LOGERR( klogErr, rc, "cannot use filesize of zero when checking if cached file is complete" );
619 }
620 else
621 {
622 uint32_t block_size;
623 rc = read_block_size( self, local_size, &block_size );
624 if ( rc == 0 )
625 {
626 uint64_t content_size;
627
628 rc = read_content_size( self, local_size, &content_size );
629 /* create bitmap buffer */
630 if ( rc == 0 )
631 {
632 uint64_t block_count;
633 /* size_t */ uint64_t bitmap_bytes;
634 rc = verify_file_structure( local_size, block_size, content_size, &block_count, &bitmap_bytes );
635 if ( rc == 0 )
636 {
637 #if USE_32BIT_BITMAP_WORDS
638 atomic32_t * bitmap = NULL;
639 #else
640 uint8_t volatile * bitmap = NULL;
641 #endif
642 rc = create_bitmap_buffer( &bitmap, bitmap_bytes );
643 if ( rc == 0 )
644 {
645 size_t num_read;
646 /* read the bitmap into the memory ... */
647 rc = KFileReadAll ( self, content_size, ( void * ) bitmap, bitmap_bytes, &num_read );
648 if ( rc != 0 )
649 {
650 LOGERR( klogErr, rc, "cannot read bitmap from local file" );
651 }
652 else if ( num_read != bitmap_bytes )
653 {
654 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
655 PLOGERR( klogErr, ( klogErr, rc, "cannot read $(ls) bitmap-bytes from local file, read $(rs) instead",
656 "ls=%lu,rs=%lu", bitmap_bytes, num_read ));
657 }
658 }
659 if ( rc == 0 )
660 *is_complete = is_bitmap_full( bitmap, bitmap_bytes, block_count );
661
662 if ( bitmap != NULL )
663 free( ( void * ) bitmap );
664 }
665 }
666 }
667 }
668 }
669 return rc;
670 }
671
672
TruncateCacheFile(struct KFile * self)673 LIB_EXPORT rc_t CC TruncateCacheFile( struct KFile * self )
674 {
675 rc_t rc;
676 if ( self == NULL )
677 rc = RC( rcFS, rcFile, rcResizing, rcSelf, rcNull );
678 else
679 {
680 uint64_t local_size;
681 rc = KFileSize( self, &local_size );
682 if ( rc != 0 )
683 {
684 LOGERR( klogErr, rc, "cannot detect filesize when truncating cached file" );
685 }
686 else
687 {
688 uint32_t block_size;
689 rc = read_block_size( self, local_size, &block_size );
690 if ( rc == 0 )
691 {
692 uint64_t content_size;
693 rc = read_content_size( self, local_size, &content_size );
694 if ( rc == 0 )
695 {
696 uint64_t block_count, bitmap_bytes;
697 rc = verify_file_structure( local_size, block_size, content_size, &block_count, &bitmap_bytes );
698
699 /* truncate the file by setting the new (shorter) filesize */
700 if ( rc == 0 )
701 rc = KFileSetSize( self, content_size );
702 }
703 }
704 }
705 }
706 return rc;
707 }
708
709
file_exist(KDirectory * dir,const char * filename)710 static bool file_exist( KDirectory * dir, const char * filename )
711 {
712 uint32_t pt = KDirectoryPathType ( dir, "%s", filename );
713 return ( ( pt & ~kptAlias ) == kptFile );
714 }
715
716
promote_cache(KCacheTeeFile * self)717 static rc_t promote_cache( KCacheTeeFile * self )
718 {
719 char cache_file_name [ 4096 ];
720 char temp_file_name [ 4096 ];
721 size_t num_writ;
722 rc_t rc = string_printf ( cache_file_name, sizeof cache_file_name, &num_writ, "%s.cache", self -> local_path );
723 if ( rc == 0 )
724 rc = string_printf ( temp_file_name, sizeof temp_file_name, &num_writ, "%s.cache.temp", self -> local_path );
725
726 /* (1) releaes open cache file ( windows cannot rename open files ) */
727 if ( rc == 0 )
728 rc = KFileRelease( self -> local );
729
730 /* (2) rename to temporary name */
731 if ( rc == 0 )
732 {
733 self -> local = NULL;
734 rc = KDirectoryRename ( self -> dir, true, cache_file_name, temp_file_name );
735 }
736
737 /* (3) open from temporary name */
738 if ( rc == 0 )
739 rc = KDirectoryOpenFileWrite( self -> dir, &self -> local, true, "%s", temp_file_name );
740
741 /* (4) perform truncation */
742 if ( rc == 0 )
743 rc = TruncateCacheFile( self -> local );
744
745 /* (5) releaes open temp. cache file ( windows cannot rename open files ) */
746 if ( rc == 0 )
747 rc = KFileRelease( self -> local );
748
749 /* (6) rename to final filename ( windows cannot rename open files ) */
750 if ( rc == 0 )
751 {
752 self -> local = NULL;
753 rc = KDirectoryRename ( self -> dir, true, temp_file_name, self -> local_path );
754 }
755
756 /* (6) open from final filename */
757 if ( rc == 0 )
758 rc = KDirectoryOpenFileWrite( self -> dir, &self -> local, true, "%s", self -> local_path );
759
760 return rc;
761 }
762
763 #if USE_BUFFER_POOL
pop_page(KQueue * buffer_pool,uint32_t timeout_millisec)764 static void * pop_page( KQueue * buffer_pool, uint32_t timeout_millisec )
765 {
766 rc_t rc;
767 void * page;
768 struct timeout_t tm;
769 TimeoutInit ( & tm, timeout_millisec );
770 rc = KQueuePop( buffer_pool, &page, &tm );
771 if ( rc != 0 )
772 page = NULL;
773 return page;
774 }
775
776 /* helper to clean up the buffer_pool */
clean_up_buffer_pool(KQueue * buffer_pool)777 static void clean_up_buffer_pool( KQueue * buffer_pool )
778 {
779 void * pool_page;
780 while ( ( pool_page = pop_page( buffer_pool, 100 ) ) != NULL )
781 {
782 free( pool_page );
783 }
784 KQueueRelease( buffer_pool );
785 }
786 #endif
787
788 /* Destroy
789 */
KCacheTeeFileDestroy(KCacheTeeFile * self)790 static rc_t CC KCacheTeeFileDestroy( KCacheTeeFile * self )
791 {
792 rc_t rc;
793 bool already_promoted_by_other_instance = file_exist( self -> dir, self -> local_path );
794
795 #if( CACHE_STAT > 0 )
796 report_cache_stat( & self -> stat );
797 #endif
798
799 if ( !self -> local_read_only && !already_promoted_by_other_instance )
800 {
801 bool fully_in_cache;
802 rc = IsCacheFileComplete ( self -> local, &fully_in_cache );
803 if ( rc == 0 && fully_in_cache && self -> promote )
804 {
805 promote_cache( self );
806 }
807 }
808
809 if ( self->bitmap != NULL )
810 free( ( void * ) self->bitmap );
811 #if ! NO_SCRATCH_BUFFER
812 if ( self->scratch_buffer != NULL )
813 free( self->scratch_buffer );
814 #endif
815
816 #if USE_BUFFER_POOL
817 clean_up_buffer_pool( self -> buffer_pool );
818 #endif
819
820 KFileRelease ( self -> remote );
821 KFileRelease ( self -> local );
822
823 if ( already_promoted_by_other_instance )
824 KDirectoryRemove ( self -> dir, true, "%s.cache", self -> local_path );
825
826 KDirectoryRelease ( self -> dir );
827
828 free ( self );
829 return 0;
830 }
831
832
set_bitmap(const KCacheTeeFile * cself,uint64_t start_block,uint64_t block_count)833 static void set_bitmap( const KCacheTeeFile *cself, uint64_t start_block, uint64_t block_count )
834 {
835 #if USE_32BIT_BITMAP_WORDS
836 uint32_t old, bits;
837 const uint32_t block_bit = BitNr2Mask [ start_block & 31 ];
838
839 /* we should get rid of block count */
840 assert ( block_count == 1 );
841
842 old = atomic32_read ( & cself -> bitmap [ start_block >> 5 ] );
843 do
844 {
845 bits = old;
846 old = atomic32_test_and_set ( & cself -> bitmap [ start_block >> 5 ], ( int ) ( bits | block_bit ), ( int ) bits );
847 }
848 while ( old != bits );
849
850 #else
851 uint64_t block_idx, block_nr;
852 for ( block_idx = 0, block_nr = start_block;
853 block_idx < block_count;
854 ++block_idx, ++block_nr )
855 {
856 cself->bitmap[ block_nr >> 3 ] |= BitNr2Mask[ block_nr & 0x07 ];
857 }
858 #endif
859 }
860
861
switch_to_read_only(const KCacheTeeFile * cself,rc_t rc)862 static rc_t switch_to_read_only( const KCacheTeeFile *cself, rc_t rc )
863 {
864 KCacheTeeFile *self = ( KCacheTeeFile * )cself;
865 self->local_read_only = true;
866 LOGERR( klogInt, rc, "switching cache-tee-file to read-only" );
867 return 0;
868 }
869
write_bitmap(const KCacheTeeFile * cself,uint64_t block)870 static rc_t write_bitmap( const KCacheTeeFile *cself, uint64_t block )
871 {
872 rc_t rc;
873 size_t written;
874 uint64_t pos;
875 size_t to_write;
876 #if USE_32BIT_BITMAP_WORDS
877 uint32_t block_word = ( uint32_t ) ( block >> 5 );
878 uint64_t bitmap_pos = ( block_word << 2 );
879 pos = cself->remote_size + bitmap_pos;
880 to_write = 4;
881
882 // last word may go outside bitmap ranges in the file, since bitmap has 1-byte alignment in the file, but 4-byte alignment in memory
883 if (bitmap_pos + to_write > cself->bitmap_bytes)
884 to_write = cself->bitmap_bytes - bitmap_pos;
885
886 rc = KFileWriteAll( cself->local, pos, ( const void * ) &cself->bitmap[ block_word ], to_write, &written );
887 #else
888 uint32_t block_byte = ( uint32_t ) ( block >> 3 );
889 pos = cself->remote_size + block_byte;
890 to_write = 1;
891 rc = KFileWriteAll( cself->local, pos, ( const void * ) &cself->bitmap[ block_byte ], to_write, &written );
892 #endif
893 if ( rc != 0 )
894 {
895 /* it can happen that we are not able to write to the bitmap because we run out of space
896 on the local filesystem. */
897 rc = switch_to_read_only( cself, rc );
898 /*
899 PLOGERR( klogErr, ( klogErr, rc, "cannot write local-file-bitmap block $(block) at $(pos) $(to_write) bytes",
900 "block=%lu,pos=%lu,to_write=%zu", block, pos, to_write ) );
901 */
902 }
903 return rc;
904 }
905
906
907 #if ! NO_SCRATCH_BUFFER
resize_scratch_buffer(const KCacheTeeFile * cself,uint64_t new_size)908 static rc_t resize_scratch_buffer( const KCacheTeeFile *cself, uint64_t new_size )
909 {
910 rc_t rc = 0;
911 KCacheTeeFile *self = ( KCacheTeeFile * )cself;
912
913 /* create scratch-buffer or resize it if needed */
914 if ( self->scratch_size == 0 )
915 {
916 self->scratch_buffer = malloc( new_size );
917 if ( self->scratch_buffer == NULL )
918 rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
919 else
920 self->scratch_size = new_size;
921 }
922 else if ( self->scratch_size < new_size )
923 {
924 uint8_t * tmp = realloc( self->scratch_buffer, new_size );
925 if ( tmp == NULL )
926 rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
927 else
928 {
929 self->scratch_buffer = tmp;
930 self->scratch_size = new_size;
931 }
932 }
933 return rc;
934 }
935 #endif
936
937
check_rd_len(const KCacheTeeFile * cself,uint64_t pos,size_t bsize)938 size_t check_rd_len( const KCacheTeeFile *cself, uint64_t pos, size_t bsize )
939 {
940 size_t res = bsize;
941 uint64_t last_pos = pos;
942 last_pos += bsize;
943 if ( last_pos >= cself->remote_size )
944 {
945 if ( pos >= cself->remote_size )
946 res = 0;
947 else
948 res = ( cself->remote_size - pos );
949 }
950 return res;
951 }
952
rd_remote_wr_local(const KCacheTeeFile * cself,uint64_t pos,void * buffer,size_t bsize,size_t * num_read)953 static rc_t rd_remote_wr_local( const KCacheTeeFile *cself, uint64_t pos,
954 void *buffer, size_t bsize, size_t *num_read )
955 {
956 rc_t rc = 0;
957 if ( bsize > 0 )
958 {
959 size_t bytes_read;
960 *num_read = 0;
961 rc = KFileReadAll( cself->remote, pos, buffer, bsize, &bytes_read );
962 if ( rc != 0 || bytes_read == 0 ) /** try again **/
963 {
964 rc = KFileReadAll( cself->remote, pos, buffer, bsize, &bytes_read );
965 if ( rc == 0 && bytes_read == 0 )
966 { /*old behavior */
967 rc = RC ( rcFS, rcFile, rcReading, rcBuffer, rcEmpty );
968 }
969 }
970 if ( rc == 0 )
971 {
972 if ( cself->local_read_only )
973 *num_read = bytes_read;
974 else
975 {
976 /* it can happen that we are running out of space in the local filesystem,
977 that means we cannot write ( any more ) */
978 rc = KFileWriteAll( cself->local, pos, buffer, bytes_read, num_read );
979 if ( rc != 0 )
980 rc = switch_to_read_only( cself, rc );
981 }
982 }
983 }
984 return rc;
985 }
986
987
KCacheTeeFileRead_simple2(const KCacheTeeFile * cself,uint64_t pos,void * buffer,size_t bsize,size_t * num_read)988 static rc_t KCacheTeeFileRead_simple2( const KCacheTeeFile *cself, uint64_t pos,
989 void *buffer, size_t bsize, size_t *num_read )
990 {
991 uint64_t block = pos / cself->block_size;
992 size_t offset = pos % cself->block_size;
993 size_t to_read_total = bsize;
994 int64_t salvage_block = -1;
995
996 #if NO_SCRATCH_BUFFER
997 rc_t rc = 0;
998 uint64_t first_block_in_scratch = -1;
999 uint64_t valid_scratch_bytes = 0;
1000 uint8_t * scratch_buffer = NULL;
1001
1002 #if USE_BUFFER_POOL
1003 scratch_buffer = pop_page( cself -> buffer_pool, 200 );
1004 #endif
1005
1006 if ( scratch_buffer == NULL )
1007 scratch_buffer = malloc ( cself -> block_size );
1008
1009 if ( scratch_buffer == NULL )
1010 return RC ( rcFS, rcFile, rcReading, rcMemory, rcExhausted );
1011 #else
1012 uint64_t first_block_in_scratch = cself -> first_block_in_scratch;
1013 uint64_t valid_scratch_bytes = cself -> valid_scratch_bytes;
1014 rc_t rc = resize_scratch_buffer( cself, cself->block_size );
1015 uint8_t * scratch_buffer = cself -> scratch_buffer;
1016 #endif
1017
1018 *num_read = 0;
1019
1020 while ( rc == 0 && to_read_total > 0 )
1021 {
1022 size_t to_read = cself->block_size - offset;
1023
1024 if ( to_read > to_read_total )
1025 to_read = to_read_total;
1026
1027 if ( first_block_in_scratch == block )
1028 {
1029 if ( valid_scratch_bytes <= offset )
1030 { /** EOF in remote file and nothing to read **/
1031 to_read_total = to_read = 0;
1032 }
1033 else
1034 {
1035 if ( to_read > valid_scratch_bytes - offset )
1036 { /** EOF in remote file something left**/
1037 to_read_total = to_read = valid_scratch_bytes - offset;
1038 }
1039 memmove( buffer, scratch_buffer + offset, to_read );
1040 }
1041
1042 /*** move source counters **/
1043 offset += to_read;
1044 block += offset / cself->block_size;
1045 offset %= cself->block_size;
1046
1047 /*** move output counters **/
1048 to_read_total -= to_read;
1049 *num_read += to_read;
1050 buffer = ((char*)buffer) + to_read;
1051 }
1052 else if ( IS_CACHE_BIT( cself, block ) )
1053 {
1054 uint64_t fpos = block * cself->block_size;
1055 if ( fpos < cself -> remote_size )
1056 {
1057 int64_t fbsize = cself -> remote_size - fpos;
1058 size_t nread = 0;
1059
1060 if( fbsize > cself->block_size )
1061 fbsize = cself -> block_size;
1062
1063 rc = KFileReadAll( cself->local, fpos, scratch_buffer, fbsize, &nread );
1064 if ( rc == 0 )
1065 {
1066 int i;
1067 uint64_t *b = ( uint64_t* )scratch_buffer;
1068 first_block_in_scratch = block;
1069 valid_scratch_bytes = nread;
1070
1071 if ( block != salvage_block )
1072 { /** check for fully space page, but don't do it in infinite loop **/
1073 for ( i = 0; i < ( nread/ sizeof( *b ) ) && b [ i]==0; i++ ) { }
1074 if ( i == ( nread / sizeof( *b ) ) )
1075 {
1076 rc = rd_remote_wr_local( cself, block*cself->block_size, scratch_buffer, fbsize, &nread );
1077 if ( rc == 0 )
1078 salvage_block = block;
1079 }
1080 else
1081 {
1082 salvage_block = -1;
1083 }
1084 }
1085 }
1086 }
1087 else
1088 {
1089 to_read_total = 0;
1090 }
1091 }
1092 else
1093 {
1094 uint64_t fpos = block * cself->block_size;
1095 if ( fpos < cself -> remote_size )
1096 {
1097 int64_t fbsize = cself->remote_size - fpos;
1098 size_t nread = 0;
1099
1100 if ( fbsize > cself->block_size )
1101 fbsize = cself->block_size;
1102
1103 rc = rd_remote_wr_local( cself, fpos, scratch_buffer, fbsize, &nread );
1104 if ( rc == 0 )
1105 {
1106 first_block_in_scratch = block;
1107 valid_scratch_bytes = nread;
1108 if ( !cself->local_read_only )
1109 {
1110 set_bitmap( cself, block, 1 );
1111 rc = write_bitmap( cself, block );
1112 }
1113 }
1114 }
1115 else
1116 {
1117 to_read_total = 0;
1118 }
1119 }
1120
1121 }
1122
1123 #if NO_SCRATCH_BUFFER
1124 #if USE_BUFFER_POOL
1125 if ( KQueuePush( cself -> buffer_pool, scratch_buffer, NULL ) != 0 )
1126 free ( scratch_buffer );
1127 #else
1128 free ( scratch_buffer );
1129 #endif
1130 #else
1131 ( ( KCacheTeeFile * )cself ) -> first_block_in_scratch = first_block_in_scratch;
1132 ( ( KCacheTeeFile * )cself ) -> valid_scratch_bytes = valid_scratch_bytes;
1133 #endif
1134
1135 return rc;
1136 }
1137
1138 #if 0
1139 /**********************************************************************************************
1140 try #3
1141 **********************************************************************************************/
1142 static rc_t KCacheTeeFileRead_3( const KCacheTeeFile *cself, uint64_t pos,
1143 void * buffer, size_t bsize, size_t *num_read )
1144 {
1145 rc_t rc = 0;
1146 uint64_t i_pos = pos;
1147 uint8_t * i_buffer = buffer;
1148 size_t still_to_read = bsize;
1149 size_t total_read = 0;
1150 size_t bytes_read;
1151
1152 /* we have to exclude that we are requesting beyond EOF, we can do that because we know the size
1153 of the file, the 'constructor' of the KCacheTeeFile has requested it and rejects construction
1154 if it cannot get this size */
1155 if ( ( i_pos + still_to_read ) > cself -> remote_size )
1156 {
1157 still_to_read = ( cself -> remote_size - i_pos );
1158 }
1159
1160 while ( rc == 0 && still_to_read > 0 )
1161 {
1162 uint64_t block = ( i_pos / cself->block_size );
1163 if ( IS_CACHE_BIT( cself, block ) )
1164 {
1165 /* i_pos is in a cached block... */
1166 size_t to_read = ( ( ( block + 1 ) * cself->block_size ) - i_pos );
1167 if ( to_read > still_to_read ) to_read = still_to_read;
1168 rc = KFileReadAll( cself->local, i_pos, i_buffer, to_read, &bytes_read );
1169 if ( rc == 0 )
1170 {
1171 /* check what we read from the local file is completely empty */
1172 size_t i = 0;
1173 while ( i_buffer[ i ] == 0 && i < bytes_read ) { i++; }
1174 if ( i == ( bytes_read + 1 ) )
1175 {
1176 /* we have to read from remote because this segment is zero!
1177 this is a fix for broken cache-files */
1178 cself->bitmap[ block >> 3 ] &= ~( BitNr2Mask[ block & 0x07 ] );
1179 rc = write_bitmap( cself, block );
1180 /* do not advance the buffer, because in the loop this will be read remotely now */
1181 }
1182 else
1183 {
1184 i_buffer += bytes_read;
1185 total_read += bytes_read;
1186 i_pos += bytes_read;
1187 still_to_read -= bytes_read;
1188 }
1189 }
1190 }
1191 else
1192 {
1193 /* i_pos is not in a cached block... */
1194 uint64_t block_pos = ( block * cself->block_size );
1195 uint64_t ofs = ( i_pos - block_pos );
1196 if ( ofs == 0 && still_to_read >= cself->block_size )
1197 {
1198 /* read the whole block remotely, this can return less than requested if last block...
1199 and it can return 0 bytes if the http-connection times out... */
1200 rc = KFileReadAll( cself->remote, i_pos, i_buffer, cself->block_size, &bytes_read );
1201 if ( rc == 0 )
1202 {
1203 if ( bytes_read == 0 )
1204 {
1205 still_to_read = 0; /* terminate the loop, return as much as read so far, return rc ==0 */
1206 }
1207 else
1208 {
1209 /* write it to the local file */
1210 size_t bytes_written;
1211 rc = KFileWriteAll( cself->local, i_pos, i_buffer, bytes_read, &bytes_written );
1212 if ( rc == 0 )
1213 {
1214 if ( bytes_written != bytes_read )
1215 {
1216 rc = RC ( rcFS, rcFile, rcWriting, rcTransfer, rcTooShort );
1217 }
1218 else
1219 {
1220 /* write the bitmap */
1221 set_bitmap( cself, block, 1 );
1222 rc = write_bitmap( cself, block );
1223 if ( rc == 0 )
1224 {
1225 i_buffer += bytes_read;
1226 total_read += bytes_read;
1227 i_pos += bytes_read;
1228 still_to_read -= bytes_read;
1229 }
1230 }
1231 }
1232 }
1233 }
1234 }
1235 else
1236 {
1237 /* use the scratch-buffer to read the whole block, copy the smaller part that was requested */
1238 if ( cself->scratch_size < cself->block_size )
1239 {
1240 rc = resize_scratch_buffer( cself, cself->block_size );
1241 }
1242 if ( rc == 0 )
1243 {
1244 /* read the whole block remotely, this can return less than requested if last block...
1245 and it can return 0 bytes if the http-connection times out... */
1246 rc = KFileReadAll( cself->remote, block_pos, cself->scratch_buffer, cself->block_size, &bytes_read );
1247 if ( rc == 0 )
1248 {
1249 if ( bytes_read == 0 )
1250 {
1251 still_to_read = 0; /* terminate the loop, return as much as read so far, return rc ==0 */
1252 }
1253 else
1254 {
1255 /* write it to the local file */
1256 size_t bytes_written;
1257 rc = KFileWriteAll( cself->local, block_pos, cself -> scratch_buffer, bytes_read, &bytes_written );
1258 if ( rc == 0 )
1259 {
1260 if ( bytes_written != bytes_read )
1261 {
1262 rc = RC ( rcFS, rcFile, rcWriting, rcTransfer, rcTooShort );
1263 }
1264 else
1265 {
1266 /* write the bitmap */
1267 set_bitmap( cself, block, 1 );
1268 rc = write_bitmap( cself, block );
1269 if ( rc == 0 )
1270 {
1271 /* here comes the difference: copy the bytes from the offset */
1272 size_t to_copy = ( cself->block_size - ofs );
1273 if ( to_copy > still_to_read ) to_copy = still_to_read;
1274 memmove ( i_buffer, &( cself->scratch_buffer[ ofs ] ), to_copy );
1275
1276 i_buffer += to_copy;
1277 total_read += to_copy;
1278 i_pos += to_copy;
1279 still_to_read -= to_copy;
1280 }
1281 }
1282 }
1283 }
1284 }
1285 }
1286 }
1287 }
1288 }
1289
1290 if ( rc == 0 )
1291 *num_read = total_read;
1292
1293 return rc;
1294 }
1295 #endif
1296
1297 /**********************************************************************************************
1298 START vt-functions
1299 **********************************************************************************************/
1300
KCacheTeeFileGetSysFile(const KCacheTeeFile * self,uint64_t * offset)1301 static struct KSysFile* KCacheTeeFileGetSysFile( const KCacheTeeFile *self, uint64_t *offset )
1302 {
1303 * offset = 0;
1304 return NULL;
1305 }
1306
1307
KCacheTeeFileRandomAccess(const KCacheTeeFile * self)1308 static rc_t KCacheTeeFileRandomAccess( const KCacheTeeFile *self )
1309 {
1310 return 0;
1311 }
1312
1313
KCacheTeeFileSize(const KCacheTeeFile * self,uint64_t * size)1314 static rc_t KCacheTeeFileSize( const KCacheTeeFile *self, uint64_t *size )
1315 {
1316 *size = self->remote_size;
1317 return 0;
1318 }
1319
1320
KCacheTeeFileSetSize(KCacheTeeFile * self,uint64_t size)1321 static rc_t KCacheTeeFileSetSize( KCacheTeeFile *self, uint64_t size )
1322 {
1323 return RC ( rcFS, rcFile, rcUpdating, rcFile, rcReadonly );
1324 }
1325
KCacheTeeFileRead(const KCacheTeeFile * cself,uint64_t pos,void * buffer,size_t bsize,size_t * num_read)1326 static rc_t KCacheTeeFileRead( const KCacheTeeFile *cself, uint64_t pos,
1327 void *buffer, size_t bsize, size_t *num_read )
1328 {
1329
1330 /* rc_t rc = KCacheTeeFileRead_3( cself, pos, buffer, bsize, num_read ); */
1331 rc_t rc = KCacheTeeFileRead_simple2( cself, pos, buffer, bsize, num_read );
1332
1333 #if( CACHE_STAT > 0 )
1334 write_cache_stat( & ( ( ( KCacheTeeFile * )cself ) -> stat ), pos, bsize, *num_read );
1335 #endif
1336
1337 return rc;
1338 }
1339
1340
KCacheTeeFileWrite(KCacheTeeFile * self,uint64_t pos,const void * buffer,size_t size,size_t * num_writ)1341 static rc_t KCacheTeeFileWrite( KCacheTeeFile *self, uint64_t pos,
1342 const void *buffer, size_t size, size_t *num_writ )
1343 {
1344 return RC ( rcFS, rcFile, rcUpdating, rcInterface, rcUnsupported );
1345 }
1346
1347 /**********************************************************************************************
1348 END vt-functions
1349 **********************************************************************************************/
1350
1351
1352 static KFile_vt_v1 vtKCacheTeeFile =
1353 {
1354 /* version 1.0 */
1355 1, 0,
1356
1357 /* start minor version 0 methods */
1358 KCacheTeeFileDestroy,
1359 KCacheTeeFileGetSysFile,
1360 KCacheTeeFileRandomAccess,
1361 KCacheTeeFileSize,
1362 KCacheTeeFileSetSize,
1363 KCacheTeeFileRead,
1364 KCacheTeeFileWrite
1365 /* end minor version 0 methods */
1366 };
1367
hand_out_remote_file_as_tee_file(struct KFile const ** tee,struct KFile const * remote)1368 static rc_t hand_out_remote_file_as_tee_file( struct KFile const **tee, struct KFile const *remote )
1369 {
1370 rc_t rc = KFileAddRef( remote );
1371 if ( rc == 0 )
1372 *tee = remote;
1373 return rc;
1374 }
1375
make_cache_tee(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,struct KFile * local,uint32_t blocksize,bool read_only,bool promote,const char * path)1376 static rc_t make_cache_tee( struct KDirectory *self, struct KFile const **tee,
1377 struct KFile const *remote, struct KFile *local, uint32_t blocksize, bool read_only, bool promote, const char *path )
1378 {
1379 rc_t rc;
1380 size_t path_size = string_size ( path );
1381 KCacheTeeFile * cf = malloc ( sizeof * cf + path_size + 1 );
1382 if ( cf == NULL )
1383 rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
1384 else
1385 {
1386 cf -> dir = self;
1387 string_copy( cf -> local_path, path_size + 1, path, path_size );
1388 cf -> remote = remote; /* store the file-objects but no AddRef()'s yet! */
1389 cf -> local = local;
1390 cf -> block_size = ( blocksize > 0 ) ? blocksize : CACHE_TEE_DEFAULT_BLOCKSIZE;
1391 cf -> bitmap = NULL;
1392 #if ! NO_SCRATCH_BUFFER
1393 cf -> scratch_buffer = NULL;
1394 cf -> scratch_size = 0;
1395 cf -> first_block_in_scratch = -1;
1396 cf -> valid_scratch_bytes = 0;
1397 #endif
1398 cf -> local_read_only = read_only;
1399 cf -> promote = promote;
1400
1401 #if( CACHE_STAT > 0 )
1402 init_cache_stat( & cf -> stat );
1403 #endif
1404
1405 rc = KFileSize( local, &cf -> local_size );
1406 if ( rc != 0 )
1407 {
1408 LOGERR( klogErr, rc, "cannot detect size of local file" );
1409 }
1410 else
1411 {
1412 bool fully_in_cache = false;
1413
1414 rc = KFileSize( cf -> remote, &cf -> remote_size );
1415 if ( rc != 0 )
1416 {
1417 LOGERR( klogErr, rc, "cannot detect size of remote file" );
1418 }
1419 else
1420 {
1421 if ( cf -> local_size == 0 && ! cf -> local_read_only )
1422 {
1423 /* the local file was just created... */
1424 if ( cf -> remote_size == 0 )
1425 {
1426 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
1427 LOGERR( klogErr, rc, "size of remote file is zero" );
1428 }
1429 else
1430 {
1431 rc = calculate_local_size_from_remote_size( cf );
1432 if ( rc == 0 )
1433 {
1434 rc = KFileSetSize ( cf -> local, cf -> local_size );
1435 if ( rc == 0 )
1436 {
1437 rc = init_new_local_file( cf );
1438 }
1439 else
1440 {
1441 PLOGERR( klogInt, ( klogInt, rc, "cannot size local file to $(l) bytes", "l=%lu", cf->local_size ) );
1442 }
1443 }
1444 }
1445 }
1446 else
1447 rc = verify_existing_local_file( cf, &fully_in_cache );
1448 }
1449
1450 if ( rc == 0 && fully_in_cache && ! cf -> local_read_only && cf -> promote )
1451 {
1452 /* here is the up-front-test: the cache is complete and we have write access! */
1453 rc = promote_cache( cf );
1454 if ( rc == 0 )
1455 {
1456 *tee = cf -> local;
1457 free ( cf );
1458 return rc; /* here we return the truncated, promoted cache as tee-file ! */
1459 }
1460 }
1461
1462 if ( rc == 0 )
1463 {
1464 if ( cf -> remote_size == 0 )
1465 {
1466 cf -> remote_size = cf -> local_size;
1467 }
1468
1469 /* now we have to AddRef() everything we hang on until the final release! */
1470 rc = KDirectoryAddRef ( cf -> dir );
1471 if ( rc == 0 )
1472 {
1473 rc = KFileAddRef( cf -> remote );
1474 if ( rc == 0 )
1475 {
1476 #if USE_BUFFER_POOL
1477 rc = KQueueMake( &cf -> buffer_pool, 32 );
1478 #endif
1479 if ( rc == 0 )
1480 {
1481 rc = KFileInit( &cf -> dad, (const union KFile_vt *)&vtKCacheTeeFile, "KCacheTeeFile", path, true, false );
1482 if ( rc == 0 )
1483 {
1484 /* the wrapper is ready to use now! */
1485 *tee = ( const KFile * ) &cf -> dad;
1486 return 0;
1487 }
1488 else
1489 {
1490 LOGERR( klogErr, rc, "cannot initialize KFile-structure" );
1491 #if USE_BUFFER_POOL
1492 KQueueRelease( cf -> buffer_pool );
1493 #endif
1494 /* TODO: check if we actually need to release cf->local here, since we never attached to it */
1495 KFileRelease( cf -> local );
1496 KFileRelease( cf -> remote );
1497 KDirectoryRelease ( cf -> dir );
1498 }
1499 }
1500 else
1501 {
1502 KFileRelease( cf -> remote );
1503 KDirectoryRelease ( cf -> dir );
1504 }
1505 }
1506 else
1507 KDirectoryRelease ( cf -> dir );
1508 }
1509 }
1510 }
1511 free ( cf );
1512 /* if we arrived here, we do not have enough space on the local filesystem */
1513 rc = hand_out_remote_file_as_tee_file( tee, remote );
1514 LOGERR( klogInt, rc, "skipping the cache-tee completely" );
1515 }
1516 return rc;
1517 }
1518
make_read_only_cache_tee(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path)1519 static rc_t make_read_only_cache_tee( struct KDirectory *self,
1520 struct KFile const **tee, struct KFile const *remote, uint32_t blocksize, const char *path )
1521 {
1522 const struct KFile * local;
1523 rc_t rc = KDirectoryOpenFileRead( self, &local, "%s.cache", path );
1524 if ( rc == 0 )
1525 rc = make_cache_tee( self, tee, remote, ( struct KFile * )local, blocksize, true, false, path );
1526 return rc;
1527 }
1528
1529
1530 static
KDirectoryVMakeCacheTeeInt(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path,va_list args,bool promote)1531 rc_t KDirectoryVMakeCacheTeeInt ( struct KDirectory *self,
1532 struct KFile const **tee, struct KFile const *remote,
1533 uint32_t blocksize, const char *path, va_list args, bool promote )
1534 {
1535 rc_t rc;
1536 if ( tee == NULL || remote == NULL )
1537 rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcNull );
1538 else
1539 {
1540 if ( self == NULL )
1541 rc = RC ( rcFS, rcFile, rcAllocating, rcSelf, rcNull );
1542 else if ( path == NULL )
1543 rc = RC ( rcFS, rcFile, rcAllocating, rcPath, rcNull );
1544 else if ( path [ 0 ] == 0 )
1545 rc = RC ( rcFS, rcFile, rcAllocating, rcPath, rcEmpty );
1546 else
1547 {
1548 char full [ 4096 ];
1549
1550 rc = KDirectoryVResolvePath ( self, false, full, sizeof full, path, args );
1551 if ( rc != 0 )
1552 {
1553 PLOGERR( klogErr, ( klogErr, rc, "cannot resolve path of cache file '$(path)'",
1554 "path=%s", full ) );
1555 }
1556 else
1557 {
1558 if ( rc == 0 )
1559 {
1560 /* it was possible to aquire the lock on the cache-file */
1561 struct KFile * local;
1562 rc = KDirectoryOpenFileSharedWrite( self, &local, true, "%s.cache", full );
1563 if ( rc == 0 )
1564 {
1565 /* we have the exclusive rd/wr access to the cache file !*/
1566 rc = make_cache_tee( self, tee, remote, local, blocksize, false, promote, full );
1567 }
1568 else if ( GetRCState( rc ) == rcNotFound )
1569 {
1570 rc = KDirectoryCreateFile( self, &local, true,
1571 0664, kcmOpen | kcmParents, "%s.cache", full );
1572 if ( rc == 0 )
1573 {
1574 /* we have the exclusive rd/wr access to the cache file !*/
1575 rc = make_cache_tee( self, tee, remote, local, blocksize, false, promote, full );
1576 }
1577 }
1578 else
1579 {
1580 /* we do not have the exclusive rd/wr access to the cache file !*/
1581 rc = make_read_only_cache_tee( self, tee, remote, blocksize, full );
1582 }
1583 }
1584 else if ( GetRCState ( rc ) == rcBusy )
1585 {
1586 /* it was NOT possible to aquire the lock on the cache-file */
1587 rc = make_read_only_cache_tee( self, tee, remote, blocksize, full );
1588 }
1589 else
1590 {
1591 PLOGERR( klogErr, ( klogErr, rc, "cannot create lock-file '$(path).cache.lock'",
1592 "path=%s", full ) );
1593 }
1594 }
1595 }
1596 }
1597 return rc;
1598 }
1599
1600
KDirectoryVMakeCacheTee(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path,va_list args)1601 LIB_EXPORT rc_t CC KDirectoryVMakeCacheTee ( struct KDirectory *self,
1602 struct KFile const **tee, struct KFile const *remote,
1603 uint32_t blocksize, const char *path, va_list args )
1604 {
1605 return KDirectoryVMakeCacheTeeInt ( self, tee, remote, blocksize, path, args, false );
1606 }
1607
1608
KDirectoryMakeCacheTee(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path,...)1609 LIB_EXPORT rc_t CC KDirectoryMakeCacheTee ( struct KDirectory *self,
1610 struct KFile const **tee, struct KFile const *remote,
1611 uint32_t blocksize, const char *path, ... )
1612 {
1613 rc_t rc;
1614 va_list args;
1615 va_start ( args, path );
1616
1617 rc = KDirectoryVMakeCacheTee ( self, tee, remote, blocksize, path, args );
1618
1619 va_end ( args );
1620
1621 return rc;
1622 }
1623
1624
KDirectoryVMakeCacheTeePromote(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path,va_list args)1625 LIB_EXPORT rc_t CC KDirectoryVMakeCacheTeePromote ( struct KDirectory *self,
1626 struct KFile const **tee, struct KFile const *remote,
1627 uint32_t blocksize, const char *path, va_list args )
1628 {
1629 return KDirectoryVMakeCacheTeeInt ( self, tee, remote, blocksize, path, args, true );
1630 }
1631
1632
KDirectoryMakeCacheTeePromote(struct KDirectory * self,struct KFile const ** tee,struct KFile const * remote,uint32_t blocksize,const char * path,...)1633 LIB_EXPORT rc_t CC KDirectoryMakeCacheTeePromote ( struct KDirectory *self,
1634 struct KFile const **tee, struct KFile const *remote,
1635 uint32_t blocksize, const char *path, ... )
1636 {
1637 rc_t rc;
1638 va_list args;
1639 va_start ( args, path );
1640
1641 rc = KDirectoryVMakeCacheTeePromote ( self, tee, remote, blocksize, path, args );
1642
1643 va_end ( args );
1644
1645 return rc;
1646 }
1647
1648
1649 #if USE_32BIT_BITMAP_WORDS
count_bits_in_bitmap(const uint64_t block_count,const uint64_t bitmap_bytes,const atomic32_t * bitmap)1650 static uint64_t count_bits_in_bitmap( const uint64_t block_count, const uint64_t bitmap_bytes, const atomic32_t * bitmap )
1651 {
1652 uint64_t res = 0;
1653 uint64_t idx;
1654 for ( idx = 0; idx < block_count; ++idx )
1655 {
1656 if ( IS_BITMAP_BIT( bitmap, idx ) )
1657 res++;
1658 }
1659 return res;
1660 }
1661 #else
count_bits_in_bitmap(const uint64_t block_count,const uint64_t bitmap_bytes,const uint8_t volatile * bitmap)1662 static uint64_t count_bits_in_bitmap( const uint64_t block_count, const uint64_t bitmap_bytes, const uint8_t volatile * bitmap )
1663 {
1664 uint64_t res = 0;
1665 uint64_t idx;
1666 for ( idx = 0; idx < block_count; ++idx )
1667 {
1668 if ( IS_BITMAP_BIT( bitmap, idx ) )
1669 res++;
1670 }
1671 return res;
1672 }
1673 #endif
1674
GetCacheCompleteness(const struct KFile * self,float * percent,uint64_t * bytes_in_cache)1675 LIB_EXPORT rc_t CC GetCacheCompleteness( const struct KFile * self, float * percent, uint64_t * bytes_in_cache )
1676 {
1677 rc_t rc;
1678 if ( self == NULL )
1679 rc = RC( rcFS, rcFile, rcValidating, rcSelf, rcNull );
1680 else
1681 {
1682 uint64_t local_size;
1683 if ( percent != NULL ) *percent = 0;
1684 if ( bytes_in_cache != NULL ) *bytes_in_cache = 0;
1685 rc = KFileSize( self, &local_size );
1686 if ( rc != 0 )
1687 {
1688 LOGERR( klogErr, rc, "cannot detect filesize when checking if cached file is complete" );
1689 }
1690 else
1691 {
1692 uint32_t block_size;
1693 rc = read_block_size( self, local_size, &block_size );
1694 if ( rc == 0 )
1695 {
1696 uint64_t content_size;
1697 rc = read_content_size( self, local_size, &content_size );
1698 /* create bitmap buffer */
1699 if ( rc == 0 )
1700 {
1701 uint64_t block_count;
1702 uint64_t bitmap_bytes;
1703 rc = verify_file_structure( local_size, block_size, content_size, &block_count, &bitmap_bytes );
1704 if ( rc == 0 )
1705 {
1706 #if USE_32BIT_BITMAP_WORDS
1707 atomic32_t * bitmap = NULL;
1708 #else
1709 uint8_t volatile * bitmap = NULL;
1710 #endif
1711 rc = create_bitmap_buffer( &bitmap, bitmap_bytes );
1712 if ( rc == 0 )
1713 {
1714 size_t num_read;
1715 /* read the bitmap into the memory ... */
1716 rc = KFileReadAll ( self, content_size, ( void * ) bitmap, bitmap_bytes, &num_read );
1717 if ( rc != 0 )
1718 {
1719 LOGERR( klogErr, rc, "cannot read bitmap from local file" );
1720 }
1721 else if ( num_read != bitmap_bytes )
1722 {
1723 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
1724 PLOGERR( klogErr, ( klogErr, rc, "cannot read $(ls) bitmap-bytes from local file, read $(rs) instead",
1725 "ls=%lu,rs=%lu", bitmap_bytes, num_read ));
1726 }
1727 }
1728 if ( rc == 0 )
1729 {
1730 uint64_t in_cache = count_bits_in_bitmap( block_count, bitmap_bytes, bitmap );
1731 if ( in_cache > 0 && block_count > 0 )
1732 {
1733 float res = ( float ) in_cache;
1734 res *= 100;
1735 res /= block_count;
1736 if ( percent != NULL ) ( *percent ) = res;
1737 if ( bytes_in_cache != NULL ) ( *bytes_in_cache ) = ( in_cache * block_size );
1738 }
1739 }
1740 if ( bitmap != NULL )
1741 free( ( void * ) bitmap );
1742 }
1743 }
1744 }
1745 }
1746 }
1747 return rc;
1748 }
1749
1750
1751 /* -----
1752 * examens the file, and reports the size of the original file ( without the cachefile-footer )
1753 *
1754 */
GetCacheTruncatedSize(const struct KFile * self,uint64_t * truncated_size)1755 LIB_EXPORT rc_t CC GetCacheTruncatedSize( const struct KFile * self, uint64_t * truncated_size )
1756 {
1757 rc_t rc;
1758 if ( self == NULL )
1759 rc = RC( rcFS, rcFile, rcValidating, rcSelf, rcNull );
1760 else if ( truncated_size == NULL )
1761 rc = RC( rcFS, rcFile, rcValidating, rcParam, rcNull );
1762 else
1763 {
1764 uint64_t local_size;
1765 *truncated_size = 0;
1766 rc = KFileSize( self, &local_size );
1767 if ( rc != 0 )
1768 {
1769 LOGERR( klogErr, rc, "cannot detect filesize when checking if cached file is complete" );
1770 }
1771 /* TODO: need to do a better constant tha a magic number */
1772 else if (local_size < 13)
1773 {
1774 rc = RC ( rcFS, rcFile, rcValidating, rcSize, rcInsufficient );
1775 LOGERR( klogErr, rc, "cannot use filesize of zero when checking if cached file is complete" );
1776 }
1777 else
1778 {
1779 uint32_t block_size;
1780 rc = read_block_size( self, local_size, &block_size );
1781 if ( rc == 0 )
1782 {
1783 uint64_t content_size;
1784 rc = read_content_size( self, local_size, &content_size );
1785 if ( rc == 0 )
1786 {
1787 uint64_t block_count, bitmap_bytes;
1788 rc = verify_file_structure( local_size, block_size, content_size, &block_count, &bitmap_bytes );
1789 if ( rc == 0 )
1790 {
1791 *truncated_size = content_size;
1792 }
1793 }
1794 }
1795 }
1796 }
1797 return rc;
1798 }
1799
Has_Cache_Zero_Blocks(const struct KFile * self,uint64_t * checked_blocks,uint64_t * empty_blocks)1800 LIB_EXPORT rc_t CC Has_Cache_Zero_Blocks( const struct KFile * self, uint64_t * checked_blocks, uint64_t * empty_blocks )
1801 {
1802 rc_t rc;
1803 if ( self == NULL )
1804 rc = RC( rcFS, rcFile, rcValidating, rcSelf, rcNull );
1805 else if ( checked_blocks == NULL || empty_blocks == NULL )
1806 rc = RC( rcFS, rcFile, rcValidating, rcParam, rcNull );
1807 else
1808 {
1809 uint64_t local_size;
1810 *checked_blocks = 0;
1811 *empty_blocks = 0;
1812 rc = KFileSize( self, &local_size );
1813 if ( rc != 0 )
1814 {
1815 LOGERR( klogErr, rc, "cannot detect filesize when checking if cached file is complete" );
1816 }
1817 else
1818 {
1819 uint32_t block_size;
1820 rc = read_block_size( self, local_size, &block_size );
1821 if ( rc == 0 )
1822 {
1823 uint64_t content_size;
1824 rc = read_content_size( self, local_size, &content_size );
1825 /* create bitmap buffer */
1826 if ( rc == 0 )
1827 {
1828 uint64_t block_count;
1829 uint64_t bitmap_bytes;
1830 rc = verify_file_structure( local_size, block_size, content_size, &block_count, &bitmap_bytes );
1831 if ( rc == 0 )
1832 {
1833 #if USE_32BIT_BITMAP_WORDS
1834 atomic32_t * bitmap = NULL;
1835 #else
1836 uint8_t volatile * bitmap = NULL;
1837 #endif
1838 rc = create_bitmap_buffer( &bitmap, bitmap_bytes );
1839 if ( rc == 0 )
1840 {
1841 size_t num_read;
1842 /* read the bitmap into the memory ... */
1843 rc = KFileReadAll ( self, content_size, ( void * ) bitmap, bitmap_bytes, &num_read );
1844 if ( rc != 0 )
1845 {
1846 LOGERR( klogErr, rc, "cannot read bitmap from local file" );
1847 }
1848 else if ( num_read != bitmap_bytes )
1849 {
1850 rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcInvalid );
1851 PLOGERR( klogErr, ( klogErr, rc, "cannot read $(ls) bitmap-bytes from local file, read $(rs) instead",
1852 "ls=%lu,rs=%lu", bitmap_bytes, num_read ));
1853 }
1854 }
1855 if ( rc == 0 )
1856 {
1857 /* loop through the bitmap for each bit set verify that the corresponding block is not zero */
1858 uint8_t * buffer = malloc( block_size );
1859 if ( buffer != NULL )
1860 {
1861 uint64_t idx;
1862 for ( idx = 0; idx < block_count && rc == 0; ++idx )
1863 {
1864 if ( IS_BITMAP_BIT( bitmap, idx ) )
1865 {
1866 size_t num_read;
1867 rc = KFileReadAll ( self, idx * block_size, buffer, block_size, &num_read );
1868 if ( rc == 0 )
1869 {
1870 ( *checked_blocks )++;
1871 if ( num_read > 0 )
1872 {
1873 size_t i, n;
1874 for ( i = 0, n = 0; i < num_read; ++i )
1875 if ( buffer[ i ] != 0 ) n++;
1876 if ( n == num_read ) ( * empty_blocks )++;
1877 }
1878 }
1879 }
1880 }
1881 free( buffer );
1882 }
1883 }
1884 if ( bitmap != NULL )
1885 free( ( void * ) bitmap );
1886 }
1887 }
1888 }
1889 }
1890
1891 }
1892 return rc;
1893 }
1894
1895
1896 /* -----
1897 * reports in the boolean if the file is a cachetoofile and it is complete...
1898 *
1899 * KFile is the struct returned by KDirectoryMakeCacheTee()..
1900 */
IsCacheTeeComplete(const struct KFile * self,bool * complete)1901 LIB_EXPORT rc_t CC IsCacheTeeComplete( const struct KFile * self, bool * complete )
1902 {
1903 rc_t rc = 0;
1904 if ( self == NULL || complete == NULL )
1905 rc = RC ( rcFS, rcFile, rcValidating, rcParam, rcNull );
1906 else
1907 {
1908 if ( &self->vt->v1 != &vtKCacheTeeFile )
1909 rc = RC ( rcFS, rcFile, rcValidating, rcParam, rcInvalid );
1910 else
1911 {
1912 struct KCacheTeeFile * ctf = ( struct KCacheTeeFile * )self;
1913 *complete = is_bitmap_full( ctf->bitmap, ctf->bitmap_bytes, ctf->block_count );
1914 }
1915 }
1916 return rc;
1917 }
1918
KFileIsKCacheTeeFile(const struct KFile * self)1919 LIB_EXPORT bool CC KFileIsKCacheTeeFile( const struct KFile * self )
1920 {
1921 return self != NULL && &self->vt->v1 == &vtKCacheTeeFile;
1922 }
1923