1 /*===========================================================================
2 *
3 * PUBLIC DOMAIN NOTICE
4 * National Center for Biotechnology Information
5 *
6 * This software/database is a "United States Government Work" under the
7 * terms of the United States Copyright Act. It was written as part of
8 * the author's official duties as a United States Government employee and
9 * thus cannot be copyrighted. This software/database is freely available
10 * to the public for use. The National Library of Medicine and the U.S.
11 * Government have not placed any restriction on its use or reproduction.
12 *
13 * Although all reasonable efforts have been taken to ensure the accuracy
14 * and reliability of the software and data, the NLM and the U.S.
15 * Government do not and cannot warrant the performance or results that
16 * may be obtained by using this software or data. The NLM and the U.S.
17 * Government disclaim all warranties, express or implied, including
18 * warranties of performance, merchantability or fitness for any particular
19 * purpose.
20 *
21 * Please cite the author in any work or product based on this material.
22 *
23 * ===========================================================================
24 *
25 */
26
27 #include <kapp/main.h>
28 #include <kapp/args.h>
29
30 #include <klib/out.h>
31 #include <klib/log.h>
32 #include <klib/text.h>
33 #include <klib/rc.h>
34 #include <klib/time.h>
35
36 #include <kfs/directory.h>
37 #include <kfs/file.h>
38 #include <kfs/buffile.h>
39 #include <kfs/cacheteefile.h>
40 //#include <kfs/big_block_reader.h>
41
42 #include <kns/manager.h>
43 #include <kns/kns-mgr-priv.h>
44 #include <kns/http.h>
45 #include <kns/stream.h>
46
47 #include <kproc/timeout.h>
48
49 #include <os-native.h>
50 #include <sysalloc.h>
51
52 #include <stdio.h>
53 #include <string.h>
54 #include <stdlib.h>
55
56 /*===========================================================================
57
58 kget is a works like a simple version of wget
59
60 =========================================================================== */
61
62 const char UsageDefaultName[] = "kget";
63
UsageSummary(const char * progname)64 rc_t CC UsageSummary ( const char * progname )
65 {
66 return KOutMsg ("\n"
67 "Usage:\n"
68 " %s <path> [<path> ...] [options]\n"
69 "\n", progname);
70 }
71
72
73 #define OPTION_VERB "verb"
74 #define ALIAS_VERB "b"
75 static const char * verb_usage[] = { "execute verbose", NULL };
76
77 #define OPTION_BLOCK "block-size"
78 #define ALIAS_BLOCK "s"
79 static const char * block_usage[] = { "how many bytes per block", NULL };
80
81 #define OPTION_SHOW "show-size"
82 #define ALIAS_SHOW "w"
83 static const char * show_usage[] = { "query size of remote file first", NULL };
84
85 #define OPTION_CACHE "cache"
86 #define ALIAS_CACHE "c"
87 static const char * cache_usage[] = { "wrap the remote-file into a KCacheTeeFile", NULL };
88
89 #define OPTION_CACHE_BLK "cache-block"
90 static const char * cache_blk_usage[] = { "blocksize inside a KCacheTeeFile", NULL };
91
92 #define OPTION_PROXY "proxy"
93 static const char * proxy_usage[] = { "use a proxy to download remote file", NULL };
94
95 #define OPTION_RAND "random"
96 #define ALIAS_RAND "r"
97 static const char * random_usage[] = { "request blocks in random order", NULL };
98
99 #define OPTION_REP "repeat"
100 #define ALIAS_REP "e"
101 static const char * repeat_usage[] = { "request blocks with repeats if in random order", NULL };
102
103 #define OPTION_CREPORT "report"
104 #define ALIAS_CREPORT "p"
105 static const char * creport_usage[] = { "report cache usage", NULL };
106
107 #define OPTION_COMPLETE "complete"
108 static const char * complete_usage[] = { "check if 1st parameter is a complete", NULL };
109
110 #define OPTION_TRUNC "truncate"
111 static const char * truncate_usage[] = { "truncate the file ( 1st parameter ) / remove trailing cache-bitmap", NULL };
112
113 #define OPTION_START "start"
114 static const char * start_usage[] = { "offset where to read from", NULL };
115
116 #define OPTION_COUNT "count"
117 static const char * count_usage[] = { "number of bytes to read", NULL };
118
119 #define OPTION_PROGRESS "progress"
120 static const char * progress_usage[] = { "show progress", NULL };
121
122 #define OPTION_RELIABLE "reliable"
123 static const char * reliable_usage[] = { "use reliable version of http-file", NULL };
124
125 #define OPTION_BUFFER "buffer"
126 #define ALIAS_BUFFER "u"
127 static const char * buffer_usage[] = { "wrap remote file into KBufFile with this buffer-size", NULL };
128
129 #define OPTION_SLEEP "sleep"
130 #define ALIAS_SLEEP "i"
131 static const char * sleep_usage[] = { "sleep inbetween requests by this amount of ms", NULL };
132
133 #define OPTION_TIMEOUT "timeout"
134 #define ALIAS_TIMEOUT "m"
135 static const char * timeout_usage[] = { "use timed read with tis amount of ms as timeout", NULL };
136
137 #define OPTION_CCOMPL "cache-complete"
138 #define ALIAS_CCOMPL "a"
139 static const char * ccompl_usage[] = { "check completeness on open cacheteefile", NULL };
140
141 #define OPTION_FULL "full"
142 #define ALIAS_FULL "f"
143 static const char * full_usage[] = { "download via one http-request, not partial requests in a loop", NULL };
144
145 OptDef MyOptions[] =
146 {
147 /* name alias fkt usage-txt, cnt, needs value, required */
148 { OPTION_VERB, ALIAS_VERB, NULL, verb_usage, 1, false, false },
149 { OPTION_BLOCK, ALIAS_BLOCK, NULL, block_usage, 1, true, false },
150 { OPTION_SHOW, ALIAS_SHOW, NULL, show_usage, 1, false, false },
151 { OPTION_CACHE, ALIAS_CACHE, NULL, cache_usage, 1, true, false },
152 { OPTION_CACHE_BLK, NULL, NULL, cache_blk_usage, 1, true, false },
153 { OPTION_PROXY, NULL, NULL, proxy_usage, 1, true, false },
154 { OPTION_RAND, ALIAS_RAND, NULL, random_usage, 1, false, false },
155 { OPTION_REP, ALIAS_REP, NULL, repeat_usage, 1, false, false },
156 { OPTION_CREPORT, ALIAS_CREPORT, NULL, creport_usage, 1, false, false },
157 { OPTION_BUFFER, ALIAS_BUFFER, NULL, buffer_usage, 1, true, false },
158 { OPTION_SLEEP, ALIAS_SLEEP, NULL, sleep_usage, 1, true, false },
159 { OPTION_TIMEOUT, ALIAS_TIMEOUT, NULL, timeout_usage, 1, true, false },
160 { OPTION_COMPLETE, NULL, NULL, complete_usage, 1, false, false },
161 { OPTION_CCOMPL, ALIAS_CCOMPL, NULL, ccompl_usage, 1, false, false },
162 { OPTION_TRUNC, NULL, NULL, truncate_usage, 1, false, false },
163 { OPTION_START, NULL, NULL, start_usage, 1, true, false },
164 { OPTION_COUNT, NULL, NULL, count_usage, 1, true, false },
165 { OPTION_PROGRESS, NULL, NULL, progress_usage, 1, false, false },
166 { OPTION_RELIABLE, NULL, NULL, reliable_usage, 1, false, false },
167 { OPTION_FULL, ALIAS_FULL, NULL, full_usage, 1, false, false }
168 };
169
Usage(const Args * args)170 rc_t CC Usage ( const Args * args )
171 {
172 const char * progname = UsageDefaultName;
173 const char * fullpath = UsageDefaultName;
174 uint32_t i;
175 rc_t rc;
176
177 if ( args == NULL )
178 rc = RC ( rcApp, rcArgv, rcAccessing, rcSelf, rcNull );
179 else
180 rc = ArgsProgram ( args, &fullpath, &progname );
181
182 if ( rc )
183 progname = fullpath = UsageDefaultName;
184
185 UsageSummary ( progname );
186
187 KOutMsg ( "Options:\n" );
188
189 HelpOptionsStandard ();
190 for ( i = 0; i < sizeof ( MyOptions ) / sizeof ( MyOptions[ 0 ] ); ++i )
191 HelpOptionLine ( MyOptions[i].aliases, MyOptions[i].name, NULL, MyOptions[i].help );
192
193 return rc;
194 }
195
196 typedef struct fetch_ctx
197 {
198 const char *url;
199 const char *destination;
200 const char *cache_file;
201 const char * proxy;
202 size_t blocksize;
203 size_t start, count;
204 size_t buffer_size;
205 size_t sleep_time;
206 size_t timeout_time;
207 size_t cache_blk;
208 bool verbose;
209 bool show_filesize;
210 bool random;
211 bool with_repeats;
212 bool show_curl_version;
213 bool report_cache;
214 bool check_cache_complete;
215 bool check_completeness;
216 bool truncate_cache;
217 bool local_read_only;
218 bool show_progress;
219 bool reliable;
220 bool full_download;
221 } fetch_ctx;
222
223
src_2_dst(const KFile * src,KFile * dst,char * buffer,uint64_t pos,size_t * num_read,fetch_ctx * ctx)224 static rc_t src_2_dst( const KFile *src, KFile *dst, char * buffer,
225 uint64_t pos, size_t * num_read, fetch_ctx * ctx )
226 {
227 rc_t rc;
228 size_t n_transfer = ( ctx->count == 0 ? ctx->blocksize : ctx->count );
229
230 if ( ctx->timeout_time == 0 )
231 rc = KFileReadAll ( src, pos, buffer, n_transfer, num_read );
232 else
233 {
234 timeout_t tm;
235 rc = TimeoutInit ( &tm, ctx->timeout_time );
236 if ( rc == 0 )
237 rc = KFileTimedReadAll ( src, pos, buffer, n_transfer, num_read, &tm );
238 }
239 if ( rc == 0 && *num_read > 0 )
240 {
241 size_t num_writ;
242 rc = KFileWriteAll ( dst, pos, buffer, *num_read, &num_writ );
243 }
244 return rc;
245 }
246
247
block_loop_in_order(const KFile * src,KFile * dst,char * buffer,uint64_t * bytes_copied,fetch_ctx * ctx)248 static rc_t block_loop_in_order( const KFile *src, KFile *dst, char * buffer,
249 uint64_t * bytes_copied, fetch_ctx * ctx )
250 {
251 rc_t rc = 0;
252 uint64_t pos = 0;
253 uint32_t blocks = 0;
254 size_t num_read = 1;
255 KOutMsg( "copy-mode : linear read/write\n" );
256 while ( rc == 0 && num_read > 0 )
257 {
258 rc = src_2_dst( src, dst, buffer, pos, &num_read, ctx );
259 if ( rc == 0 ) pos += num_read;
260 if ( ctx->show_progress && ( ( blocks & 0x0F ) == 0 ) ) KOutMsg( "." );
261 blocks++;
262 if ( ctx->sleep_time > 0 ) KSleepMs( ctx->sleep_time );
263 }
264 *bytes_copied = pos;
265 if ( ctx->show_progress ) KOutMsg( "\n" );
266 KOutMsg( "%d blocks a %d bytes\n", blocks, ctx->blocksize );
267 return rc;
268 }
269
270
271
randr(uint32_t min,uint32_t max)272 static uint32_t randr( uint32_t min, uint32_t max )
273 {
274 double scaled = ( ( double )rand() / RAND_MAX );
275 return ( ( max - min + 1 ) * scaled ) + min;
276 }
277
278
block_loop_random(const KFile * src,KFile * dst,char * buffer,uint64_t * bytes_copied,fetch_ctx * ctx)279 static rc_t block_loop_random( const KFile *src, KFile *dst, char * buffer,
280 uint64_t *bytes_copied, fetch_ctx * ctx )
281 {
282 uint64_t src_size;
283 rc_t rc = KFileSize ( src, &src_size );
284 KOutMsg( "copy-mode : random blocks\n" );
285 if ( rc == 0 )
286 {
287 rc = KFileSetSize ( dst, src_size );
288 if ( rc == 0 )
289 {
290 uint32_t block_count = ( src_size / ctx->blocksize ) + 1;
291 uint32_t * block_vector = malloc( block_count * ( sizeof * block_vector ) );
292 if ( block_vector != NULL )
293 {
294 uint32_t loop;
295
296 /* fill the block_vector with ascending numbers */
297 for ( loop = 0; loop < block_count; loop++ )
298 block_vector[ loop ] = loop;
299
300 /* randomize them */
301 for ( loop = 0; loop < block_count; loop++ )
302 {
303 uint32_t src_idx = randr( 0, block_count - 1 );
304 uint32_t dst_idx = randr( 0, block_count - 1 );
305 /* swap it... */
306 uint32_t tmp = block_vector[ dst_idx ];
307 block_vector[ dst_idx ] = block_vector[ src_idx ];
308 block_vector[ src_idx ] = tmp;
309 }
310
311 for ( loop = 0; rc == 0 && loop < block_count; loop++ )
312 {
313 size_t num_read;
314 uint64_t pos = ctx->blocksize;
315 pos *= block_vector[ loop ];
316 rc = src_2_dst( src, dst, buffer, pos, &num_read, ctx );
317 if ( rc == 0 ) *bytes_copied += num_read;
318 if ( ctx->show_progress && ( ( loop & 0x0F ) == 0 ) ) KOutMsg( "." );
319 if ( ctx->sleep_time > 0 ) KSleepMs( ctx->sleep_time );
320
321 }
322 free( block_vector );
323 if ( ctx->show_progress ) KOutMsg( "\n" );
324 KOutMsg( "%d blocks a %d bytes\n", loop, ctx->blocksize );
325 }
326 }
327 }
328 return rc;
329 }
330
331
copy_file(const KFile * src,KFile * dst,fetch_ctx * ctx)332 static rc_t copy_file( const KFile * src, KFile * dst, fetch_ctx * ctx )
333 {
334 rc_t rc = 0;
335 size_t buffer_size = ( ctx->count == 0 ? ctx->blocksize : ctx->count );
336 char * buffer = malloc( buffer_size );
337 if ( buffer == NULL )
338 {
339 rc = RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted );
340 KOutMsg( "cant make buffer of size %u\n", buffer_size );
341 }
342 else
343 {
344 uint64_t bytes_copied = 0;
345 if ( ctx->count == 0 )
346 {
347 if ( ctx->random )
348 rc = block_loop_random( src, dst, buffer, &bytes_copied, ctx );
349 else
350 rc = block_loop_in_order( src, dst, buffer, &bytes_copied, ctx );
351 }
352 else
353 {
354 size_t num_read;
355 rc = src_2_dst( src, dst, buffer, ctx->start, &num_read, ctx );
356 if ( rc == 0 ) bytes_copied = num_read;
357 }
358 KOutMsg( "%lu bytes copied\n", bytes_copied );
359 free( buffer );
360 }
361 return rc;
362 }
363
364
365 #define CACHE_TEE_DEFAULT_BLOCKSIZE ( 32 * 1024 * 4 )
366
fetch_cached(KDirectory * dir,const KFile * src,KFile * dst,fetch_ctx * ctx)367 static rc_t fetch_cached( KDirectory *dir, const KFile *src, KFile *dst, fetch_ctx *ctx )
368 {
369 size_t bs = ctx->cache_blk == 0 ? CACHE_TEE_DEFAULT_BLOCKSIZE : ctx->cache_blk;
370 rc_t rc = KOutMsg( "persistent cache created : '%s' (blk-size: %d)\n", ctx->cache_file, bs );
371 if ( rc == 0 )
372 {
373 const KFile *tee; /* this is the file that forks persistent_content with remote */
374 rc = KDirectoryMakeCacheTee ( dir, /* the KDirectory for the the sparse-file */
375 &tee, /* the newly created cache-tee-file */
376 src, /* the file that we are wrapping ( usually the remote http-file ) */
377 ctx->cache_blk, /* how big one block in the cache-tee-file will be */
378 ctx->cache_file ); /* the sparse-file we use write to */
379 if ( rc == 0 )
380 {
381 KOutMsg( "cache tee created\n" );
382 rc = copy_file( tee, dst, ctx );
383 KFileRelease( tee );
384 }
385 }
386 return rc;
387 }
388
389
extract_name(char ** dst,const char * url)390 static void extract_name( char ** dst, const char * url )
391 {
392 char * last_slash = string_rchr ( url, string_size( url ), '/' );
393 if ( last_slash == NULL )
394 *dst = string_dup_measure( "out.bin", NULL );
395 else {
396 size_t size = 0;
397 *dst = string_dup_measure( last_slash + 1, &size );
398 if ( *dst != NULL ) {
399 char * query = string_chr ( *dst, size, '?' );
400 if ( query != NULL )
401 *query = '\0';
402 }
403 }
404 }
405
406
fetch_from(KDirectory * dir,fetch_ctx * ctx,char * outfile,const KFile * src)407 static rc_t fetch_from( KDirectory *dir, fetch_ctx *ctx, char * outfile,
408 const KFile * src )
409 {
410 uint64_t file_size;
411 rc_t rc = KFileSize( src, &file_size );
412 if ( rc != 0 )
413 KOutMsg( "cannot disover src-size >%R<\n", rc );
414 else
415 {
416 KFile *dst;
417 KOutMsg( "src-size = %lu\n", file_size );
418 rc = KDirectoryCreateFile ( dir, &dst, false, 0664, kcmInit, outfile );
419 if ( rc == 0 )
420 {
421 KOutMsg( "dst >%s< created\n", outfile );
422 if ( rc == 0 )
423 {
424 if ( ctx->cache_file != NULL )
425 rc = fetch_cached( dir, src, dst, ctx );
426 else
427 rc = copy_file( src, dst, ctx );
428 }
429 KFileRelease( dst );
430 }
431 }
432 return rc;
433 }
434
435
make_remote_file(struct KNSManager * kns_mgr,const KFile ** src,fetch_ctx * ctx)436 static rc_t make_remote_file( struct KNSManager * kns_mgr, const KFile ** src, fetch_ctx * ctx )
437 {
438 rc_t rc;
439
440 KNSManagerSetVerbose( kns_mgr, ctx->verbose );
441 if ( ctx->reliable )
442 /*TODO: supply ceRequired and payRequired */
443 rc = KNSManagerMakeReliableHttpFile( kns_mgr, src, NULL, 0x01010000, true, false, false, ctx->url );
444 else
445 rc = KNSManagerMakeHttpFile( kns_mgr, src, NULL, 0x01010000,
446 "%s", ctx->url );
447
448 if ( rc != 0 )
449 {
450 if ( ctx->reliable )
451 (void)LOGERR( klogInt, rc, "KNSManagerMakeReliableHttpFile() failed" );
452 else
453 (void)LOGERR( klogInt, rc, "KNSManagerMakeHttpFile() failed" );
454 }
455 else
456 {
457 if ( ctx->buffer_size > 0 )
458 {
459 const KFile * temp_file;
460 rc = KBufFileMakeRead ( & temp_file, *src, ctx->buffer_size );
461 if ( rc == 0 )
462 {
463 KOutMsg( "remote-file wrapped in new big-block-reader of size %d\n", ctx->buffer_size );
464 KFileRelease ( *src );
465 *src = temp_file;
466 }
467 }
468 }
469 return rc;
470 }
471
472
fetch(KDirectory * dir,fetch_ctx * ctx)473 static rc_t fetch( KDirectory *dir, fetch_ctx *ctx )
474 {
475 rc_t rc = 0;
476 char * outfile;
477 const KFile * remote;
478 struct KNSManager * kns_mgr;
479
480 if ( ctx->destination == NULL )
481 extract_name( &outfile, ctx->url );
482 else
483 outfile = string_dup_measure( ctx->destination, NULL );
484
485 assert(outfile);
486
487 KOutMsg( "fetching: >%s<\n", ctx->url );
488 KOutMsg( "into : >%s<\n", outfile );
489 if ( ctx->count > 0 )
490 KOutMsg( "range : %u.%u\n", ctx->start, ctx->count );
491
492 rc = KNSManagerMake ( &kns_mgr );
493 if ( rc != 0 )
494 (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
495 else
496 {
497 if ( ctx->proxy != NULL )
498 {
499 rc = KNSManagerSetHTTPProxyPath( kns_mgr, "%s", ctx->proxy );
500 if ( rc != 0 )
501 (void)LOGERR( klogInt, rc, "KNSManagerSetHTTPProxyPath() failed" );
502 }
503 if ( rc == 0 )
504 {
505 rc = make_remote_file( kns_mgr, &remote, ctx );
506 if ( rc == 0 )
507 {
508 rc = fetch_from( dir, ctx, outfile, remote );
509 KFileRelease( remote );
510 }
511 }
512 KNSManagerRelease( kns_mgr );
513 }
514
515 free( outfile );
516 return rc;
517 }
518
519
520 /* -------------------------------------------------------------------------------------------------------------------- */
521
522
show_size(KDirectory * dir,fetch_ctx * ctx)523 static rc_t show_size( KDirectory *dir, fetch_ctx *ctx )
524 {
525 rc_t rc = KOutMsg( "source: >%s<\n", ctx->url );
526 if ( rc == 0 )
527 {
528 struct KNSManager * kns_mgr;
529 rc = KNSManagerMake ( &kns_mgr );
530 if ( rc != 0 )
531 (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
532 else
533 {
534 const KFile * remote;
535 rc = make_remote_file( kns_mgr, &remote, ctx );
536 if ( rc == 0 )
537 {
538 uint64_t file_size;
539 rc = KFileSize( remote, &file_size );
540 KOutMsg( "file-size = %u\n", file_size );
541 KFileRelease( remote );
542 }
543 KNSManagerRelease( kns_mgr );
544 }
545 }
546 return rc;
547 }
548
549
550 /* -------------------------------------------------------------------------------------------------------------------- */
551
552
553 /* check cache completeness on raw file in the filesystem */
check_cache_complete(KDirectory * dir,fetch_ctx * ctx)554 static rc_t check_cache_complete( KDirectory *dir, fetch_ctx *ctx )
555 {
556 rc_t rc = KOutMsg( "checking if this cache file >%s< is complete\n", ctx->url );
557 if ( rc == 0 )
558 {
559 const KFile *f;
560 rc = KDirectoryOpenFileRead( dir, &f, "%s", ctx->url );
561 if ( rc == 0 )
562 {
563 bool is_complete;
564 rc = IsCacheFileComplete( f, &is_complete );
565 if ( rc != 0 )
566 KOutMsg( "error performing IsCacheFileComplete() %R\n", rc );
567 else
568 {
569 if ( is_complete )
570 KOutMsg( "the file is complete\n" );
571 else
572 {
573 float percent = 0;
574 uint64_t bytes_cached;
575 rc = GetCacheCompleteness( f, &percent, &bytes_cached );
576 if ( rc == 0 )
577 KOutMsg( "the file is %f%% complete ( %lu bytes are cached )\n", percent, bytes_cached );
578 }
579 }
580 KFileRelease( f );
581 }
582 }
583 return rc;
584 }
585
586
587 /* -------------------------------------------------------------------------------------------------------------------- */
588
589
fetch_loop(const KFile * src,fetch_ctx * ctx)590 static rc_t fetch_loop( const KFile * src, fetch_ctx *ctx )
591 {
592 rc_t rc = 0;
593 size_t buffer_size = ( ctx->count == 0 ? ctx->blocksize : ctx->count );
594 char * buffer = malloc( buffer_size );
595 if ( buffer == NULL )
596 {
597 rc = RC( rcExe, rcFile, rcPacking, rcMemory, rcExhausted );
598 KOutMsg( "cant make buffer of size %u\n", buffer_size );
599 }
600 else
601 {
602 uint64_t pos = 0;
603 size_t num_read = 0;
604 do
605 {
606 rc = KFileReadAll( src, pos, buffer, buffer_size, &num_read );
607 if ( rc == 0 )
608 pos += num_read;
609 } while ( rc == 0 && num_read > 0 );
610 KOutMsg( "%lu bytes copied\n", pos );
611 free( buffer );
612 }
613 return rc;
614 }
615
616
617 /* check cache completeness on a open cacheteefile */
check_cache_completeness(KDirectory * dir,fetch_ctx * ctx)618 static rc_t check_cache_completeness( KDirectory *dir, fetch_ctx *ctx )
619 {
620 rc_t rc = KOutMsg( "check if IsCacheTeeComplete() works as intended\n" );
621 if ( rc == 0 )
622 {
623 struct KNSManager * kns_mgr;
624 rc = KNSManagerMake ( &kns_mgr );
625 if ( rc != 0 )
626 (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
627 else
628 {
629 const KFile * remote;
630 rc = make_remote_file( kns_mgr, &remote, ctx );
631 if ( rc == 0 )
632 {
633 const KFile *tee; /* this is the file that forks persistent_content with remote */
634 rc = KDirectoryMakeCacheTee ( dir, /* the KDirectory for the the sparse-file */
635 &tee, /* the newly created cache-tee-file */
636 remote, /* the file that we are wrapping ( usually the remote http-file ) */
637 ctx->cache_blk, /* how big one block in the cache-tee-file will be */
638 ctx->cache_file ); /* the sparse-file we use write to */
639 if ( rc == 0 )
640 rc = fetch_loop( tee, ctx );
641 if ( rc == 0 )
642 {
643
644 bool complete = false;
645 rc = IsCacheTeeComplete( tee, &complete );
646 KOutMsg( "IsCacheTeeComplete() -> %R, complete = %s\n", rc, complete ? "YES" : "NO" );
647 KFileRelease( tee );
648 }
649 KFileRelease( remote );
650 }
651 KNSManagerRelease( kns_mgr );
652 }
653 }
654 return rc;
655 }
656
657
658 /* -------------------------------------------------------------------------------------------------------------------- */
659
660
661 /* check cache completeness on a open cacheteefile */
full_download(KDirectory * dir,fetch_ctx * ctx)662 static rc_t full_download( KDirectory *dir, fetch_ctx *ctx )
663 {
664 rc_t rc = KOutMsg( "make full download without partial access\n" );
665 if ( rc == 0 )
666 {
667 struct KNSManager * kns_mgr;
668 rc = KNSManagerMake ( &kns_mgr );
669 if ( rc != 0 )
670 (void)LOGERR( klogInt, rc, "KNSManagerMake() failed" );
671 else
672 {
673 struct URLBlock url;
674 URLBlockInit( &url );
675 rc = ParseUrl( &url, ctx->url, string_size( ctx->url ) );
676 if ( rc == 0 )
677 {
678 KClientHttp * http;
679 rc = url.tls ?
680 KNSManagerMakeClientHttps( kns_mgr, &http, NULL, 0x01010000, &url.host, url.port ):
681 KNSManagerMakeClientHttp( kns_mgr, &http, NULL, 0x01010000, &url.host, url.port );
682 if ( rc == 0 )
683 {
684 KClientHttpRequest * req;
685 KOutMsg( "connection open!\n" );
686 rc = KClientHttpMakeRequest( http, &req, ctx->url );
687 if ( rc == 0 )
688 {
689 KClientHttpResult *rslt;
690
691 KOutMsg( "request made!\n" );
692
693 KClientHttpRequestConnection( req, true );
694 KClientHttpRequestSetNoCache( req );
695
696 rc = KClientHttpRequestGET( req, &rslt );
697 if ( rc == 0 )
698 {
699 uint32_t result_code;
700 size_t msg_size;
701 char buffer[ 4096 * 32 ]; /* 128k */
702
703 KOutMsg( "reply received!\n" );
704 rc = KClientHttpResultStatus( rslt, &result_code, buffer, sizeof buffer, &msg_size );
705 if ( rc == 0 )
706 {
707 struct KStream *content;
708 KOutMsg( "result-code = %d\n", result_code );
709 if ( result_code == 200 )
710 {
711 rc = KClientHttpResultGetInputStream( rslt, &content );
712 if ( rc == 0 )
713 {
714 KFile *dst;
715 char * outfile;
716
717 if ( ctx->destination == NULL )
718 extract_name( &outfile, ctx->url );
719 else
720 string_dup_measure( ctx->destination, NULL );
721
722 rc = KDirectoryCreateFile ( dir, &dst, false, 0664, kcmInit, outfile );
723 if ( rc == 0 )
724 {
725 KOutMsg( "dst >%s< created\n", outfile );
726 if ( rc == 0 )
727 {
728 uint64_t pos = 0;
729 size_t num_read;
730 struct timeout_t timeout;
731
732 TimeoutInit( &timeout, 2000 );
733 do
734 {
735 rc = KStreamTimedRead( content, buffer, sizeof buffer, &num_read, &timeout );
736 if ( rc == 0 )
737 {
738 size_t num_writ;
739 rc = KFileWriteAll( dst, pos, buffer, num_read, &num_writ );
740 pos += num_read;
741 }
742 } while ( rc == 0 && num_read > 0 );
743
744 KOutMsg( "%d bytes read!\n", pos );
745 }
746 KFileRelease( dst );
747 }
748 free( outfile );
749 KStreamRelease( content );
750 }
751 }
752 }
753 KClientHttpResultRelease( rslt );
754 }
755 KClientHttpRequestRelease( req );
756 }
757 KClientHttpRelease ( http );
758 }
759 }
760 KNSManagerRelease( kns_mgr );
761 }
762 }
763 return rc;
764 }
765
766
767 /* -------------------------------------------------------------------------------------------------------------------- */
768
769
truncate_cache(KDirectory * dir,fetch_ctx * ctx)770 static rc_t truncate_cache( KDirectory *dir, fetch_ctx *ctx )
771 {
772 rc_t rc = 0;
773 KFile *f;
774
775 KOutMsg( "truncating this cache file >%s< (from it's trailing bitmap)\n", ctx->url );
776
777 rc = KDirectoryOpenFileWrite( dir, &f, true, "%s", ctx->url );
778 if ( rc == 0 )
779 {
780 rc = TruncateCacheFile( f );
781 if ( rc == 0 )
782 KOutMsg( "the file was truncated\n" );
783 else
784 KOutMsg( "the file was not truncated: %R\n", rc );
785 KFileRelease( f );
786 }
787 return rc;
788 }
789
790
791 /* -------------------------------------------------------------------------------------------------------------------- */
792
793
get_bool(Args * args,const char * option,bool * value)794 rc_t get_bool( Args * args, const char *option, bool *value )
795 {
796 uint32_t count;
797 rc_t rc = ArgsOptionCount( args, option, &count );
798 *value = ( rc == 0 && count > 0 );
799 return rc;
800 }
801
802
get_str(Args * args,const char * option,const char ** value)803 rc_t get_str( Args * args, const char *option, const char ** value )
804 {
805 uint32_t count;
806 rc_t rc = ArgsOptionCount( args, option, &count );
807 *value = NULL;
808 if ( rc == 0 && count > 0 )
809 rc = ArgsOptionValue( args, option, 0, (const void **)value );
810 return rc;
811 }
812
813
get_size_t(Args * args,const char * option,size_t * value,size_t dflt)814 rc_t get_size_t( Args * args, const char *option, size_t *value, size_t dflt )
815 {
816 const char * s;
817 rc_t rc = get_str( args, option, &s );
818 *value = dflt;
819 if ( rc == 0 && s != NULL )
820 {
821 size_t l = string_size( s );
822 if ( l == 0 )
823 *value = dflt;
824 else
825 {
826 size_t multipl = 1;
827 switch( s[ l - 1 ] )
828 {
829 case 'k' :
830 case 'K' : multipl = 1024; break;
831 case 'm' :
832 case 'M' : multipl = 1024 * 1024; break;
833 case 'g' :
834 case 'G' : multipl = 1024 * 1024 * 1024; break;
835 }
836
837 if ( multipl > 1 )
838 {
839 char * src = string_dup( s, l - 1 );
840 if ( src != NULL )
841 {
842 char * endptr;
843 *value = strtol( src, &endptr, 0 ) * multipl;
844 free( src );
845 }
846 else
847 *value = dflt;
848 }
849 else
850 {
851 char * endptr;
852 *value = strtol( s, &endptr, 0 );
853 }
854 }
855 }
856 else
857 *value = dflt;
858 return rc;
859 }
860
861
get_fetch_ctx(Args * args,fetch_ctx * ctx)862 rc_t get_fetch_ctx( Args * args, fetch_ctx * ctx )
863 {
864 rc_t rc = 0;
865 uint32_t count;
866
867 ctx->url = NULL;
868 ctx->verbose = false;
869 rc = ArgsParamCount( args, &count );
870 if ( rc == 0 && count > 0 )
871 {
872 rc = ArgsParamValue( args, 0, (const void **)&ctx->url );
873 if ( rc == 0 )
874 {
875 if ( count > 1 )
876 rc = ArgsParamValue( args, 1, (const void **)&ctx->destination );
877 else
878 ctx->destination = NULL;
879 }
880 }
881
882 if ( rc == 0 ) rc = get_bool( args, OPTION_VERB, &ctx->verbose );
883 if ( rc == 0 ) rc = get_bool( args, OPTION_SHOW, &ctx->show_filesize );
884 if ( rc == 0 ) rc = get_str( args, OPTION_CACHE, &ctx->cache_file );
885 if ( rc == 0 ) rc = get_size_t( args, OPTION_CACHE_BLK, &ctx->cache_blk, 0 );
886 if ( rc == 0 ) rc = get_str( args, OPTION_PROXY, &ctx->proxy );
887 if ( rc == 0 ) rc = get_bool( args, OPTION_RAND, &ctx->random );
888 if ( rc == 0 ) rc = get_bool( args, OPTION_REP, &ctx->with_repeats );
889 if ( rc == 0 ) rc = get_bool( args, OPTION_CREPORT, &ctx->report_cache );
890 if ( rc == 0 ) rc = get_size_t( args, OPTION_BLOCK, &ctx->blocksize, ( 32 * 1024 ) );
891 if ( rc == 0 ) rc = get_size_t( args, OPTION_BUFFER, &ctx->buffer_size, 0 );
892 if ( rc == 0 ) rc = get_size_t( args, OPTION_SLEEP, &ctx->sleep_time, 0 );
893 if ( rc == 0 ) rc = get_size_t( args, OPTION_TIMEOUT, &ctx->timeout_time, 0 );
894 if ( rc == 0 ) rc = get_bool( args, OPTION_COMPLETE, &ctx->check_cache_complete );
895 if ( rc == 0 ) rc = get_bool( args, OPTION_CCOMPL, &ctx->check_completeness );
896 if ( rc == 0 ) rc = get_bool( args, OPTION_TRUNC, &ctx->truncate_cache );
897 if ( rc == 0 ) rc = get_size_t( args, OPTION_START, &ctx->start, 0 );
898 if ( rc == 0 ) rc = get_size_t( args, OPTION_COUNT, &ctx->count, 0 );
899 if ( rc == 0 ) rc = get_bool( args, OPTION_PROGRESS, &ctx->show_progress );
900 if ( rc == 0 ) rc = get_bool( args, OPTION_RELIABLE, &ctx->reliable );
901 if ( rc == 0 ) rc = get_bool( args, OPTION_FULL, &ctx->full_download );
902
903 return rc;
904 }
905
906
KMain(int argc,char * argv[])907 rc_t CC KMain ( int argc, char *argv [] )
908 {
909 Args * args;
910 rc_t rc;
911
912 rc = ArgsMakeAndHandle ( &args, argc, argv, 1,
913 MyOptions, sizeof ( MyOptions ) / sizeof ( OptDef ) );
914 if ( rc == 0 )
915 {
916 fetch_ctx ctx;
917
918 rc = get_fetch_ctx( args, &ctx );
919 if ( rc == 0 )
920 {
921 if ( ctx.url == NULL )
922 KOutMsg( "URL is missing!\n" );
923 else
924 {
925 KDirectory *dir;
926 rc = KDirectoryNativeDir ( &dir );
927 if ( rc == 0 )
928 {
929 if ( ctx.check_cache_complete )
930 rc = check_cache_complete( dir, &ctx );
931 else if ( ctx.check_completeness )
932 rc = check_cache_completeness( dir, &ctx );
933 else if ( ctx.truncate_cache )
934 rc = truncate_cache( dir, &ctx );
935 else if ( ctx.show_filesize )
936 rc = show_size( dir, &ctx );
937 else if ( ctx.full_download )
938 rc = full_download( dir, &ctx );
939 else
940 rc = fetch( dir, &ctx );
941
942 KDirectoryRelease ( dir );
943 }
944 }
945 }
946 ArgsWhack ( args );
947 }
948 else
949 KOutMsg( "ArgsMakeAndHandle() failed %R\n", rc );
950
951 return rc;
952 }
953