1 /*===========================================================================
2 *
3 *                            PUBLIC DOMAIN NOTICE
4 *               National Center for Biotechnology Information
5 *
6 *  This software/database is a "United States Government Work" under the
7 *  terms of the United States Copyright Act.  It was written as part of
8 *  the author's official duties as a United States Government employee and
9 *  thus cannot be copyrighted.  This software/database is freely available
10 *  to the public for use. The National Library of Medicine and the U.S.
11 *  Government have not placed any restriction on its use or reproduction.
12 *
13 *  Although all reasonable efforts have been taken to ensure the accuracy
14 *  and reliability of the software and data, the NLM and the U.S.
15 *  Government do not and cannot warrant the performance or results that
16 *  may be obtained by using this software or data. The NLM and the U.S.
17 *  Government disclaim all warranties, express or implied, including
18 *  warranties of performance, merchantability or fitness for any particular
19 *  purpose.
20 *
21 *  Please cite the author in any work or product based on this material.
22 *
23 * ===========================================================================
24 *
25 */
26 
27 #include "helper.h"
28 
29 #include <klib/log.h>
30 #include <klib/printf.h>
31 #include <klib/out.h>
32 #include <kfs/defs.h>
33 #include <kfs/file.h>
34 #include <kfs/buffile.h>
35 #include <search/nucstrstr.h>
36 
37 #include <kdb/manager.h>
38 #include <vdb/manager.h>
39 #include <vfs/manager.h>
40 #include <vfs/path.h>
41 
42 #include <atomic32.h>
43 
ErrMsg(const char * fmt,...)44 rc_t ErrMsg( const char * fmt, ... )
45 {
46     rc_t rc;
47     char buffer[ 4096 ];
48     size_t num_writ;
49 
50     va_list list;
51     va_start( list, fmt );
52     rc = string_vprintf( buffer, sizeof buffer, &num_writ, fmt, list );
53     if ( 0 == rc )
54     {
55         rc = pLogMsg( klogErr, "$(E)", "E=%s", buffer );
56     }
57     va_end( list );
58     return rc;
59 }
60 
61 rc_t CC ArgsOptionCount( const struct Args * self, const char * option_name, uint32_t * count );
62 rc_t CC ArgsOptionValue( const struct Args * self, const char * option_name, uint32_t iteration, const void ** value );
63 
get_str_option(const struct Args * args,const char * name,const char * dflt)64 const char * get_str_option( const struct Args *args, const char *name, const char * dflt )
65 {
66     const char* res = dflt;
67     uint32_t count;
68     rc_t rc = ArgsOptionCount( args, name, &count );
69     if ( 0 == rc && count > 0 )
70     {
71         rc = ArgsOptionValue( args, name, 0, (const void**)&res );
72         if ( 0 != rc )
73         {
74             res = dflt;
75         }
76     }
77     return res;
78 }
79 
get_bool_option(const struct Args * args,const char * name)80 bool get_bool_option( const struct Args *args, const char *name )
81 {
82     uint32_t count;
83     rc_t rc = ArgsOptionCount( args, name, &count );
84     return ( 0 == rc && count > 0 );
85 }
86 
87 
get_uint64_t_option(const struct Args * args,const char * name,uint64_t dflt)88 uint64_t get_uint64_t_option( const struct Args * args, const char *name, uint64_t dflt )
89 {
90     uint64_t res = dflt;
91     const char * s = get_str_option( args, name, NULL );
92     if ( NULL != s )
93     {
94         size_t l = string_size( s );
95         if ( l > 0 )
96         {
97             char * endptr;
98             res = strtol( s, &endptr, 0 );
99         }
100     }
101     return res;
102 }
103 
get_uint32_t_option(const struct Args * args,const char * name,uint32_t dflt)104 uint32_t get_uint32_t_option( const struct Args * args, const char *name, uint32_t dflt )
105 {
106     uint32_t res = dflt;
107     const char * s = get_str_option( args, name, NULL );
108     if ( NULL != s )
109     {
110         size_t l = string_size( s );
111         if ( l > 0 )
112         {
113             char * endptr;
114             res = ( uint32_t )strtol( s, &endptr, 0 );
115         }
116     }
117     return res;
118 
119 }
120 
get_size_t_option(const struct Args * args,const char * name,size_t dflt)121 size_t get_size_t_option( const struct Args * args, const char *name, size_t dflt )
122 {
123     size_t res = dflt;
124     const char * s = get_str_option( args, name, NULL );
125     if ( NULL != s )
126     {
127         size_t l = string_size( s );
128         if ( l > 0 )
129         {
130             size_t multipl = 1;
131             switch( s[ l - 1 ] )
132             {
133                 case 'k' :
134                 case 'K' : multipl = 1024; break;
135                 case 'm' :
136                 case 'M' : multipl = 1024 * 1024; break;
137                 case 'g' :
138                 case 'G' : multipl = 1024 * 1024 * 1024; break;
139             }
140 
141             if ( multipl > 1 )
142             {
143                 char * src = string_dup( s, l - 1 );
144                 if ( NULL != src )
145                 {
146                     char * endptr;
147                     res = strtol( src, &endptr, 0 ) * multipl;
148                     free( src );
149                 }
150             }
151             else
152             {
153                 char * endptr;
154                 res = strtol( s, &endptr, 0 );
155             }
156         }
157     }
158     return res;
159 }
160 
format_cmp(String * Format,const char * test,format_t test_fmt)161 static format_t format_cmp( String * Format, const char * test, format_t test_fmt )
162 {
163     String TestFormat;
164     StringInitCString( &TestFormat, test );
165     if ( 0 == StringCaseCompare ( Format, &TestFormat ) )
166     {
167         return test_fmt;
168     }
169     return ft_unknown;
170 }
171 
get_format_t(const char * format,bool split_spot,bool split_file,bool split_3,bool whole_spot)172 format_t get_format_t( const char * format,
173             bool split_spot, bool split_file, bool split_3, bool whole_spot )
174 {
175     format_t res = ft_unknown;
176     if ( NULL != format && 0 != format[ 0 ] )
177     {
178         /* the format option has been used, try to recognize one of the options,
179            the legacy options will be ignored now */
180 
181         String Format;
182         StringInitCString( &Format, format );
183 
184         res = format_cmp( &Format, "special", ft_special );
185         if ( ft_unknown == res )
186         {
187             res = format_cmp( &Format, "whole-spot", ft_whole_spot );
188         }
189         if ( ft_unknown == res )
190         {
191             res = format_cmp( &Format, "fastq-split-spot", ft_fastq_split_spot );
192         }
193         if ( ft_unknown == res )
194         {
195             res = format_cmp( &Format, "fastq-split-file", ft_fastq_split_file );
196         }
197         if ( ft_unknown == res )
198         {
199             res = format_cmp( &Format, "fastq-split-3", ft_fastq_split_3 );
200         }
201     }
202     else
203     {
204         /* the format option has not been used, let us see if some of the legacy-options
205             have been used */
206         if ( split_3 )
207         {
208             res = ft_fastq_split_3;
209         }
210         else if ( split_file )
211         {
212             res = ft_fastq_split_file;
213         }
214         else if ( split_spot )
215         {
216             res = ft_fastq_split_spot;
217         }
218         else if ( whole_spot )
219         {
220             res = ft_whole_spot;
221         }
222     }
223 
224     /* default to split_3 if no format has been given at all */
225     if ( ft_unknown == res )
226     {
227         res = ft_fastq_split_3;
228     }
229     return res;
230 }
231 
make_SBuffer(SBuffer * buffer,size_t len)232 rc_t make_SBuffer( SBuffer * buffer, size_t len )
233 {
234     rc_t rc = 0;
235     String * S = &buffer -> S;
236     S -> addr = malloc( len );
237     if ( NULL == S -> addr )
238     {
239         S -> size = S -> len = 0;
240         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
241         ErrMsg( "make_SBuffer().malloc( %d ) -> %R", ( len ), rc );
242     }
243     else
244     {
245         S -> size = 0;
246         S -> len = 0;
247         buffer -> buffer_size = len;
248     }
249     return rc;
250 }
251 
release_SBuffer(SBuffer * self)252 void release_SBuffer( SBuffer * self )
253 {
254     if ( NULL != self )
255     {
256         String * S = &self -> S;
257         if ( NULL != S -> addr )
258         {
259             free( ( void * ) S -> addr );
260         }
261     }
262 }
263 
increase_SBuffer(SBuffer * self,size_t by)264 rc_t increase_SBuffer( SBuffer * self, size_t by )
265 {
266     rc_t rc;
267     if ( NULL == self )
268     {
269         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
270     }
271     else
272     {
273         size_t new_size = self -> buffer_size + by;
274         release_SBuffer( self );
275         rc = make_SBuffer( self, new_size );
276     }
277     return rc;
278 }
279 
print_to_SBufferV(SBuffer * self,const char * fmt,va_list args)280 rc_t print_to_SBufferV( SBuffer * self, const char * fmt, va_list args )
281 {
282     rc_t rc = 0;
283     if ( NULL == self )
284     {
285         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcSelf, rcNull );
286     }
287     else
288     {
289         char * dst = ( char * )( self -> S . addr );
290         size_t num_writ = 0;
291 
292         rc = string_vprintf( dst, self -> buffer_size, &num_writ, fmt, args );
293         if ( 0 == rc )
294         {
295             self -> S . size = num_writ;
296             self -> S . len = ( uint32_t )self -> S . size;
297         }
298     }
299     return rc;
300 }
301 
try_to_enlarge_SBuffer(SBuffer * self,rc_t rc_err)302 rc_t try_to_enlarge_SBuffer( SBuffer * self, rc_t rc_err )
303 {
304     rc_t rc = rc_err;
305     if ( ( GetRCObject( rc ) == ( enum RCObject )rcBuffer ) && ( GetRCState( rc ) == rcInsufficient ) )
306     {
307         rc = increase_SBuffer( self, self -> buffer_size ); /* double it's size */
308         if ( 0 != rc )
309         {
310             ErrMsg( "try_to_enlarge_SBuffer().increase_SBuffer() -> %R", rc );
311         }
312     }
313     return rc;
314 }
315 
print_to_SBuffer(SBuffer * self,const char * fmt,...)316 rc_t print_to_SBuffer( SBuffer * self, const char * fmt, ... )
317 {
318     rc_t rc = 0;
319     bool done = false;
320     while ( 0 == rc && !done )
321     {
322         va_list args;
323 
324         va_start( args, fmt );
325         rc = print_to_SBufferV( self, fmt, args );
326         va_end( args );
327 
328         done = ( 0 == rc );
329         if ( !done )
330         {
331             rc = try_to_enlarge_SBuffer( self, rc );
332         }
333     }
334     return rc;
335 }
336 
337 
make_and_print_to_SBuffer(SBuffer * self,size_t len,const char * fmt,...)338 rc_t make_and_print_to_SBuffer( SBuffer * self, size_t len, const char * fmt, ... )
339 {
340     rc_t rc = make_SBuffer( self, len );
341     if ( 0 == rc )
342     {
343         va_list args;
344 
345         va_start( args, fmt );
346         rc = print_to_SBufferV( self, fmt, args );
347         va_end( args );
348     }
349     return rc;
350 }
351 
split_string(String * in,String * p0,String * p1,uint32_t ch)352 rc_t split_string( String * in, String * p0, String * p1, uint32_t ch )
353 {
354     rc_t rc = 0;
355     char * ch_ptr = string_chr( in -> addr, in -> size, ch );
356     if ( NULL == ch_ptr )
357     {
358         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcTransfer, rcInvalid );
359     }
360     else
361     {
362         p0 -> addr = in -> addr;
363         p0 -> size = ( ch_ptr - p0 -> addr );
364         p0 -> len  = ( uint32_t ) p0 -> size;
365         p1 -> addr = ch_ptr + 1;
366         p1 -> size = in -> len - ( p0 -> len + 1 );
367         p1 -> len  = ( uint32_t ) p1 -> size;
368     }
369     return rc;
370 }
371 
split_string_r(String * in,String * p0,String * p1,uint32_t ch)372 rc_t split_string_r( String * in, String * p0, String * p1, uint32_t ch )
373 {
374     rc_t rc = 0;
375     char * ch_ptr = string_rchr( in -> addr, in -> size, ch );
376     if ( NULL == ch_ptr )
377     {
378         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcTransfer, rcInvalid );
379     }
380     else
381     {
382         p0 -> addr = in -> addr;
383         p0 -> size = ( ch_ptr - p0 -> addr );
384         p0 -> len  = ( uint32_t ) p0 -> size;
385         p1 -> addr = ch_ptr + 1;
386         p1 -> size = in -> len - ( p0 -> len + 1 );
387         p1 -> len  = ( uint32_t ) p1 -> size;
388     }
389     return rc;
390 }
391 
split_filename_insert_idx(SBuffer * dst,size_t dst_size,const char * filename,uint32_t idx)392 rc_t split_filename_insert_idx( SBuffer * dst, size_t dst_size,
393                                 const char * filename, uint32_t idx )
394 {
395     rc_t rc;
396     if ( idx > 0 )
397     {
398         /* we have to split md -> cmn -> output_filename into name and extension
399            then append '_%u' to the name, then re-append the extension */
400         String S_in, S_name, S_ext;
401         StringInitCString( &S_in, filename );
402         rc = split_string_r( &S_in, &S_name, &S_ext, '.' ); /* helper.c */
403         if ( 0 == rc )
404         {
405             /* we found a dot to split the filename! */
406             rc = make_and_print_to_SBuffer( dst, dst_size, "%S_%u.%S",
407                         &S_name, idx, &S_ext ); /* helper.c */
408         }
409         else
410         {
411             /* we did not find a dot to split the filename! */
412             rc = make_and_print_to_SBuffer( dst, dst_size, "%s_%u.fastq",
413                         filename, idx ); /* helper.c */
414         }
415     }
416     else
417     {
418         rc = make_and_print_to_SBuffer( dst, dst_size, "%s", filename ); /* helper.c */
419     }
420 
421     if ( 0 != rc )
422     {
423         release_SBuffer( dst );
424     }
425     return rc;
426 }
427 
get_compress_t(bool gzip,bool bzip2)428 compress_t get_compress_t( bool gzip, bool bzip2 )
429 {
430     if ( gzip && bzip2 )
431     {
432         return ct_bzip2;
433     }
434     else if ( gzip )
435     {
436         return ct_gzip;
437     }
438     else if ( bzip2 )
439     {
440         return ct_bzip2;
441     }
442     return ct_none;
443 }
444 
make_key(int64_t seq_spot_id,uint32_t seq_read_id)445 uint64_t make_key( int64_t seq_spot_id, uint32_t seq_read_id )
446 {
447     uint64_t key = seq_spot_id;
448     key <<= 1;
449     key |= ( 2 == seq_read_id ) ? 1 : 0;
450     return key;
451 }
452 
pack_4na(const String * unpacked,SBuffer * packed)453 rc_t pack_4na( const String * unpacked, SBuffer * packed )
454 {
455     rc_t rc = 0;
456     if ( unpacked -> len < 1 )
457     {
458         rc = RC( rcVDB, rcNoTarg, rcWriting, rcFormat, rcNull );
459     }
460     else
461     {
462         if ( unpacked -> len > 0xFFFF )
463         {
464             rc = RC( rcVDB, rcNoTarg, rcWriting, rcFormat, rcExcessive );
465         }
466         else
467         {
468             uint32_t i;
469             uint8_t * src = ( uint8_t * )unpacked -> addr;
470             uint8_t * dst = ( uint8_t * )packed -> S . addr;
471             uint16_t dna_len = ( unpacked -> len & 0xFFFF );
472             uint32_t len = 0;
473             dst[ len++ ] = ( dna_len >> 8 );
474             dst[ len++ ] = ( dna_len & 0xFF );
475             for ( i = 0; i < unpacked -> len; ++i )
476             {
477                 if ( len < packed -> buffer_size )
478                 {
479                     uint8_t base = ( src[ i ] & 0x0F );
480                     if ( 0 == ( i & 0x01 ) )
481                     {
482                         dst[ len ] = ( base << 4 );
483                     }
484                     else
485                     {
486                         dst[ len++ ] |= base;
487                     }
488                 }
489             }
490             if ( unpacked -> len & 0x01 )
491             {
492                 len++;
493             }
494             packed -> S . size = packed -> S . len = len;
495         }
496     }
497     return rc;
498 }
499 
500 static const char xASCII_to_4na[ 256 ] =
501 {
502     /* 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F */
503        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
504 
505     /* 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1A 0x1B 0x1C 0x1D 0x1E 0x1F */
506        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
507 
508     /* 0x20 0x21 0x22 0x23 0x24 0x25 0x26 0x27 0x28 0x29 0x2A 0x2B 0x2C 0x2D 0x2E 0x2F */
509        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
510 
511     /* 0x30 0x31 0x32 0x33 0x34 0x35 0x36 0x37 0x38 0x39 0x3A 0x3B 0x3C 0x3D 0x3E 0x3F */
512        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
513 
514     /* 0x40 0x41 0x42 0x43 0x44 0x45 0x46 0x47 0x48 0x49 0x4A 0x4B 0x4C 0x4D 0x4E 0x4F */
515     /* @    A    B    C    D    E    F    G    H    I    J    K    L    M    N    O */
516        0,   1,   0,   2,   0,   0,   0,   4,   0,   0,   0,   0,   0,   0,   0,   0,
517 
518     /* 0x50 0x51 0x52 0x53 0x54 0x55 0x56 0x57 0x58 0x59 0x5A 0x5B 0x5C 0x5D 0x5E 0x5F */
519     /* P    Q    R    S    T    U    V    W    X    Y    Z    [    \    ]    ^    _ */
520        0,   0,   0,   0,   8,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
521 
522     /* 0x60 0x61 0x62 0x63 0x64 0x65 0x66 0x67 0x68 0x69 0x6A 0x6B 0x6C 0x6D 0x6E 0x6F */
523        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
524 
525     /* 0x70 0x71 0x72 0x73 0x74 0x75 0x76 0x77 0x78 0x79 0x7A 0x7B 0x7C 0x7D 0x7E 0x7F */
526        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
527 
528     /* 0x80 0x81 0x82 0x83 0x84 0x85 0x86 0x87 0x88 0x89 0x8A 0x8B 0x8C 0x8D 0x8E 0x8F */
529        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
530 
531     /* 0x90 0x91 0x92 0x93 0x94 0x95 0x96 0x97 0x98 0x99 0x9A 0x9B 0x9C 0x9D 0x9E 0x9F */
532        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
533 
534     /* 0xA0 0xA1 0xA2 0xA3 0xA4 0xA5 0xA6 0xA7 0xA8 0xA9 0xAA 0xAB 0xAC 0xAD 0xAE 0xAF */
535        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
536 
537     /* 0xB0 0xB1 0xB2 0xB3 0xB4 0xB5 0xB6 0xB7 0xB8 0xB9 0xBA 0xBB 0xBC 0xBD 0xBE 0xBF */
538        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
539 
540     /* 0xC0 0xC1 0xC2 0xC3 0xC4 0xC5 0xC6 0xC7 0xC8 0xC9 0xCA 0xCB 0xCC 0xCD 0xCE 0xCF */
541        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
542 
543     /* 0xD0 0xD1 0xD2 0xD3 0xD4 0xD5 0xD6 0xD7 0xD8 0xD9 0xDA 0xDB 0xDC 0xDD 0xDE 0xDF */
544        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
545 
546     /* 0xE0 0xE1 0xE2 0xE3 0xE4 0xE5 0xE6 0xE7 0xE8 0xE9 0xEA 0xEB 0xEC 0xED 0xEE 0xEF */
547        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
548 
549     /* 0xF0 0xF1 0xF2 0xF3 0xF4 0xF5 0xF6 0xF7 0xF8 0xF9 0xFA 0xFB 0xFC 0xFD 0xFE 0xFF */
550        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0
551 };
552 
pack_read_2_4na(const String * read,SBuffer * packed)553 rc_t pack_read_2_4na( const String * read, SBuffer * packed )
554 {
555     rc_t rc = 0;
556     if ( read -> len < 1 )
557     {
558         rc = RC( rcVDB, rcNoTarg, rcWriting, rcFormat, rcNull );
559     }
560     else
561     {
562         if ( read -> len > 0xFFFF )
563         {
564             rc = RC( rcVDB, rcNoTarg, rcWriting, rcFormat, rcExcessive );
565         }
566         else
567         {
568             uint32_t i;
569             uint8_t * src = ( uint8_t * )read -> addr;
570             uint8_t * dst = ( uint8_t * )packed -> S . addr;
571             uint16_t dna_len = ( read -> len & 0xFFFF );
572             uint32_t len = 0;
573             dst[ len++ ] = ( dna_len >> 8 );
574             dst[ len++ ] = ( dna_len & 0xFF );
575             for ( i = 0; i < read -> len; ++i )
576             {
577                 if ( len < packed -> buffer_size )
578                 {
579                     uint8_t base = ( xASCII_to_4na[ src[ i ] ] & 0x0F );
580                     if ( 0 == ( i & 0x01 ) )
581                     {
582                         dst[ len ] = ( base << 4 );
583                     }
584                     else
585                     {
586                         dst[ len++ ] |= base;
587                     }
588                 }
589             }
590             if ( read -> len & 0x01 )
591             {
592                 len++;
593             }
594             packed -> S . size = packed -> S . len = len;
595         }
596     }
597     return rc;
598 }
599 
600 static const char x4na_to_ASCII_fwd[ 16 ] =
601 {
602     /* 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F */
603        'N', 'A', 'C', 'N', 'G', 'N', 'N', 'N', 'T', 'N', 'N', 'N', 'N', 'N', 'N', 'N'
604 };
605 
606 static const char x4na_to_ASCII_rev[ 16 ] =
607 {
608     /* 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F */
609        'N', 'T', 'G', 'N', 'C', 'N', 'N', 'N', 'A', 'N', 'N', 'N', 'N', 'N', 'N', 'N'
610 };
611 
unpack_4na(const String * packed,SBuffer * unpacked,bool reverse)612 rc_t unpack_4na( const String * packed, SBuffer * unpacked, bool reverse )
613 {
614     rc_t rc = 0;
615     uint8_t * src = ( uint8_t * )packed -> addr;
616     uint16_t dna_len;
617 
618     /* the first 2 bytes are the 16-bit dna-length */
619     dna_len = src[ 0 ];
620     dna_len <<= 8;
621     dna_len |= src[ 1 ];
622 
623     if ( dna_len > unpacked -> buffer_size )
624     {
625         rc = increase_SBuffer( unpacked, dna_len - unpacked -> buffer_size );
626     }
627     if ( 0 == rc )
628     {
629         uint8_t * dst = ( uint8_t * )unpacked -> S . addr;
630         uint32_t dst_idx;
631         uint32_t i;
632 
633         /* use the complement-lookup-table in case of reverse */
634         const char * lookup = reverse ? x4na_to_ASCII_rev : x4na_to_ASCII_fwd;
635 
636         dst_idx = reverse ? dna_len - 1 : 0;
637 
638         for ( i = 2; i < packed -> len; ++i )
639         {
640             /* get the packed byte out of the packed input */
641             uint8_t packed_byte = src[ i ];
642 
643             /* write the first unpacked byte */
644             if ( dst_idx < unpacked -> buffer_size )
645             {
646                 dst[ dst_idx ] = lookup[ ( packed_byte >> 4 ) & 0x0F ];
647                 dst_idx += reverse ? -1 : 1;
648             }
649 
650             /* write the second unpacked byte */
651             if ( dst_idx < unpacked -> buffer_size )
652             {
653                 dst[ dst_idx ] = lookup[ packed_byte & 0x0F ];
654                 dst_idx += reverse ? -1 : 1;
655             }
656         }
657 
658         /* set the dna-length in the output-string */
659         unpacked -> S . size = dna_len;
660         unpacked -> S . len = ( uint32_t )unpacked -> S . size;
661 
662         /* terminated the output-string, just in case */
663         dst[ dna_len ] = 0;
664     }
665     return rc;
666 }
667 
check_expected(const KDirectory * dir,uint32_t expected,const char * fmt,va_list args)668 bool check_expected( const KDirectory * dir, uint32_t expected, const char * fmt, va_list args )
669 {
670     bool res = false;
671     char buffer[ 4096 ];
672     size_t num_writ;
673 
674     rc_t rc = string_vprintf( buffer, sizeof buffer, &num_writ, fmt, args );
675     if ( 0 == rc )
676     {
677         uint32_t pt = KDirectoryPathType( dir, "%s", buffer );
678         res = ( pt == expected );
679     }
680     return res;
681 }
682 
file_exists(const KDirectory * dir,const char * fmt,...)683 bool file_exists( const KDirectory * dir, const char * fmt, ... )
684 {
685     bool res = false;
686     va_list args;
687     va_start( args, fmt );
688     res = check_expected( dir, kptFile, fmt, args ); /* because KDirectoryPathType() uses vsnprintf */
689     va_end( args );
690     return res;
691 }
692 
dir_exists(const KDirectory * dir,const char * fmt,...)693 bool dir_exists( const KDirectory * dir, const char * fmt, ... )
694 {
695     bool res = false;
696     va_list args;
697     va_start( args, fmt );
698     res = check_expected( dir, kptDir, fmt, args ); /* because KDirectoryPathType() uses vsnprintf */
699     va_end( args );
700     return res;
701 }
702 
join_and_release_threads(Vector * threads)703 rc_t join_and_release_threads( Vector * threads )
704 {
705     rc_t rc = 0;
706     uint32_t i, n = VectorLength( threads );
707     for ( i = VectorStart( threads ); i < n; ++i )
708     {
709         KThread * thread = VectorGet( threads, i );
710         if ( NULL != thread )
711         {
712             rc_t rc1;
713             KThreadWait( thread, &rc1 );
714             if ( 0 == rc && rc1 != 0 )
715             {
716                 rc = rc1;
717             }
718             KThreadRelease( thread );
719         }
720     }
721     VectorWhack ( threads, NULL, NULL );
722     return rc;
723 }
724 
725 
clear_join_stats(join_stats * stats)726 void clear_join_stats( join_stats * stats )
727 {
728     if ( stats != NULL )
729     {
730         stats -> spots_read = 0;
731         stats -> reads_read = 0;
732         stats -> reads_written = 0;
733         stats -> reads_zero_length = 0;
734         stats -> reads_technical = 0;
735         stats -> reads_too_short = 0;
736         stats -> reads_invalid = 0;
737     }
738 }
739 
add_join_stats(join_stats * stats,const join_stats * to_add)740 void add_join_stats( join_stats * stats, const join_stats * to_add )
741 {
742     if ( NULL != stats && NULL != to_add )
743     {
744         stats -> spots_read += to_add -> spots_read;
745         stats -> reads_read += to_add -> reads_read;
746         stats -> reads_written += to_add -> reads_written;
747         stats -> reads_zero_length += to_add -> reads_zero_length;
748         stats -> reads_technical += to_add -> reads_technical;
749         stats -> reads_too_short += to_add -> reads_too_short;
750         stats -> reads_invalid += to_add -> reads_invalid;
751     }
752 }
753 
delete_files(KDirectory * dir,const VNamelist * files)754 rc_t delete_files( KDirectory * dir, const VNamelist * files )
755 {
756     uint32_t count;
757     rc_t rc = VNameListCount( files, &count );
758     if ( 0 != rc )
759     {
760         ErrMsg( "delete_files().VNameListCount() -> %R", rc );
761     }
762     else
763     {
764         uint32_t idx;
765         for ( idx = 0; 0 == rc && idx < count; ++idx )
766         {
767             const char * filename;
768             rc = VNameListGet( files, idx, &filename );
769             if ( 0 != rc )
770             {
771                 ErrMsg( "delete_files.VNameListGet( #%d ) -> %R", idx, rc );
772             }
773             else
774             {
775                 if ( file_exists( dir, "%s", filename ) )
776                 {
777                     rc = KDirectoryRemove( dir, true, "%s", filename );
778                     if ( 0 != rc )
779                     {
780                         ErrMsg( "delete_files.KDirectoryRemove( '%s' ) -> %R", filename, rc );
781                     }
782                 }
783             }
784         }
785     }
786     return rc;
787 }
788 
delete_dirs(KDirectory * dir,const VNamelist * dirs)789 rc_t delete_dirs( KDirectory * dir, const VNamelist * dirs )
790 {
791     uint32_t count;
792     rc_t rc = VNameListCount( dirs, &count );
793     if ( 0 != rc )
794     {
795         ErrMsg( "delete_dirs().VNameListCount() -> %R", rc );
796     }
797     else
798     {
799         uint32_t idx;
800         for ( idx = 0; 0 == rc && idx < count; ++idx )
801         {
802             const char * dirname;
803             rc = VNameListGet( dirs, idx, &dirname );
804             if ( 0 != rc )
805             {
806                 ErrMsg( "delete_dirs().VNameListGet( #%d ) -> %R", idx, rc );
807             }
808             else if ( dir_exists( dir, "%s", dirname ) )
809             {
810                 rc = KDirectoryClearDir ( dir, true, "%s", dirname );
811                 if ( 0 != rc )
812                 {
813                     ErrMsg( "delete_dirs().KDirectoryClearDir( %s ) -> %R", dirname, rc );
814                 }
815                 else
816                 {
817                     rc = KDirectoryRemove ( dir, true, "%s", dirname );
818                     if ( 0 != rc )
819                     {
820                         ErrMsg( "delete_dirs().KDirectoryRemove( %s ) -> %R", dirname, rc );
821                     }
822                 }
823             }
824         }
825     }
826     return rc;
827 }
828 
total_size_of_files_in_list(KDirectory * dir,const VNamelist * files)829 uint64_t total_size_of_files_in_list( KDirectory * dir, const VNamelist * files )
830 {
831     uint64_t res = 0;
832     if ( NULL != dir && NULL != files )
833     {
834         uint32_t idx, count;
835         rc_t rc = VNameListCount( files, &count );
836         if ( 0 != rc )
837         {
838             ErrMsg( "total_size_of_files_in_list().VNameListCount() -> %R", rc );
839         }
840         else
841         {
842             for ( idx = 0; 0 == rc && idx < count; ++idx )
843             {
844                 const char * filename;
845                 rc = VNameListGet( files, idx, &filename );
846                 if ( 0 != rc )
847                 {
848                     ErrMsg( "total_size_of_files_in_list().VNameListGet( #%d ) -> %R", idx, rc );
849                 }
850                 else
851                 {
852                     uint64_t size;
853                     rc_t rc1 = KDirectoryFileSize( dir, &size, "%s", filename );
854                     if ( 0 != rc1 )
855                     {
856                         ErrMsg( "total_size_of_files_in_list().KDirectoryFileSize( %s ) -> %R", filename, rc );
857                     }
858                     else
859                     {
860                         res += size;
861                     }
862                 }
863             }
864         }
865     }
866     return res;
867 }
868 
869 /*
870 int get_vdb_pathtype( KDirectory * dir, const VDBManager * vdb_mgr, const char * accession )
871 {
872     int res = kptAny;
873     rc_t rc = 0;
874     bool release_mgr = false;
875     const VDBManager * mgr = vdb_mgr != NULL ? vdb_mgr : NULL;
876     if ( mgr == NULL )
877     {
878         rc = VDBManagerMakeRead( &mgr, dir );
879         if ( rc != 0 )
880             ErrMsg( "get_vdb_pathtype().VDBManagerMakeRead() -> %R\n", rc );
881         else
882             release_mgr = true;
883     }
884     if ( rc == 0 )
885     {
886         res = ( VDBManagerPathType ( mgr, "%s", accession ) & ~ kptAlias );
887         if ( release_mgr )
888             VDBManagerRelease( mgr );
889     }
890     return res;
891 }
892 */
893 
894 /* ===================================================================================== */
895 
ends_in_slash(const char * s)896 bool ends_in_slash( const char * s )
897 {
898     bool res = false;
899     if ( NULL != s )
900     {
901         uint32_t len = string_measure( s, NULL );
902         if ( len > 0 )
903         {
904             res = ( '/' == s[ len - 1 ] );
905         }
906     }
907     return res;
908 }
909 
ends_in_sra(const char * s)910 static bool ends_in_sra( const char * s )
911 {
912     bool res = false;
913     if ( NULL != s )
914     {
915         uint32_t len = string_measure( s, NULL );
916         if ( len > 4 )
917         {
918             res = ( ( 'a' == s[ len - 1 ] ) &&
919                     ( 'r' == s[ len - 2 ] ) &&
920                     ( 's' == s[ len - 3 ] ) &&
921                     ( '.' == s[ len - 4 ] ) );
922         }
923     }
924     return res;
925 }
926 
extract_path(const char * s,String * path)927 bool extract_path( const char * s, String * path )
928 {
929     bool res = false;
930     if ( NULL != s && NULL != path )
931     {
932         path -> addr = s;
933         res = ends_in_slash( s );
934         if ( res )
935         {
936             path -> len = string_measure( s, & path -> size );
937         }
938         else
939         {
940             size_t size = string_size ( s );
941             char * slash = string_rchr ( s, size, '/' );
942             res = ( NULL != slash );
943             if ( res )
944             {
945                 path -> len = ( slash - s );
946                 path -> size = ( size_t ) path -> len;
947             }
948         }
949     }
950     return res;
951 }
952 
extract_acc(const char * s)953 const char * extract_acc( const char * s )
954 {
955     const char * res = NULL;
956     if ( ( NULL != s ) && ( !ends_in_slash( s ) ) )
957     {
958         size_t size = string_size ( s );
959         char * slash = string_rchr ( s, size, '/' );
960         if ( NULL == slash )
961         {
962             if ( ends_in_sra( s ) )
963             {
964                 res = string_dup ( s, size - 4 );
965             }
966             else
967             {
968                 res = string_dup ( s, size );
969             }
970         }
971         else
972         {
973             char * tmp = slash + 1;
974             if ( ends_in_sra( tmp ) )
975             {
976                 res = string_dup ( tmp, string_size ( tmp ) - 4 );
977             }
978             else
979             {
980                 res = string_dup ( tmp, string_size ( tmp ) );
981             }
982         }
983     }
984     return res;
985 }
986 
extract_acc2(const char * s)987 const char * extract_acc2( const char * s )
988 {
989     const char * res = NULL;
990     VFSManager * mgr;
991     rc_t rc = VFSManagerMake ( &mgr );
992     if ( 0 != rc )
993     {
994         ErrMsg( "extract_acc2( '%s' ).VFSManagerMake() -> %R", s, rc );
995     }
996     else
997     {
998         VPath * orig;
999         rc = VFSManagerMakePath ( mgr, &orig, "%s", s );
1000         if ( 0 != rc )
1001         {
1002             ErrMsg( "extract_acc2( '%s' ).VFSManagerMakePath() -> %R", s, rc );
1003         }
1004         else
1005         {
1006             VPath * acc_or_oid = NULL;
1007             rc = VFSManagerExtractAccessionOrOID( mgr, &acc_or_oid, orig );
1008             if ( 0 != rc )
1009             {
1010                 ErrMsg( "extract_acc2( '%s' ).VFSManagerExtractAccessionOrOID() -> %R", s, rc );
1011             }
1012             else
1013             {
1014                 char buffer[ 1024 ];
1015                 size_t num_read;
1016                 rc = VPathReadPath ( acc_or_oid, buffer, sizeof buffer, &num_read );
1017                 if ( 0 != rc )
1018                 {
1019                     ErrMsg( "extract_acc2( '%s' ).VPathReadPath() -> %R", s, rc );
1020                 }
1021                 else
1022                 {
1023                     res = string_dup ( buffer, num_read );
1024                 }
1025 
1026                 rc = VPathRelease ( acc_or_oid );
1027                 if ( 0 != rc )
1028                 {
1029                     ErrMsg( "extract_acc2( '%s' ).VPathRelease().1 -> %R", s, rc );
1030                 }
1031             }
1032 
1033             rc = VPathRelease ( orig );
1034             if ( 0 != rc )
1035             {
1036                 ErrMsg( "extract_acc2( '%s' ).VPathRelease().2 -> %R", s, rc );
1037             }
1038         }
1039 
1040         rc = VFSManagerRelease ( mgr );
1041         if ( 0 != rc )
1042         {
1043             ErrMsg( "extract_acc2( '%s' ).VFSManagerRelease() -> %R", s, rc );
1044         }
1045     }
1046     return res;
1047 }
1048 
create_this_file(KDirectory * dir,const char * filename,bool force)1049 rc_t create_this_file( KDirectory * dir, const char * filename, bool force )
1050 {
1051     struct KFile * f;
1052     KCreateMode create_mode = force ? kcmInit : kcmCreate;
1053     rc_t rc = KDirectoryCreateFile( dir, &f, false, 0664, create_mode | kcmParents, "%s", filename );
1054     if ( 0 != rc )
1055     {
1056         ErrMsg( "create_this_file().KDirectoryCreateFile( '%s' ) -> %R", filename, rc );
1057     }
1058     else
1059     {
1060         rc_t rc2 = KFileRelease( f );
1061         if ( 0 != rc2 )
1062         {
1063             ErrMsg( "create_this_file( '%s' ).KFileRelease() -> %R", filename, rc2 );
1064             rc = ( 0 == rc ) ? rc2 : rc;
1065         }
1066     }
1067     return rc;
1068 }
1069 
create_this_dir(KDirectory * dir,const String * dir_name,bool force)1070 rc_t create_this_dir( KDirectory * dir, const String * dir_name, bool force )
1071 {
1072     KCreateMode create_mode = force ? kcmInit : kcmCreate;
1073     rc_t rc = KDirectoryCreateDir( dir, 0774, create_mode | kcmParents, "%.*s", dir_name -> len, dir_name -> addr );
1074     if ( 0 != rc )
1075     {
1076         ErrMsg( "create_this_dir().KDirectoryCreateDir( '%.*s' ) -> %R", dir_name -> len, dir_name -> addr, rc );
1077     }
1078     return rc;
1079 }
1080 
create_this_dir_2(KDirectory * dir,const char * dir_name,bool force)1081 rc_t create_this_dir_2( KDirectory * dir, const char * dir_name, bool force )
1082 {
1083     KCreateMode create_mode = force ? kcmInit : kcmCreate;
1084     rc_t rc = KDirectoryCreateDir( dir, 0774, create_mode | kcmParents, "%s", dir_name );
1085     if ( 0 != rc )
1086     {
1087         ErrMsg( "create_this_dir_2().KDirectoryCreateDir( '%s' ) -> %R", dir_name, rc );
1088     }
1089     return rc;
1090 }
1091 
make_buffered_for_read(KDirectory * dir,const struct KFile ** f,const char * filename,size_t buf_size)1092 rc_t make_buffered_for_read( KDirectory * dir, const struct KFile ** f,
1093                              const char * filename, size_t buf_size )
1094 {
1095     const struct KFile * fr;
1096     rc_t rc = KDirectoryOpenFileRead( dir, &fr, "%s", filename );
1097     if ( 0 != rc )
1098     {
1099         ErrMsg( "make_buffered_for_read().KDirectoryOpenFileRead( '%s' ) -> %R", filename, rc );
1100     }
1101     else
1102     {
1103         if ( buf_size > 0 )
1104         {
1105             const struct KFile * fb;
1106             rc = KBufFileMakeRead( &fb, fr, buf_size );
1107             if ( 0 != rc )
1108             {
1109                 ErrMsg( "make_buffered_for_read( '%s' ).KBufFileMakeRead() -> %R", filename, rc );
1110             }
1111             else
1112             {
1113                 rc = KFileRelease( fr );
1114                 if ( 0 != rc )
1115                 {
1116                     ErrMsg( "make_buffered_for_read( '%s' ).KFileRelease().1 -> %R", filename, rc );
1117                 }
1118                 else
1119                 {
1120                     fr = fb;
1121                 }
1122             }
1123         }
1124 
1125         if ( 0 == rc )
1126         {
1127             *f = fr;
1128         }
1129         else
1130         {
1131             rc_t rc2 = KFileRelease( fr );
1132             if ( 0 != rc2 )
1133             {
1134                 ErrMsg( "make_buffered_for_read( '%s' ).KFileRelease().2 -> %R", filename, rc2 );
1135             }
1136         }
1137     }
1138     return rc;
1139 }
1140 
1141 /* ===================================================================================== */
1142 
locked_file_list_init(locked_file_list * self,uint32_t alloc_blocksize)1143 rc_t locked_file_list_init( locked_file_list * self, uint32_t alloc_blocksize )
1144 {
1145     rc_t rc;
1146     if ( NULL == self || 0 == alloc_blocksize )
1147     {
1148         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1149         ErrMsg( "locked_file_list_init() -> %R", rc );
1150     }
1151     else
1152     {
1153         rc = KLockMake ( &( self -> lock ) );
1154         if ( 0 != rc )
1155         {
1156             ErrMsg( "locked_file_list_init().KLockMake() -> %R", rc );
1157         }
1158         else
1159         {
1160             rc = VNamelistMake ( & self -> files, alloc_blocksize );
1161             if ( 0 != rc )
1162             {
1163                 ErrMsg( "locked_file_list_init().VNamelistMake() -> %R", rc );
1164             }
1165         }
1166     }
1167     return rc;
1168 }
1169 
locked_file_list_release(locked_file_list * self,KDirectory * dir)1170 rc_t locked_file_list_release( locked_file_list * self, KDirectory * dir )
1171 {
1172     rc_t rc = 0;
1173     /* tolerates to be called with self == NULL */
1174     if ( NULL != self )
1175     {
1176         rc = KLockRelease ( self -> lock );
1177         if ( 0 != rc )
1178         {
1179             ErrMsg( "locked_file_list_release().KLockRelease() -> %R", rc );
1180         }
1181         else
1182         {
1183             /* tolerates to be called with dir == NULL */
1184             if ( NULL != dir )
1185             {
1186                 rc = delete_files( dir, self -> files );
1187             }
1188         }
1189         {
1190             rc_t rc2 = VNamelistRelease ( self -> files );
1191             if ( 0 != rc2 )
1192             {
1193                 ErrMsg( "locked_file_list_release().VNamelistRelease() -> %R", rc );
1194             }
1195         }
1196     }
1197     return rc;
1198 }
1199 
locked_file_list_unlock(const locked_file_list * self,const char * function,rc_t rc)1200 static rc_t locked_file_list_unlock( const locked_file_list * self, const char * function, rc_t rc )
1201 {
1202     rc_t rc2 = KLockUnlock ( self -> lock );
1203     if ( 0 != rc2 )
1204     {
1205         ErrMsg( "%s().KLockUnlock() -> %R", function, rc2 );
1206         rc = ( 0 == rc ) ? rc2 : rc;
1207     }
1208     return rc;
1209 }
1210 
locked_file_list_append(const locked_file_list * self,const char * filename)1211 rc_t locked_file_list_append( const locked_file_list * self, const char * filename )
1212 {
1213     rc_t rc = 0;
1214     if ( NULL == self || NULL == filename )
1215     {
1216         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1217         ErrMsg( "locked_file_list_append() -> %R", rc );
1218     }
1219     else
1220     {
1221         rc = KLockAcquire ( self -> lock );
1222         if ( 0 != rc )
1223         {
1224             ErrMsg( "locked_file_list_append( '%s' ).KLockAcquire() -> %R", filename, rc );
1225         }
1226         else
1227         {
1228             rc = VNamelistAppend ( self -> files, filename );
1229             if ( 0 != rc )
1230             {
1231                 ErrMsg( "locked_file_list_append( '%s' ).VNamelistAppend() -> %R", filename, rc );
1232             }
1233             rc = locked_file_list_unlock( self, "locked_file_list_append", rc );
1234         }
1235     }
1236     return rc;
1237 }
1238 
locked_file_list_delete_files(KDirectory * dir,locked_file_list * self)1239 rc_t locked_file_list_delete_files( KDirectory * dir, locked_file_list * self )
1240 {
1241     rc_t rc = 0;
1242     if ( NULL == self || NULL == dir )
1243     {
1244         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1245         ErrMsg( "locked_file_list_delete_files() -> %R", rc );
1246     }
1247     else
1248     {
1249         rc = KLockAcquire ( self -> lock );
1250         if ( 0 != rc )
1251         {
1252             ErrMsg( "locked_file_list_delete_files().KLockAcquire() -> %R", rc );
1253         }
1254         else
1255         {
1256             rc = delete_files( dir, self -> files );
1257             if ( 0 != rc )
1258             {
1259                 ErrMsg( "locked_file_list_delete_files().delete_files() -> %R", rc );
1260             }
1261             rc = locked_file_list_unlock( self, "locked_file_list_delete_files", rc );
1262         }
1263     }
1264     return rc;
1265 }
1266 
locked_file_list_delete_dirs(KDirectory * dir,locked_file_list * self)1267 rc_t locked_file_list_delete_dirs( KDirectory * dir, locked_file_list * self )
1268 {
1269     rc_t rc = 0;
1270     if ( NULL == self || NULL == dir )
1271     {
1272         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1273         ErrMsg( "locked_file_list_delete_dirs() -> %R", rc );
1274     }
1275     else
1276     {
1277         rc = KLockAcquire ( self -> lock );
1278         if ( 0 != rc )
1279         {
1280             ErrMsg( "locked_file_list_delete_dirs().KLockAcquire() -> %R", rc );
1281         }
1282         else
1283         {
1284             rc = delete_dirs( dir, self -> files );
1285             if ( 0 != rc )
1286             {
1287                 ErrMsg( "locked_file_list_delete_dirs().delete_dirs() -> %R", rc );
1288             }
1289             rc = locked_file_list_unlock( self, "locked_file_list_delete_dirs", rc );
1290         }
1291     }
1292     return rc;
1293 }
1294 
locked_file_list_count(const locked_file_list * self,uint32_t * count)1295 rc_t locked_file_list_count( const locked_file_list * self, uint32_t * count )
1296 {
1297     rc_t rc = 0;
1298     if ( NULL == self || NULL == count )
1299     {
1300         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1301         ErrMsg( "locked_file_list_delete_dirs() -> %R", rc );
1302     }
1303     else
1304     {
1305         rc = KLockAcquire ( self -> lock );
1306         if ( 0 != rc )
1307         {
1308             ErrMsg( "locked_file_list_count().KLockAcquire() -> %R", rc );
1309         }
1310         else
1311         {
1312             rc = VNameListCount( self -> files, count );
1313             if ( 0 != rc )
1314             {
1315                 ErrMsg( "locked_file_list_count().VNameListCount() -> %R", rc );
1316             }
1317             rc = locked_file_list_unlock( self, "locked_file_list_count", rc );
1318         }
1319     }
1320     return rc;
1321 }
1322 
locked_file_list_pop(locked_file_list * self,const String ** item)1323 rc_t locked_file_list_pop( locked_file_list * self, const String ** item )
1324 {
1325     rc_t rc = 0;
1326     if ( NULL == self || NULL == item )
1327     {
1328         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1329         ErrMsg( "locked_file_list_pop() -> %R", rc );
1330     }
1331     else
1332     {
1333         *item = NULL;
1334         rc = KLockAcquire ( self -> lock );
1335         if ( 0 != rc )
1336         {
1337             ErrMsg( "locked_file_list_pop().KLockAcquire() -> %R", rc );
1338         }
1339         else
1340         {
1341             const char * s;
1342             rc = VNameListGet ( self -> files, 0, &s );
1343             if ( 0 != rc )
1344             {
1345                 ErrMsg( "locked_file_list_pop().VNameListGet() -> %R", rc );
1346             }
1347             else
1348             {
1349                 String S;
1350                 StringInitCString( &S, s );
1351                 rc = StringCopy ( item, &S );
1352                 if ( 0 != rc )
1353                 {
1354                     ErrMsg( "locked_file_list_pop().StringCopy() -> %R", rc );
1355                 }
1356                 else
1357                 {
1358                     rc = VNamelistRemoveIdx( self -> files, 0 );
1359                     if ( 0 != rc )
1360                     {
1361                         ErrMsg( "locked_file_list_pop().VNamelistRemoveIdx() -> %R", rc );
1362                     }
1363                 }
1364             }
1365             rc = locked_file_list_unlock( self, "locked_file_list_pop", rc );
1366         }
1367     }
1368     return rc;
1369 }
1370 
1371 /* ===================================================================================== */
1372 
locked_vector_init(locked_vector * self,uint32_t alloc_blocksize)1373 rc_t locked_vector_init( locked_vector * self, uint32_t alloc_blocksize )
1374 {
1375     rc_t rc;
1376     if ( NULL == self || 0 == alloc_blocksize )
1377     {
1378         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1379         ErrMsg( "locked_vector_init() -> %R", rc );
1380     }
1381     else
1382     {
1383         rc = KLockMake ( &( self -> lock ) );
1384         if ( 0 != rc )
1385         {
1386             ErrMsg( "locked_vector_init().KLockMake() -> %R", rc );
1387         }
1388         else
1389         {
1390             VectorInit ( &( self -> vector ), 0, alloc_blocksize );
1391             self -> sealed = false;
1392         }
1393     }
1394     return rc;
1395 }
1396 
locked_vector_unlock(const locked_vector * self,const char * function,rc_t rc)1397 static rc_t locked_vector_unlock( const locked_vector * self, const char * function, rc_t rc )
1398 {
1399     rc_t rc2 = KLockUnlock ( self -> lock );
1400     if ( 0 != rc2 )
1401     {
1402         ErrMsg( "%s().KLockUnlock() -> %R", function, rc2 );
1403         rc = ( 0 == rc ) ? rc2 : rc;
1404     }
1405     return rc;
1406 }
1407 
locked_vector_release(locked_vector * self,void (CC * whack)(void * item,void * data),void * data)1408 void locked_vector_release( locked_vector * self,
1409                             void ( CC * whack ) ( void *item, void *data ), void *data )
1410 {
1411     if ( NULL == self )
1412     {
1413         rc_t rc = KLockAcquire ( self -> lock );
1414         if ( 0 != rc )
1415         {
1416             ErrMsg( "locked_vector_release().KLockAcquire() -> %R", rc );
1417         }
1418         else
1419         {
1420             VectorWhack ( &( self -> vector ), whack, data );
1421             rc = locked_vector_unlock( self, "locked_vector_release", rc );
1422         }
1423         rc = KLockRelease ( self -> lock );
1424         if ( 0 != rc )
1425         {
1426             ErrMsg( "locked_vector_release().KLockRelease() -> %R", rc );
1427         }
1428     }
1429 }
1430 
locked_vector_push(locked_vector * self,const void * item,bool seal)1431 rc_t locked_vector_push( locked_vector * self, const void * item, bool seal )
1432 {
1433     rc_t rc;
1434     if ( NULL == self || NULL == item )
1435     {
1436         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1437         ErrMsg( "locked_vector_push() -> %R", rc );
1438     }
1439     else
1440     {
1441         rc = KLockAcquire ( self -> lock );
1442         if ( 0 != rc )
1443         {
1444             ErrMsg( "locked_vector_push().KLockAcquire -> %R", rc );
1445         }
1446         else
1447         {
1448             rc = VectorAppend ( &( self -> vector ), NULL, item );
1449             if ( 0 != rc )
1450             {
1451                 ErrMsg( "locked_vector_push().VectorAppend -> %R", rc );
1452             }
1453             if ( seal )
1454             {
1455                 self -> sealed = true;
1456             }
1457             rc = locked_vector_unlock( self, "locked_vector_push", rc );
1458         }
1459     }
1460     return rc;
1461 }
1462 
locked_vector_pop(locked_vector * self,void ** item,bool * sealed)1463 rc_t locked_vector_pop( locked_vector * self, void ** item, bool * sealed )
1464 {
1465     rc_t rc;
1466     if ( NULL == self || NULL == item || NULL == sealed )
1467     {
1468         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1469         ErrMsg( "locked_vector_pop() -> %R", rc );
1470     }
1471     else
1472     {
1473         rc = KLockAcquire ( self -> lock );
1474         if ( 0 != rc )
1475         {
1476             ErrMsg( "locked_vector_pop().KLockAcquire -> %R", rc );
1477         }
1478         else
1479         {
1480             if ( 0 == VectorLength( &( self -> vector ) ) )
1481             {
1482                 rc = 0;
1483                 *sealed = self -> sealed;
1484                 *item = NULL;
1485             }
1486             else
1487             {
1488                 *sealed = false;
1489                 rc = VectorRemove ( &( self -> vector ), 0, item );
1490                 if ( 0 != rc )
1491                 {
1492                     ErrMsg( "locked_vector_pop().VectorRemove -> %R", rc );
1493                 }
1494             }
1495             rc = locked_vector_unlock( self, "locked_vector_pop", rc );
1496         }
1497     }
1498     return rc;
1499 }
1500 
1501 /* ===================================================================================== */
locked_value_init(locked_value * self,uint64_t init_value)1502 rc_t locked_value_init( locked_value * self, uint64_t init_value )
1503 {
1504     rc_t rc;
1505     if ( NULL == self )
1506     {
1507         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1508         ErrMsg( "locked_value_init() -> %R", rc );
1509     }
1510     else
1511     {
1512         rc = KLockMake ( &( self -> lock ) );
1513         if ( 0 != rc )
1514         {
1515             ErrMsg( "locked_value_init().KLockMake() -> %R", rc );
1516         }
1517         else
1518         {
1519             self -> value = init_value;
1520         }
1521     }
1522     return rc;
1523 }
1524 
locked_value_release(locked_value * self)1525 void locked_value_release( locked_value * self )
1526 {
1527     if ( NULL != self )
1528     {
1529         rc_t rc = KLockRelease ( self -> lock );
1530         if ( 0 != rc )
1531         {
1532             ErrMsg( "locked_value_init().KLockRelease() -> %R", rc );
1533         }
1534     }
1535 }
1536 
locked_value_get(locked_value * self,uint64_t * value)1537 rc_t locked_value_get( locked_value * self, uint64_t * value )
1538 {
1539     rc_t rc;
1540     if ( NULL == self || NULL == value )
1541     {
1542         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1543         ErrMsg( "locked_value_get() -> %R", rc );
1544     }
1545     else
1546     {
1547         rc = KLockAcquire ( self -> lock );
1548         if ( 0 != rc )
1549         {
1550             ErrMsg( "locked_value_get().KLockAcquire -> %R", rc );
1551         }
1552         else
1553         {
1554             *value = self -> value;
1555             rc = KLockUnlock ( self -> lock );
1556             if ( 0 != rc )
1557             {
1558                 ErrMsg( "locked_value_get().KLockUnlock -> %R", rc );
1559             }
1560         }
1561     }
1562     return rc;
1563 }
1564 
locked_value_set(locked_value * self,uint64_t value)1565 rc_t locked_value_set( locked_value * self, uint64_t value )
1566 {
1567     rc_t rc;
1568     if ( NULL == self )
1569     {
1570         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1571         ErrMsg( "locked_value_set() -> %R", rc );
1572     }
1573     else
1574     {
1575         rc = KLockAcquire ( self -> lock );
1576         if ( 0 != rc )
1577         {
1578             ErrMsg( "locked_value_set().KLockAcquire -> %R", rc );
1579         }
1580         else
1581         {
1582             self -> value = value;
1583             rc = KLockUnlock ( self -> lock );
1584             if ( 0 != rc )
1585             {
1586                 ErrMsg( "locked_value_set().KLockUnlock -> %R", rc );
1587             }
1588         }
1589     }
1590     return rc;
1591 }
1592 
1593 /* ===================================================================================== */
1594 typedef struct Buf2NA
1595 {
1596     unsigned char map [ 1 << ( sizeof ( char ) * 8 ) ];
1597     size_t shiftLeft[ 4 ];
1598     NucStrstr * nss;
1599     uint8_t * buffer;
1600     size_t allocated;
1601 } Buf2NA;
1602 
make_Buf2NA(Buf2NA ** self,size_t size,const char * pattern)1603 rc_t make_Buf2NA( Buf2NA ** self, size_t size, const char * pattern )
1604 {
1605     rc_t rc = 0;
1606     NucStrstr * nss;
1607     int res = NucStrstrMake ( &nss, 0, pattern, string_size ( pattern ) );
1608     if ( 0 != res )
1609     {
1610         rc = RC( rcVDB, rcNoTarg, rcConstructing, rcParam, rcInvalid );
1611         ErrMsg( "make_Buf2NA().NucStrstrMake() -> %R", rc );
1612     }
1613     else
1614     {
1615         uint8_t * buffer = calloc( size, sizeof * buffer );
1616         if ( NULL == buffer )
1617         {
1618             rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
1619             ErrMsg( "make_Buf2NA().calloc().1() -> %R", rc );
1620             NucStrstrWhack ( nss );
1621         }
1622         else
1623         {
1624             Buf2NA * res = calloc( 1, sizeof * res );
1625             if ( NULL == res )
1626             {
1627                 rc = RC( rcVDB, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
1628                 ErrMsg( "make_Buf2NA().calloc().2() -> %R", rc );
1629                 NucStrstrWhack ( nss );
1630                 free( ( void * ) buffer );
1631             }
1632             else
1633             {
1634                 res -> nss = nss;
1635                 res -> buffer = buffer;
1636                 res -> allocated = size;
1637                 res -> map[ 'A' ] = res -> map[ 'a' ] = 0;
1638                 res -> map[ 'C' ] = res -> map[ 'c' ] = 1;
1639                 res -> map[ 'G' ] = res -> map[ 'g' ] = 2;
1640                 res -> map[ 'T' ] = res -> map[ 't' ] = 3;
1641                 res -> shiftLeft [ 0 ] = 6;
1642                 res -> shiftLeft [ 1 ] = 4;
1643                 res -> shiftLeft [ 2 ] = 2;
1644                 res -> shiftLeft [ 3 ] = 0;
1645                 *self = res;
1646             }
1647         }
1648     }
1649     return rc;
1650 }
1651 
release_Buf2NA(Buf2NA * self)1652 void release_Buf2NA( Buf2NA * self )
1653 {
1654     if ( self != NULL )
1655     {
1656         if ( self -> buffer != NULL )
1657             free( ( void * ) self -> buffer );
1658         if ( self -> nss != NULL )
1659             NucStrstrWhack ( self -> nss );
1660         free( ( void * ) self );
1661     }
1662 }
1663 
match_Buf2NA(Buf2NA * self,const String * ascii)1664 bool match_Buf2NA( Buf2NA * self, const String * ascii )
1665 {
1666     bool res = false;
1667     if ( self != NULL && ascii != NULL )
1668     {
1669         int i;
1670         size_t needed = ( ( ascii -> len + 3 ) / 4 );
1671         if ( needed > self -> allocated )
1672         {
1673             free( ( void * )self -> buffer );
1674             self -> buffer = calloc( needed, sizeof *( self -> buffer ) );
1675         }
1676         else
1677             memset( self -> buffer, 0, needed );
1678 
1679         if ( self -> buffer != NULL )
1680         {
1681             unsigned int selflen;
1682             int dst = 0;
1683             int src = 0;
1684             i = ascii -> len;
1685             while ( i >= 4 )
1686             {
1687                 self -> buffer[ dst++ ] =
1688                     self -> map[ ( unsigned char )ascii -> addr[ src ] ] << 6 |
1689                     self -> map[ ( unsigned char )ascii -> addr[ src + 1 ] ] << 4 |
1690                     self -> map[ ( unsigned char )ascii -> addr[ src + 2 ] ] << 2 |
1691                     self -> map[ ( unsigned char )ascii -> addr[ src + 3 ] ];
1692                 src += 4;
1693                 i -= 4;
1694             }
1695             switch( i )
1696             {
1697                 case 3 : self -> buffer[ dst ] =
1698                             self -> map[ ( unsigned char )ascii -> addr[ src ] ] << 6 |
1699                             self -> map[ ( unsigned char )ascii -> addr[ src + 1 ] ] << 4 |
1700                             self -> map[ ( unsigned char )ascii -> addr[ src + 2 ] ] << 2; break;
1701                 case 2 : self -> buffer[ dst ] =
1702                             self -> map[ ( unsigned char )ascii -> addr[ src ] ] << 6 |
1703                             self -> map[ ( unsigned char )ascii -> addr[ src + 1 ] ] << 4; break;
1704                 case 1 : self -> buffer[ dst ] =
1705                             self -> map[ ( unsigned char )ascii -> addr[ src ] ] << 6; break;
1706             }
1707             res = ( 0 != NucStrstrSearch ( self -> nss, self -> buffer, 0, ascii -> len, & selflen ) );
1708         }
1709     }
1710     return res;
1711 }
1712 
helper_make_thread(KThread ** self,rc_t (CC * run_thread)(const KThread * self,void * data),void * data,size_t stacksize)1713 rc_t helper_make_thread( KThread ** self,
1714                          rc_t ( CC * run_thread ) ( const KThread * self, void * data ),
1715                          void * data,
1716                          size_t stacksize )
1717 {
1718     rc_t rc = KThreadMakeStackSize( self, run_thread, data, stacksize );
1719     return rc;
1720 }
1721 
1722 /* ===================================================================================== */
1723 static atomic32_t quit_flag;
1724 
get_quitting(void)1725 rc_t get_quitting( void )
1726 {
1727     rc_t rc = Quitting();
1728     if ( 0 == rc )
1729     {
1730         if ( 0 != atomic32_read ( & quit_flag ) )
1731         rc = RC ( rcExe, rcProcess, rcExecuting, rcProcess, rcCanceled );
1732     }
1733     return rc;
1734 }
1735 
set_quitting(void)1736 void set_quitting( void )
1737 {
1738     atomic32_inc ( & quit_flag );
1739 }
1740