1 /*===========================================================================
2  *
3  *                            PUBLIC DOMAIN NOTICE
4  *               National Center for Biotechnology Information
5  *
6  *  This software/database is a "United States Government Work" under the
7  *  terms of the United States Copyright Act.  It was written as part of
8  *  the author's official duties as a United States Government employee and
9  *  thus cannot be copyrighted.  This software/database is freely available
10  *  to the public for use. The National Library of Medicine and the U.S.
11  *  Government have not placed any restriction on its use or reproduction.
12  *
13  *  Although all reasonable efforts have been taken to ensure the accuracy
14  *  and reliability of the software and data, the NLM and the U.S.
15  *  Government do not and cannot warrant the performance or results that
16  *  may be obtained by using this software or data. The NLM and the U.S.
17  *  Government disclaim all warranties, express or implied, including
18  *  warranties of performance, merchantability or fitness for any particular
19  *  purpose.
20  *
21  *  Please cite the author in any work or product based on this material.
22  *
23  * ===========================================================================
24  *
25  */
26 
27 #include <kfg/config.h> /* KConfigMake */
28 #include <kfg/kart-priv.h> /* KartMake2 */
29 #include <kfg/keystore.h> /* KKeyStoreRelease */
30 #include <kfg/ngc.h> /* KNgcObj */
31 #include <kfg/repository.h> /* KRepositoryMgr */
32 
33 #include <kfs/directory.h> /* KDirectoryOpenFileRead */
34 #include <kfs/file.h> /* KFile */
35 #include <kfs/gzip.h> /* KFileMakeGzipForRead */
36 #include <kfs/subfile.h> /* KFileMakeSubRead */
37 
38 #include <klib/data-buffer.h> /* KDataBuffer */
39 #include <klib/out.h> /* OUTMSG */
40 #include <klib/printf.h> /* string_printf */
41 #include <klib/rc.h>
42 #include <klib/refcount.h> /* KRefcount */
43 #include <klib/vector.h> /* Vector */
44 
45 #include <strtol.h> /* strtou64 */
46 #include <sysalloc.h>
47 
48 #include <assert.h>
49 #include <stdlib.h> /* free */
50 #include <string.h> /* memcmp */
51 
52 #define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
53     if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
54 
55 typedef enum {
56     eVersion1, // from kart 0x01000000
57     eVersion2, // from kart 0x02000000
58 } Eversion;
59 
60 struct KartItem {
61     KRefcount refcount;
62 
63     const Kart *dad;
64 
65     Eversion version;
66 
67     String projId;
68     String itemId;
69     String accession; /* 1.0 */
70     String name;
71     String itemDesc;
72 
73     /* 2.0 */
74     String objType;
75     String path;
76     String size;
77 };
78 
KartItemWhack(KartItem * self)79 static void KartItemWhack(KartItem *self) {
80     assert(self);
81 
82     if ( self -> version < eVersion2 ) {
83         KartRelease(self->dad);
84     } else {
85         free ( ( void * ) self -> projId . addr );
86     }
87 
88     memset(self, 0, sizeof *self);
89 
90     free(self);
91 }
92 
KartItemMake2(KartItem ** self,const char * buffer,size_t size)93 static rc_t KartItemMake2
94     ( KartItem ** self, const char * buffer, size_t size )
95 {
96     bool BUG = false;
97     rc_t rc = 0;
98     KartItem * obj = NULL;
99     int i = 0;
100     assert ( self );
101     obj = calloc ( 1, sizeof * obj );
102     if ( obj == NULL ) {
103         return RC ( rcKFG, rcData, rcAllocating, rcMemory, rcExhausted );
104     }
105     obj -> version = eVersion2; /* 0x02000000; */
106     for ( i = 0; ; ++i ) {
107         size_t l = 0;
108         String * next = NULL;
109         const char * p = string_chr ( buffer, size, '|' );
110         if ( p == NULL ) {
111             if ( i != 7 ) {
112                 rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInsufficient);
113                 break;
114             }
115             l = size;
116         }
117         else {
118             l = p - buffer;
119         }
120         switch ( i ) {
121             case 0:
122                 next = & obj -> projId;
123                 break;
124             case 1:
125                 next = & obj -> objType;
126                 break;
127             case 2:
128                 next = & obj -> itemId;
129                 break;
130             case 3:
131                 next = & obj -> name;
132                 break;
133             case 4:
134                 next = & obj -> path;
135                 break;
136             case 5:
137                 next = & obj -> size;
138                 break;
139             case 6:
140                 next = & obj -> itemDesc;
141                 break;
142             case 7:
143                 BUG = true;
144                 break;
145             default:
146                 rc = RC ( rcKFG, rcFile, rcParsing, rcFile, rcExcessive );
147                 break;
148         }
149         if ( ! BUG ) {
150             assert ( next );
151             StringInit ( next, buffer, l, ( uint32_t ) l );
152             if ( l > size ) {
153                 rc = RC ( rcKFG, rcFile, rcParsing, rcFile, rcInvalid );
154             }
155         }
156         if ( size == l ) {
157             break;
158         }
159         ++ l;
160         buffer += l;
161         size -= l;
162     }
163     if (rc == 0) {
164         KRefcountInit ( & obj -> refcount, 1,
165             "KartItem", "KartItemMake2", "kartitem" );
166         * self = obj;
167     }
168     else {
169         free ( obj );
170         obj = NULL;
171     }
172     return rc;
173 }
174 
175 /* AddRef
176  * Release
177  *  all objects are reference counted
178  *  NULL references are ignored
179  */
KartItemAddRef(const KartItem * self)180 LIB_EXPORT rc_t CC KartItemAddRef(const KartItem *self) {
181     if (self != NULL) {
182         switch (KRefcountAdd(&self->refcount, "KartItem")) {
183             case krefLimit:
184                 return RC(rcKFG, rcFile, rcAttaching, rcRange, rcExcessive);
185         }
186     }
187 
188     return 0;
189 }
190 
KartItemRelease(const KartItem * self)191 LIB_EXPORT rc_t CC KartItemRelease(const KartItem *self) {
192     if (self != NULL) {
193         switch (KRefcountDrop(&self -> refcount, "KartItem")) {
194             case krefWhack:
195                 KartItemWhack((KartItem*)self);
196                 break;
197             case krefLimit:
198                 return RC(rcKFG, rcFile, rcReleasing, rcRange, rcExcessive);
199         }
200     }
201 
202     return 0;
203 }
204 
StringAsUint64(const String * self,uint64_t * pid)205 static rc_t StringAsUint64(const String *self, uint64_t *pid) {
206     uint64_t id = 0;
207 
208     char buffer[21] = "";
209     size_t bytes = 0;
210     char *end = NULL;
211 
212     assert(self);
213 
214     if (pid == NULL) {
215         return RC(rcKFG, rcFile, rcAccessing, rcParam, rcNull);
216     }
217 
218     *pid = 0;
219 
220     if (sizeof buffer - 1 < self->len) {
221         return RC(rcKFG, rcFile, rcAccessing, rcBuffer, rcInsufficient);
222     }
223 
224     if (self->len == 0 || self->size == 0) {
225         return RC(rcKFG, rcFile, rcAccessing, rcItem, rcEmpty);
226     }
227 
228     bytes = string_copy(buffer, sizeof buffer, self->addr, self->len);
229     if (bytes != self->len) {
230         return RC(rcKFG, rcFile, rcAccessing, rcBuffer, rcInsufficient);
231     }
232 
233     id = strtou64(buffer, &end, 0);
234     if (end[0] != 0) {
235         return RC(rcKFG, rcFile, rcAccessing, rcParam, rcInvalid);
236     }
237 
238     *pid = id;
239 
240     return 0;
241 }
242 
243 LIB_EXPORT
KartItemProjIdNumber(const KartItem * self,uint64_t * pid)244 rc_t CC KartItemProjIdNumber(const KartItem *self, uint64_t *pid)
245 {
246     if (self == NULL) {
247         return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
248     }
249     return StringAsUint64(&self->projId, pid);
250 }
251 
252 LIB_EXPORT
KartItemItemIdNumber(const KartItem * self,uint64_t * pid)253 rc_t CC KartItemItemIdNumber(const KartItem *self, uint64_t *pid)
254 {
255     if (self == NULL) {
256         return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
257     }
258     return StringAsUint64(&self->itemId, pid);
259 }
260 
KartItemCheck(const KartItem * self,const String ** elem)261 static rc_t KartItemCheck(const KartItem *self, const String **elem) {
262     if (elem == NULL) {
263         return RC(rcKFG, rcFile, rcAccessing, rcParam, rcNull);
264     }
265 
266     *elem = NULL;
267 
268     if (self == NULL) {
269         return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
270     }
271 
272     return 0;
273 }
274 
275 LIB_EXPORT
KartItemProjId(const KartItem * self,const String ** elem)276 rc_t CC KartItemProjId(const KartItem *self, const String **elem)
277 {
278     rc_t rc = KartItemCheck(self, elem);
279     if (rc == 0) {
280         *elem = &self->projId;
281     }
282     return rc;
283 }
284 LIB_EXPORT
KartItemItemId(const KartItem * self,const String ** elem)285 rc_t CC KartItemItemId(const KartItem *self, const String **elem)
286 {
287     rc_t rc = KartItemCheck(self, elem);
288     if (rc == 0) {
289         *elem = &self->itemId;
290     }
291     return rc;
292 }
293 LIB_EXPORT
KartItemAccession(const KartItem * self,const String ** elem)294 rc_t CC KartItemAccession(const KartItem *self, const String **elem)
295 {
296     rc_t rc = KartItemCheck(self, elem);
297     if (rc == 0) {
298         *elem = &self->accession;
299     }
300     return rc;
301 }
302 LIB_EXPORT
KartItemName(const KartItem * self,const String ** elem)303 rc_t CC KartItemName(const KartItem *self, const String **elem)
304 {
305     rc_t rc = KartItemCheck(self, elem);
306     if (rc == 0) {
307         *elem = &self->name;
308     }
309     return rc;
310 }
311 LIB_EXPORT
KartItemItemDesc(const KartItem * self,const String ** elem)312 rc_t CC KartItemItemDesc(const KartItem *self, const String **elem)
313 {
314     rc_t rc = KartItemCheck(self, elem);
315     if (rc == 0) {
316         *elem = &self->itemDesc;
317     }
318     return rc;
319 }
320 LIB_EXPORT
KartItemObjType(const KartItem * self,const String ** elem)321 rc_t CC KartItemObjType (const KartItem *self, const String **elem )
322 {
323     rc_t rc = KartItemCheck(self, elem);
324     if (rc == 0) {
325         *elem = &self->objType;
326     }
327     return rc;
328 }
329 LIB_EXPORT
KartItemPath(const KartItem * self,const String ** elem)330 rc_t CC KartItemPath (const KartItem *self, const String **elem )
331 {
332     rc_t rc = KartItemCheck(self, elem);
333     if (rc == 0) {
334         *elem = &self->path;
335     }
336     return rc;
337 }
338 LIB_EXPORT
KartItemSize(const KartItem * self,const String ** elem)339 rc_t CC KartItemSize (const KartItem *self, const String **elem )
340 {
341     rc_t rc = KartItemCheck(self, elem);
342     if (rc == 0) {
343         *elem = &self->size;
344     }
345     return rc;
346 }
347 /*LIB_EXPORT rc_t CC KartItemTypeId(const KartItem *self, const String **elem)
348 {
349     rc_t rc = KartItemCheck(self, elem);
350     if (rc == 0) {
351         *elem = &self->typeId;
352     }
353     return rc;
354 }*/
355 
356 /** Print KartItem using OUTMSG; if (self == NULL) then print the header */
KartItemPrint(const KartItem * self)357 LIB_EXPORT rc_t CC KartItemPrint(const KartItem *self) { /* AA-833 */
358     if (self != NULL) {
359         return OUTMSG(("'%S'\t'%S'\t'%S'\t'%S'\t'%S'\n", &self->projId,
360             &self->itemId, &self->accession, &self->name, &self->itemDesc));
361     }
362     return 0;
363 }
364 
365 struct Kart {
366     KRefcount refcount;
367 
368     const KRepositoryMgr *mgr;
369 
370     Eversion version;
371 
372     /* version eVersion1 0x01000000 */
373     KDataBuffer mem;
374     const char *text;
375     uint64_t len;
376     uint16_t itemsProcessed;
377 
378     KKeyStore *keystore;
379 
380     /* version eVersion2 0x02000000 */
381     Vector rows;
382 
383     const KNgcObj *ngcObj;
384 };
385 
whackKartItem(void * self,void * ignore)386 static void whackKartItem ( void * self, void * ignore ) {
387     KartItemRelease ( ( KartItem * ) self);
388 }
389 
KartWhack(Kart * self)390 static void KartWhack(Kart *self) {
391     assert(self);
392 
393     if ( self -> version < eVersion2 ) {
394         KDataBufferWhack(&self->mem);
395     } else {
396         VectorWhack ( & self -> rows, whackKartItem, NULL );
397     }
398 
399     KKeyStoreRelease ( self -> keystore );
400 
401     KRepositoryMgrRelease ( self -> mgr );
402 
403     KNgcObjRelease ( self -> ngcObj );
404 
405     memset(self, 0, sizeof *self);
406 
407     free(self);
408 }
409 
410 /* AddRef
411  * Release
412  *  all objects are reference counted
413  *  NULL references are ignored
414  */
KartAddRef(const Kart * self)415 LIB_EXPORT rc_t CC KartAddRef(const Kart *self) {
416     if (self != NULL) {
417         switch (KRefcountAdd(&self->refcount, "Kart")) {
418             case krefLimit:
419                 return RC(rcKFG, rcFile, rcAttaching, rcRange, rcExcessive);
420         }
421     }
422 
423     return 0;
424 }
425 
KartRelease(const Kart * self)426 LIB_EXPORT rc_t CC KartRelease(const Kart *self) {
427     if (self != NULL) {
428         switch (KRefcountDrop(&self -> refcount, "Kart")) {
429             case krefWhack:
430                 KartWhack((Kart*)self);
431                 break;
432             case krefLimit:
433                 return RC(rcKFG, rcFile, rcReleasing, rcRange, rcExcessive);
434         }
435     }
436 
437     return 0;
438 }
439 
440 LIB_EXPORT
KartItemsProcessed(const Kart * self,uint16_t * number)441 rc_t CC KartItemsProcessed(const Kart *self, uint16_t *number)
442 {
443     if (number == NULL) {
444         return RC(rcKFG, rcFile, rcLoading, rcParam, rcNull);
445     }
446 
447     *number = 0;
448 
449     if (self == NULL) {
450         return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
451     }
452 
453     *number = self->itemsProcessed;
454 
455     return 0;
456 }
457 
KartItemInitFromKartRow(const Kart * self,const KartItem ** item,const char * line,size_t len)458 static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
459     const char *line, size_t len)
460 {
461     rc_t rc = 0;
462     int i = 0;
463     KartItem *obj = NULL;
464     assert(self && item && line && len);
465     obj = calloc(1, sizeof *obj);
466     if (obj == NULL) {
467         return RC(rcKFG, rcData, rcAllocating, rcMemory, rcExhausted);
468     }
469     for (i = 0; ; ++i) {
470         size_t l = 0;
471         String *next = NULL;
472         const char *p = string_chr(line, len, '|');
473         if (p == NULL) {
474             if (i != 4) {
475                 rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInsufficient);
476                 break;
477             }
478             l = len;
479         }
480         else {
481             l = p - line;
482         }
483         switch (i) { /* AA-833 */
484             case 0:
485                 next = &obj->projId;
486                 break;
487             case 1:
488                 next = &obj->itemId;
489                 break;
490             case 2:
491                 next = &obj->accession;
492                 break;
493             case 3:
494                 next = &obj->name;
495                 break;
496             case 4:
497                 next = &obj->itemDesc;
498                 break;
499             default:
500                 rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcExcessive);
501                 break;
502         }
503         assert(next);
504         StringInit(next, line, l, (uint32_t)l);
505         if (l > len) {
506             rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInvalid);
507         }
508         if (len == l) {
509             break;
510         }
511         ++l;
512         line += l;
513         len -= l;
514     }
515     if (rc == 0) {
516         rc = KartAddRef(self);
517     }
518     if (rc == 0) {
519         KRefcountInit(&obj->refcount, 1, "KartItem", "KartItemInitFromKartRow", "kartitem");
520         ++((Kart*)self)->itemsProcessed;
521         obj->dad = self;
522         *item = obj;
523     }
524     else {
525         free(obj);
526         obj = NULL;
527     }
528     return rc;
529 }
530 
KartPrint(const Kart * self)531 LIB_EXPORT rc_t CC KartPrint(const Kart *self) {
532     if (self == NULL)
533         return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
534 
535     if ( self -> version == eVersion1 ) {
536         uint32_t l = ( uint32_t ) self -> mem . elem_count;
537         OUTMSG ( ( "%.*s", l, self -> mem.base ) );
538     }
539     else {
540         uint32_t i = 0;
541         uint32_t l = VectorLength ( & self -> rows );
542         for ( i = 0; i < l; ++ i ) {
543             KartItem * result = VectorGet ( & self -> rows, i );
544             assert ( result );
545             OUTMSG ( ( "%S|%S|%S|%S|%S|%S|%S\n", & result -> projId,
546                 & result -> objType, & result -> itemId, & result -> name,
547                 & result -> path, & result -> size, & result -> itemDesc ) );
548         }
549     }
550     return 0;
551 }
552 
KartPrintNumbered(const Kart * self)553 LIB_EXPORT rc_t CC KartPrintNumbered(const Kart *self) {
554     rc_t rc = 0;
555     rc_t rc2 = 0;
556     const char *start = NULL;
557     int32_t remaining = 0;
558     int32_t len = 0;
559     uint32_t i = 0;
560     const char *next = NULL;
561     bool done = false;
562 
563     if (self == NULL) {
564         return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
565     }
566 
567     if ( self -> version > eVersion1 )
568         return RC ( rcKFG, rcFile, rcAccessing, rcInterface, rcBadVersion );
569 
570     remaining = (uint32_t)self->mem.elem_count;
571     start = self->mem.base;
572 
573     {
574         const char version[] = "version ";
575         size_t l = sizeof version - 1;
576         if (string_cmp(version, l, start, remaining, (uint32_t)l) != 0) {
577             return RC(rcKFG, rcFile, rcAccessing, rcFormat, rcUnrecognized);
578         }
579     }
580 
581     next = string_chr(start, remaining, '\n');
582     if (next == NULL) {
583         len = remaining;
584     }
585     else {
586         ++next;
587         len = next - start;
588     }
589     remaining -= len;
590     rc2 = OUTMSG(("%.*s", len, start));
591     if (rc2 != 0 && rc == 0) {
592         rc = rc2;
593     }
594     start = next;
595 
596     rc2 = OUTMSG(("row\tproj-id|item-id|accession|name|item-desc\n"));
597     if (rc2 != 0 && rc == 0) {
598         rc = rc2;
599     }
600 
601     for (i = 1; remaining > 0; ++i) {
602         if (*start == '$') {
603             const char end[] = "$end";
604             size_t l = sizeof end - 1;
605             if (string_cmp(end, l, start, remaining, (uint32_t)l) != 0) {
606                 return RC(rcKFG, rcFile, rcAccessing, rcFormat, rcUnrecognized);
607             }
608             else {
609                 done = true;
610             }
611         }
612         next = string_chr(start, remaining, '\n');
613         if (next == NULL) {
614             len = remaining;
615         }
616         else {
617             ++next;
618             len = next - start;
619         }
620         remaining -= len;
621         if (done) {
622             rc2 = OUTMSG(("%.*s", len, start));
623         }
624         else {
625             rc2 = OUTMSG(("%d\t%.*s", i, len, start));
626         }
627         if (rc2 != 0 && rc == 0) {
628             rc = rc2;
629         }
630         start = next;
631     }
632 
633     return rc;
634 }
635 
KartItemGetTicket(const KartItem * self,char * ticket,size_t ticket_size,size_t * written)636 LIB_EXPORT rc_t CC KartItemGetTicket(const KartItem *self,
637     char * ticket, size_t ticket_size, size_t * written)
638 {
639     if (self == NULL)
640         return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
641     return KNgcObjGetTicket(self->dad->ngcObj, ticket, ticket_size, written);
642 }
643 
644 static
KartRegisterObject(const Kart * self,const KartItem * item)645 rc_t KartRegisterObject ( const Kart * self, const KartItem * item )
646 {
647     rc_t rc = 0;
648 
649     const KRepository * repo = NULL;
650 
651     uint64_t itemId = 0;
652     uint64_t projId = 0;
653     const String * acc = NULL;
654     const String * name = NULL;
655 
656     char ticket [ 4096 ] = "";
657 
658     char b [ 4096 ] = "";
659     String id = { b, 0, 0 };
660 
661     if ( item == NULL )
662         return 0;
663 
664     rc = KartItemItemIdNumber ( item, & itemId );
665     if ( rc == SILENT_RC ( rcKFG, rcFile, rcAccessing, rcItem, rcEmpty ) ||
666 
667         /* not a number (accession) */
668          rc == SILENT_RC ( rcKFG, rcFile, rcAccessing, rcParam, rcInvalid ) )
669 
670     {   return 0; }
671 
672     assert ( self );
673 
674     if ( rc == 0 )
675         rc = KartItemProjIdNumber ( item, & projId );
676 
677     if ( rc == 0 )
678         rc = KartItemAccession ( item, & acc );
679 
680     if ( rc == 0 )
681         rc = KartItemName ( item, & name );
682 
683     if ( rc == 0 ) {
684         rc = KRepositoryMgrGetProtectedRepository ( self -> mgr, projId,
685                                                     & repo );
686         if ( GetRCModule ( rc ) == rcKFG && GetRCState ( rc ) == rcNotFound )
687             rc = RC ( rcKFG, rcNode, rcAccessing, rcNode, rcNotFound );
688 
689         if ( rc == 0 ) {
690             rc = KRepositoryDownloadTicket( repo, ticket, sizeof ticket, NULL );
691             if ( GetRCState ( rc ) == rcNotFound )
692                 rc = RC ( rcKFG, rcNode, rcAccessing, rcNode, rcNotFound );
693         }
694         else if (self->ngcObj != NULL)
695             rc = KNgcObjGetTicket(self->ngcObj, ticket, sizeof ticket, NULL);
696 
697         if ( rc == 0 ) {
698             if ( acc != NULL && acc -> size != 0 )
699                 rc = string_printf ( b, sizeof b, & id . size,
700                                  "ncbi-acc:%S?tic=%s", acc, ticket );
701             else
702                 rc = string_printf ( b, sizeof b, & id . size,
703                                  "ncbi-file:%S?tic=%s", name, ticket );
704             id . len = id . size;
705         }
706     }
707 
708     if ( rc == 0 )
709         rc = KKeyStoreRegisterObject ( self->keystore, itemId, & id );
710 
711     RELEASE ( KRepository, repo );
712 
713     return rc;
714 }
715 
KartReset(const Kart * cself)716 LIB_EXPORT rc_t CC KartReset(const Kart *cself) {
717     Kart * self = (Kart *)cself;
718     if (self == NULL)
719         return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
720 
721     if (self->version < eVersion2) {
722         self->text = self->mem.base;
723         self->len = self->mem.elem_count;
724     }
725     else
726         self->len = 0;
727 
728     return 0;
729 }
730 
731 LIB_EXPORT
KartMakeNextItem(const Kart * cself,const KartItem ** item)732 rc_t CC KartMakeNextItem ( const Kart * cself, const KartItem **item )
733 {
734     rc_t rc = 0;
735 
736     Kart * self = ( Kart * ) cself;
737     size_t len = 0;
738     const char *line = NULL;
739     const char *next = NULL;
740     const KartItem * result = NULL;
741 
742     if (item == NULL) {
743         return RC(rcKFG, rcFile, rcLoading, rcParam, rcNull);
744     }
745     *item = NULL;
746     if (self == NULL) {
747         return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
748     }
749 
750     if ( self -> version < eVersion2 ) {
751         while (self->len > 0
752             && (self->text[0] == '\r' || self->text[0] == '\n'))
753         {
754             ++self->text;
755             --self->len;
756         }
757 
758         line = self->text;
759         next = string_chr(self->text, ( size_t ) self->len, '\n');
760         if (next == NULL) {
761             return RC(rcKFG, rcFile, rcLoading, rcFile, rcInsufficient);
762         }
763 
764         len = next - self->text;
765         if (*(next - 1) == '\r') {
766             --len;
767         }
768 
769         if (self->len >= (uint64_t) (next - self->text + 1) ){
770             self->len -= next - self->text + 1;
771         }
772         else {
773             OUTMSG(("WARNING: STRING OVERFLOW DURING KART ROW PARSING"));
774             self->len = 0;
775         }
776 
777         self->text = next + 1;
778 
779         {
780             const char end[] = "$end";
781             if (string_cmp(line, len, end, sizeof end - 1, sizeof end - 1) == 0)
782             {
783                 return 0;
784             }
785         }
786 
787         rc = KartItemInitFromKartRow(self, & result, line, len);
788     }
789     else {
790         uint32_t l = VectorLength ( & self -> rows );
791         if ( self -> len < l ) {
792             result = VectorGet ( & self -> rows,
793                                             ( uint32_t ) self -> len ++ );
794             if ( result != NULL ) {
795                 rc = KartItemAddRef ( result );
796             }
797         }
798     }
799 
800     if ( rc == 0 )
801         rc = KartRegisterObject ( self, result );
802 
803     if ( rc == 0 )
804         * item = result;
805     else
806         KartItemRelease ( result );
807 
808     return rc;
809 }
810 
811 static
decode_kart(KDataBuffer * mem,const KFile * orig,size_t hdr_sz)812 rc_t decode_kart(KDataBuffer *mem, const KFile *orig, size_t hdr_sz)
813 {
814     rc_t rc = 0;
815     size_t num_read;
816     uint64_t eof;
817     assert(mem && orig && hdr_sz);
818     rc = KFileSize ( orig, & eof );
819     if ( rc == 0 )
820     {
821         const KFile *sub;
822         rc = KFileMakeSubRead(&sub, orig, hdr_sz, eof - hdr_sz);
823         if ( rc == 0 )
824         {
825             const KFile *gzip;
826             rc = KFileMakeGzipForRead ( & gzip, sub );
827             if ( rc == 0 )
828             {
829                 rc = KDataBufferMakeBytes ( mem, 0 );
830                 if ( rc == 0 )
831                 {
832                     size_t total, to_read;
833 
834                     /* after all of that, we're ready to decompress */
835                     for ( total = 0; ; )
836                     {
837                         char *buff;
838 
839                         rc = KDataBufferResize ( mem,
840                             total + 32 * 1024 );
841                         if ( rc != 0 )
842                             break;
843 
844                         buff = mem -> base;
845                         to_read = ( size_t ) mem -> elem_count - total;
846 
847                         rc = KFileReadAll ( gzip, total,
848                             & buff [ total ], to_read, & num_read );
849                         if ( rc != 0 )
850                             break;
851 
852                         total += num_read;
853 
854                         if ( num_read < to_read )
855                         {
856                             buff [ total ] = 0;
857                             mem -> elem_count = total;
858                             break;
859                         }
860                     }
861                 }
862 
863                 KFileRelease ( gzip );
864             }
865 
866             KFileRelease ( sub );
867         }
868     }
869 
870     return rc;
871 }
872 
KartProcessHeader(Kart * self)873 static rc_t KartProcessHeader(Kart *self) {
874     assert(self);
875 
876     self->text = self->mem.base;
877     self->len = self->mem.elem_count;
878 
879     {
880         const char version[] = "version ";
881         size_t l = sizeof version - 1;
882         if (string_cmp(version, l,
883                        self->text, ( size_t ) self->len, (uint32_t)l) != 0)
884         {
885             return RC(rcKFG, rcFile, rcAccessing, rcFormat, rcUnrecognized);
886         }
887 
888         self->text += l;
889         self->len -= l;
890     }
891 
892     {
893         const char version[] = "1.0";
894         size_t l = sizeof version - 1;
895         if (string_cmp(version, l, self->text, l, (uint32_t)l) != 0) {
896             return RC(rcKFG, rcFile, rcAccessing, rcFormat, rcUnsupported);
897         }
898 
899         self->text += l;
900         self->len -= l;
901     }
902 
903     while (self->len > 0 && (self->text[0] == '\r' || self->text[0] == '\n')) {
904         ++self->text;
905         --self->len;
906     }
907 
908     return 0;
909 }
910 
911 #ifdef _DEBUGGING
read_textkart(KDataBuffer * mem,const KFile * orig)912 static rc_t read_textkart(KDataBuffer *mem, const KFile *orig) {
913     rc_t rc = 0;
914     size_t num_read;
915     uint64_t eof;
916     assert(mem && orig);
917     rc = KFileSize ( orig, & eof );
918     if ( rc == 0 )
919     {
920         rc = KDataBufferMakeBytes ( mem, 0 );
921         if ( rc == 0 ) {
922             /* after all of that, we're ready to read */
923             rc = KDataBufferResize(mem, eof);
924             if ( rc != 0 )
925                 return rc;
926             rc = KFileReadAll ( orig, 0, mem -> base, ( size_t ) eof,
927                                 & num_read );
928             if ( rc != 0 )
929                 return rc;
930         }
931     }
932     return rc;
933 }
KartMakeText(const struct KDirectory * dir,const char * path,Kart ** kart,bool * isKart)934 KFG_EXTERN rc_t CC KartMakeText(const struct KDirectory *dir, const char *path,
935     Kart **kart, bool *isKart)
936 {
937     rc_t rc = 0;
938     const KFile *f = NULL;
939 
940     if (dir == NULL || path == NULL || kart == NULL || isKart == NULL) {
941         return RC(rcKFG, rcFile, rcReading, rcParam, rcNull);
942     }
943 
944     *isKart = false;
945     *kart = NULL;
946 
947     rc = KDirectoryOpenFileRead(dir, &f, "%s", path);
948     if (rc != 0) {
949         return rc;
950     }
951 
952     {
953         Kart *obj = NULL;
954 
955         *isKart = true;
956 
957         obj = calloc(1, sizeof *obj);
958         if (obj == NULL) {
959             return RC(rcKFG, rcData, rcAllocating, rcMemory, rcExhausted);
960         }
961 
962         rc = read_textkart(&obj->mem, f);
963         if (rc == 0) {
964             rc = KartProcessHeader(obj);
965         }
966         if (rc == 0) {
967             KRefcountInit(&obj->refcount, 1, "Kart", "MakeText", "kart");
968             *kart = obj;
969         }
970         else {
971             KartWhack(obj);
972         }
973     }
974 
975     RELEASE(KFile, f);
976     return rc;
977 }
978 #endif
979 
KartMake2(Kart ** kart)980 LIB_EXPORT rc_t CC KartMake2 ( Kart ** kart ) {
981     Kart * obj = NULL;
982 
983     if ( kart == NULL )
984         return RC ( rcKFG, rcFile, rcReading, rcParam, rcNull );
985 
986     obj = calloc ( 1, sizeof * obj );
987     if ( obj == NULL )
988         return RC ( rcKFG, rcData, rcAllocating, rcMemory, rcExhausted );
989 
990     obj -> version = eVersion2;
991 
992     KRefcountInit ( & obj->refcount, 1, "Kart", "KartMake2", "kart" );
993 
994     * kart = obj;
995 
996     return 0;
997 }
998 
KartAddRow(Kart * self,const char * row,size_t size)999 LIB_EXPORT rc_t CC KartAddRow ( Kart * self,
1000     const char * row, size_t size )
1001 {
1002     if ( self == NULL )
1003         return RC ( rcKFG, rcFile, rcUpdating, rcSelf, rcNull );
1004     if ( row == NULL )
1005         return RC ( rcKFG, rcFile, rcUpdating, rcParam, rcNull );
1006 
1007     if ( self -> version < eVersion2 )
1008         return RC ( rcKFG, rcFile, rcUpdating, rcInterface, rcBadVersion );
1009 
1010     {
1011         rc_t rc = 0;
1012 
1013         KartItem * item = NULL;
1014 
1015         const char * p = string_dup ( row, size );
1016         if ( p == NULL )
1017             return RC ( rcKFG, rcFile, rcUpdating, rcMemory, rcExhausted );
1018 
1019         rc = KartItemMake2 ( & item, p, size );
1020         if ( rc == 0 ) {
1021             rc = VectorAppend ( & self -> rows, NULL, item );
1022 
1023             if ( rc != 0 )
1024                 KartItemRelease ( item );
1025         }
1026 
1027         return rc;
1028     }
1029 }
1030 
KartNgcInit(Kart * self,const char * ngcPath,const KDirectory * dir)1031 static rc_t KartNgcInit(Kart * self,
1032     const char *ngcPath, const KDirectory *dir)
1033 {
1034     if (ngcPath == NULL)
1035         return 0;
1036 
1037     else {
1038         const KFile * f = NULL;
1039         rc_t rc = KDirectoryOpenFileRead(dir, &f, "%s", ngcPath);
1040 
1041         if (rc == 0) {
1042             assert(self);
1043             rc = KNgcObjMakeFromFile(&self->ngcObj, f);
1044         }
1045 
1046         RELEASE(KFile, f);
1047 
1048         return rc;
1049     }
1050 }
1051 
KartMakeWithNgc(const KDirectory * dir,const char * path,Kart ** kart,bool * isKart,const char * ngcPath)1052 LIB_EXPORT rc_t CC KartMakeWithNgc(const KDirectory *dir, const char *path,
1053     Kart **kart, bool *isKart, const char *ngcPath)
1054 {
1055     rc_t rc = 0;
1056     const KFile *f = NULL;
1057     char hdr[8] = "";
1058     size_t num_read = 0;
1059 
1060     if (dir == NULL || path == NULL || kart == NULL || isKart == NULL) {
1061         return RC(rcKFG, rcFile, rcReading, rcParam, rcNull);
1062     }
1063 
1064     *isKart = false;
1065     *kart = NULL;
1066 
1067     rc = KDirectoryOpenFileRead(dir, &f, "%s", path);
1068     if (rc != 0) {
1069         return rc;
1070     }
1071 
1072     rc = KFileReadAll(f, 0, hdr, sizeof hdr, &num_read);
1073     if (rc == 0 && num_read == sizeof hdr &&
1074         memcmp(hdr, "ncbikart", sizeof hdr) == 0)
1075     {
1076         KConfig * kfg = NULL;
1077 
1078         Kart *obj = NULL;
1079 
1080         *isKart = true;
1081 
1082         obj = calloc(1, sizeof *obj);
1083         if (obj == NULL) {
1084             return RC(rcKFG, rcData, rcAllocating, rcMemory, rcExhausted);
1085         }
1086 
1087         rc = decode_kart(&obj->mem, f, sizeof hdr);
1088         if (rc == 0) {
1089             rc = KartProcessHeader(obj);
1090         }
1091 
1092         if (rc == 0)
1093             rc = KConfigMake ( & kfg , NULL );
1094         if ( rc == 0 )
1095             rc = KKeyStoreMake ( & obj -> keystore, kfg );
1096         if ( rc == 0 )
1097             rc = KConfigMakeRepositoryMgrRead ( kfg, & obj -> mgr );
1098         RELEASE ( KConfig, kfg );
1099 
1100         if (rc == 0)
1101             rc = KartNgcInit(obj, ngcPath, dir);
1102 
1103         if (rc == 0) {
1104             KRefcountInit(&obj->refcount, 1, "Kart", "Make", "kart");
1105             *kart = obj;
1106         }
1107         else {
1108             KartWhack(obj);
1109         }
1110     }
1111 
1112     RELEASE(KFile, f);
1113 
1114     return rc;
1115 }
1116 
KartMake(const KDirectory * dir,const char * path,Kart ** kart,bool * isKart)1117 LIB_EXPORT rc_t CC KartMake(const KDirectory *dir, const char *path,
1118     Kart **kart, bool *isKart)
1119 {
1120     return KartMakeWithNgc(dir, path, kart, isKart, NULL);
1121 }
1122