1 /*===========================================================================
2 *
3 *                            PUBLIC DOMAIN NOTICE
4 *               National Center for Biotechnology Information
5 *
6 *  This software/database is a "United States Government Work" under the
7 *  terms of the United States Copyright Act.  It was written as part of
8 *  the author's official duties as a United States Government employee and
9 *  thus cannot be copyrighted.  This software/database is freely available
10 *  to the public for use. The National Library of Medicine and the U.S.
11 *  Government have not placed any restriction on its use or reproduction.
12 *
13 *  Although all reasonable efforts have been taken to ensure the accuracy
14 *  and reliability of the software and data, the NLM and the U.S.
15 *  Government do not and cannot warrant the performance or results that
16 *  may be obtained by using this software or data. The NLM and the U.S.
17 *  Government disclaim all warranties, express or implied, including
18 *  warranties of performance, merchantability or fitness for any particular
19 *  purpose.
20 *
21 *  Please cite the author in any work or product based on this material.
22 *
23 * ===========================================================================
24 *
25 */
26 
27 #ifndef _h_sra_sradb_priv_
28 #define _h_sra_sradb_priv_
29 
30 #ifndef _h_klib_container_
31 #include <klib/container.h>
32 #endif
33 
34 #ifndef _h_klib_text
35 #include <klib/text.h>
36 #endif
37 
38 #ifndef _h_klib_vector
39 #include <klib/vector.h>
40 #endif
41 
42 #ifndef _h_sra_sradb_
43 #include <sra/sradb.h>
44 #endif
45 
46 #ifndef _h_sra_srapath_
47 #include <sra/srapath.h>
48 #endif
49 
50 #ifndef _h_sra_path_extern_
51 #include <sra/path-extern.h>
52 #endif
53 
54 #ifndef _h_sra_sch_extern_
55 #include <sra/sch-extern.h>
56 #endif
57 
58 #ifdef __cplusplus
59 extern "C" {
60 #endif
61 
62 
63 /*--------------------------------------------------------------------------
64  * forwards
65  */
66 struct KFile;
67 struct KDirectory;
68 struct KDBManager;
69 struct KTable;
70 struct KLock;
71 struct KConfig;
72 struct VDBManager;
73 struct VTable;
74 struct VSchema;
75 struct SRAPath;
76 struct SRACacheUsage;
77 struct SRACacheMetrics;
78 /*--------------------------------------------------------------------------
79  * SRAMgr
80  *  opaque handle to SRA library
81  */
82 
83 /*  SRAMgrResolve
84  *  Convert accession name into a file system path
85  */
86 
87 SRA_EXTERN rc_t CC SRAMgrResolve( const SRAMgr *self, const char* acc, char* buf, size_t buf_size );
88 
89 /* Flush
90  *  flushes least recently used accessions until the cache size is under the specified threshold
91  */
92 SRA_EXTERN rc_t CC SRAMgrFlush ( struct SRAMgr const *self, const struct SRACacheMetrics* );
93 
94 /* RunBGTasks
95  *  perform single pass of garbage collection tasks and exit.
96  *  also retrieves and processes update messages.
97  */
98 SRA_EXTERN rc_t CC SRAMgrRunBGTasks ( struct SRAMgr const *self );
99 
100 /* GetVDBManager
101  *  returns a new reference to VDBManager used by SRAMgr
102  */
103 SRA_EXTERN rc_t CC SRAMgrGetVDBManagerRead ( const SRAMgr *self, struct VDBManager const **vmgr );
104 SRA_EXTERN rc_t CC SRAMgrGetVDBManagerUpdate ( SRAMgr *self, struct VDBManager **vmgr );
105 
106 /* GetKDBManager
107  *  returns a new reference to KDBManager used indirectly by SRAMgr
108  */
109 SRA_EXTERN rc_t CC SRAMgrGetKDBManagerRead ( const SRAMgr *self, struct KDBManager const **kmgr );
110 SRA_EXTERN rc_t CC SRAMgrGetKDBManagerUpdate ( SRAMgr *self, struct KDBManager **kmgr );
111 
112 /* ModDate
113  *  return a modification timestamp for table
114  */
115 SRA_EXTERN rc_t CC SRAMgrVGetTableModDate ( const SRAMgr *self,
116     KTime_t *mtime, const char *spec, va_list args );
117 
118 SRA_EXTERN rc_t CC SRAMgrGetTableModDate ( const SRAMgr *self,
119     KTime_t *mtime, const char *spec, ... );
120 
121 /* ConfigReload
122  *  update SRAPath object
123  */
124 SRA_EXTERN rc_t CC SRAMgrConfigReload( const SRAMgr *self, struct KDirectory const *wd );
125 
126 /*
127  *  Accession Cache usage stats
128  */
129 SRA_EXTERN rc_t CC SRAMgrGetCacheUsage( const SRAMgr *self, struct SRACacheUsage* stats );
130 
131 /*
132  *  Configure Accession Cache
133  *  soft_threshold, hard_threshold - new threshold values ( -1 : do not change; < -1 invalid )
134  */
135 SRA_EXTERN rc_t CC SRAMgrConfigureCache( const SRAMgr *self,  int32_t soft_threshold, int32_t hard_threshold );
136 
137 /*--------------------------------------------------------------------------
138  * SRATable
139  */
140 
141 /* OpenAltTableRead
142  *  opens a table within a database structure with a specific name
143  */
144 SRA_EXTERN rc_t CC SRAMgrOpenAltTableRead ( const SRAMgr *self,
145     const SRATable **tbl, const char *altname, const char *spec, ... );
146 
147 /* GetVTable
148  *  returns a new reference to underlying VTable
149  */
150 SRA_EXTERN rc_t CC SRATableGetVTableRead ( const SRATable *self, struct VTable const **vtbl );
151 SRA_EXTERN rc_t CC SRATableGetVTableUpdate ( SRATable *self, struct VTable **vtbl );
152 
153 /* GetKTable
154  *  returns a new reference to underlying KTable
155  */
156 SRA_EXTERN rc_t CC SRATableGetKTableRead ( const SRATable *self, struct KTable const **ktbl );
157 SRA_EXTERN rc_t CC SRATableGetKTableUpdate ( SRATable *self, struct KTable **ktbl );
158 
159 
160 /* MakeSingleFileArchive
161  *  makes a single-file-archive file from an SRA table
162  *
163  *  contents are ordered by frequency and necessity of access
164  *
165  *  "lightweight" [ IN ] - when true, include only those components
166  *  required for read and quality operations.
167  *
168  *  "ext" [OUT,NULL] - optional file name extension to use for file
169  */
170 SRA_EXTERN rc_t CC SRATableMakeSingleFileArchive ( const SRATable *self,
171     struct KFile const **sfa, bool lightweight, const char** ext );
172 
173 /* SingleFileArchiveExt
174  *  retrieve archive extension based on object in the spec
175  */
176 SRA_EXTERN rc_t CC SRAMgrSingleFileArchiveExt(const SRAMgr *self,
177     const char* spec, const bool lightweight, const char** ext);
178 /*--------------------------------------------------------------------------
179  * SRAPath
180  */
181 
182 /* FindWithRepLen
183  *  finds location of run within rep-server/volume matrix
184  *  returns length of rep-server portion
185  */
186 SRA_EXTERN rc_t CC SRAPathFindWithRepLen ( struct SRAPath const *self,
187     const char *accession, char *path, size_t path_max, size_t *rep_len );
188 
189 
190 
191 /*--------------------------------------------------------------------------
192  * SRASchema - DEPRECATED
193  */
194 SRA_EXTERN rc_t CC SRASchemaMake ( struct VSchema **schema, struct VDBManager const *mgr );
195 
196 
197 #if 0
198 
199 /*--------------------------------------------------------------------------
200  * SRATableData  - DEPRECATED
201  *  a collection of spots with several data series, minimally including
202  *  base or color calls and their quality ( confidence ) values, and
203  *  optionally signal-related values ( signal, intensity, noise, ... ).
204  */
205 union NucStrstr;
206 
207 typedef struct SRASpotStructure SRASpotStructure;
208 struct SRASpotStructure
209 {
210     /* preformatted query expression
211        for fixed_seq when search is needed */
212     union NucStrstr *q_str;
213 
214     /* read of fixed len if != 0
215        either teminated by fixed_seq or by the end */
216     uint16_t fixed_len;
217 
218     /* SRAReadTypes */
219 	uint8_t read_type;
220 
221     /* colorspace key */
222     char cs_key;
223 
224     char fixed_seq [ 1024 ];
225 
226     /* label for the read */
227     char read_label [ 54 ];
228 
229 };
230 
231 typedef struct SRASpotCoord SRASpotCoord;
232 struct SRASpotCoord
233 {
234     uint32_t x, y, tile;
235 	uint32_t  lane;
236 	spotid_t id;
237 
238     /* prefix part of spotname */
239 	uint32_t platename_len;
240 	char spotname [ 1024 ];
241 
242 };
243 
244 typedef struct SRATableData SRATableData;
245 struct SRATableData
246 {
247 	uint64_t base_count;
248 	uint64_t spot_count;
249 	uint64_t bad_spot_count;
250 	spotid_t max_spotid;
251 
252     /* the spot is always fixed len read */
253 	uint32_t fixed_len;
254 
255     /* number of reads per spot */
256 	uint32_t num_reads;
257 
258     /* read mask containing bio reads */
259 	uint32_t read_mask_bio;
260 
261     /* read description */
262 	SRASpotStructure read_descr [ 32 ];
263     uint16_t read_len [ 32 ];
264 
265     /* platform type and name */
266 	uint8_t platform;
267 	char platform_str [ 31 ];
268 
269 	uint16_t prefix_len;
270 
271     /* spot coordinates */
272 	SRASpotCoord coord;
273 
274 };
275 
276 /* GetTableData
277  *  returns a pointer to internal table data
278  *  or NULL if "self" is invalid
279  *
280  * NB - THIS OBJECT IS NOT REFERENCE COUNTED
281  */
282 SRA_EXTERN const SRATableData *CC SRATableGetTableData ( const SRATable *self );
283 
284 #endif
285 
286 /*--------------------------------------------------------------------------
287  * SRA Accession Cache
288  */
289 struct SRACacheIndex;
290 
291 typedef struct SRACacheMetrics
292 {
293     uint32_t elements; /* open accessions */
294 
295     /* not in use currently: */
296     uint64_t bytes; /* expanded cache bytes, i.e. cursor */
297     uint32_t threads;
298     uint32_t fds;
299 } SRACacheMetrics;
300 
301 #define SRACacheThresholdSoftBytesDefault       ((uint64_t)0)
302 #define SRACacheThresholdSoftElementsDefault    ((uint32_t)1000)
303 #define SRACacheThresholdSoftThreadsDefault     ((uint32_t)0)
304 #define SRACacheThresholdSoftFdsDefault         ((uint32_t)0)
305 
306 #define SRACacheThresholdHardBytesDefault       ((uint64_t)0)
307 #define SRACacheThresholdHardElementsDefault    ((uint32_t)10000)
308 #define SRACacheThresholdHardThreadsDefault     ((uint32_t)0)
309 #define SRACacheThresholdHardFdsDefault         ((uint32_t)0)
310 
311 SRA_EXTERN bool CC SRACacheMetricsLessThan(const SRACacheMetrics* a, const SRACacheMetrics* b);
312 
313 typedef struct SRACacheElement
314 {
315     DLNode dad;
316 
317     SRATable*   object;
318 
319     KTime_t lastAccessed;
320 
321     struct SRACacheIndex* index;
322     uint32_t key;
323 
324     SRACacheMetrics metrics;
325 } SRACacheElement;
326 
327 SRA_EXTERN rc_t CC SRACacheElementMake(SRACacheElement**        self,
328                                        SRATable*                object,
329                                        struct SRACacheIndex*    index,
330                                        uint32_t                 key,
331                                        const SRACacheMetrics*   metrics);
332 SRA_EXTERN rc_t CC SRACacheElementDestroy(SRACacheElement* self);
333 
334 typedef struct SRACacheIndex
335 {
336     BSTNode dad;
337 
338     String* prefix;
339     KVector* body; /* KVector<SRACacheElement*> */
340 } SRACacheIndex;
341 
342 SRA_EXTERN rc_t CC SRACacheIndexMake(SRACacheIndex** self, String* prefix);
343 SRA_EXTERN rc_t CC SRACacheIndexDestroy(SRACacheIndex* self);
344 
345 typedef struct SRACacheUsage {
346     /* config */
347     uint32_t soft_threshold;
348     uint32_t hard_threshold;
349     /* usage stats */
350     uint32_t elements;
351     uint64_t requests;
352     uint64_t hits;
353     uint64_t misses;
354     uint64_t busy;
355 } SRACacheUsage;
356 
357 typedef struct SRACache
358 {
359     BSTree indexes; /* grows as needed */
360 
361     DLList lru; /* DLList<SRACacheElement*>;  head is the oldest */
362 
363     struct KLock* mutex;
364 
365     SRACacheMetrics softThreshold;
366     SRACacheMetrics hardThreshold;
367     SRACacheMetrics current;
368 
369     uint64_t requests;
370     uint64_t hits;
371     uint64_t misses;
372     uint64_t busy;
373 } SRACache;
374 
375 SRA_EXTERN rc_t CC SRACacheInit(SRACache**, struct KConfig*);
376 
377 SRA_EXTERN rc_t CC SRACacheGetSoftThreshold(SRACache* self, SRACacheMetrics* metrics);
378 SRA_EXTERN rc_t CC SRACacheSetSoftThreshold(SRACache* self, const SRACacheMetrics* metrics);
379 
380 SRA_EXTERN rc_t CC SRACacheGetHardThreshold(SRACache* self, SRACacheMetrics* metrics);
381 SRA_EXTERN rc_t CC SRACacheSetHardThreshold(SRACache* self, const SRACacheMetrics* metrics);
382 
383 SRA_EXTERN rc_t CC SRACacheGetUsage(SRACache* self, SRACacheUsage* usage);
384 
385 /* flush tables until usage is lower than specified in self->softThreshold */
386 SRA_EXTERN rc_t CC SRACacheFlush(SRACache* self);
387 
388 /*
389     if found, moves element to the back of the list; return NULL object if not in the cache
390     if found but the refcount is not 1, returns RC( rcSRA, rcData, rcAccessing, rcParam, rcBusy)
391 */
392 SRA_EXTERN rc_t CC SRACacheGetTable(SRACache* self, const char* acc, const SRATable** object);
393 
394 /*
395  * fails if table is already in the cache.
396  * Does not affect usage stats.
397  */
398 SRA_EXTERN rc_t CC SRACacheAddTable(SRACache* self, const char* acc, SRATable*);
399 
400 SRA_EXTERN rc_t CC SRACacheWhack(SRACache* self);
401 
402 #ifdef __cplusplus
403 }
404 #endif
405 
406 #endif /* _h_sra_sradb_priv_ */
407