1 /*=========================================================================== 2 * 3 * PUBLIC DOMAIN NOTICE 4 * National Center for Biotechnology Information 5 * 6 * This software/database is a "United States Government Work" under the 7 * terms of the United States Copyright Act. It was written as part of 8 * the author's official duties as a United States Government employee and 9 * thus cannot be copyrighted. This software/database is freely available 10 * to the public for use. The National Library of Medicine and the U.S. 11 * Government have not placed any restriction on its use or reproduction. 12 * 13 * Although all reasonable efforts have been taken to ensure the accuracy 14 * and reliability of the software and data, the NLM and the U.S. 15 * Government do not and cannot warrant the performance or results that 16 * may be obtained by using this software or data. The NLM and the U.S. 17 * Government disclaim all warranties, express or implied, including 18 * warranties of performance, merchantability or fitness for any particular 19 * purpose. 20 * 21 * Please cite the author in any work or product based on this material. 22 * 23 * =========================================================================== 24 * 25 */ 26 27 #ifndef _h_sra_sradb_priv_ 28 #define _h_sra_sradb_priv_ 29 30 #ifndef _h_klib_container_ 31 #include <klib/container.h> 32 #endif 33 34 #ifndef _h_klib_text 35 #include <klib/text.h> 36 #endif 37 38 #ifndef _h_klib_vector 39 #include <klib/vector.h> 40 #endif 41 42 #ifndef _h_sra_sradb_ 43 #include <sra/sradb.h> 44 #endif 45 46 #ifndef _h_sra_srapath_ 47 #include <sra/srapath.h> 48 #endif 49 50 #ifndef _h_sra_path_extern_ 51 #include <sra/path-extern.h> 52 #endif 53 54 #ifndef _h_sra_sch_extern_ 55 #include <sra/sch-extern.h> 56 #endif 57 58 #ifdef __cplusplus 59 extern "C" { 60 #endif 61 62 63 /*-------------------------------------------------------------------------- 64 * forwards 65 */ 66 struct KFile; 67 struct KDirectory; 68 struct KDBManager; 69 struct KTable; 70 struct KLock; 71 struct KConfig; 72 struct VDBManager; 73 struct VTable; 74 struct VSchema; 75 struct SRAPath; 76 struct SRACacheUsage; 77 struct SRACacheMetrics; 78 /*-------------------------------------------------------------------------- 79 * SRAMgr 80 * opaque handle to SRA library 81 */ 82 83 /* SRAMgrResolve 84 * Convert accession name into a file system path 85 */ 86 87 SRA_EXTERN rc_t CC SRAMgrResolve( const SRAMgr *self, const char* acc, char* buf, size_t buf_size ); 88 89 /* Flush 90 * flushes least recently used accessions until the cache size is under the specified threshold 91 */ 92 SRA_EXTERN rc_t CC SRAMgrFlush ( struct SRAMgr const *self, const struct SRACacheMetrics* ); 93 94 /* RunBGTasks 95 * perform single pass of garbage collection tasks and exit. 96 * also retrieves and processes update messages. 97 */ 98 SRA_EXTERN rc_t CC SRAMgrRunBGTasks ( struct SRAMgr const *self ); 99 100 /* GetVDBManager 101 * returns a new reference to VDBManager used by SRAMgr 102 */ 103 SRA_EXTERN rc_t CC SRAMgrGetVDBManagerRead ( const SRAMgr *self, struct VDBManager const **vmgr ); 104 SRA_EXTERN rc_t CC SRAMgrGetVDBManagerUpdate ( SRAMgr *self, struct VDBManager **vmgr ); 105 106 /* GetKDBManager 107 * returns a new reference to KDBManager used indirectly by SRAMgr 108 */ 109 SRA_EXTERN rc_t CC SRAMgrGetKDBManagerRead ( const SRAMgr *self, struct KDBManager const **kmgr ); 110 SRA_EXTERN rc_t CC SRAMgrGetKDBManagerUpdate ( SRAMgr *self, struct KDBManager **kmgr ); 111 112 /* ModDate 113 * return a modification timestamp for table 114 */ 115 SRA_EXTERN rc_t CC SRAMgrVGetTableModDate ( const SRAMgr *self, 116 KTime_t *mtime, const char *spec, va_list args ); 117 118 SRA_EXTERN rc_t CC SRAMgrGetTableModDate ( const SRAMgr *self, 119 KTime_t *mtime, const char *spec, ... ); 120 121 /* ConfigReload 122 * update SRAPath object 123 */ 124 SRA_EXTERN rc_t CC SRAMgrConfigReload( const SRAMgr *self, struct KDirectory const *wd ); 125 126 /* 127 * Accession Cache usage stats 128 */ 129 SRA_EXTERN rc_t CC SRAMgrGetCacheUsage( const SRAMgr *self, struct SRACacheUsage* stats ); 130 131 /* 132 * Configure Accession Cache 133 * soft_threshold, hard_threshold - new threshold values ( -1 : do not change; < -1 invalid ) 134 */ 135 SRA_EXTERN rc_t CC SRAMgrConfigureCache( const SRAMgr *self, int32_t soft_threshold, int32_t hard_threshold ); 136 137 /*-------------------------------------------------------------------------- 138 * SRATable 139 */ 140 141 /* OpenAltTableRead 142 * opens a table within a database structure with a specific name 143 */ 144 SRA_EXTERN rc_t CC SRAMgrOpenAltTableRead ( const SRAMgr *self, 145 const SRATable **tbl, const char *altname, const char *spec, ... ); 146 147 /* GetVTable 148 * returns a new reference to underlying VTable 149 */ 150 SRA_EXTERN rc_t CC SRATableGetVTableRead ( const SRATable *self, struct VTable const **vtbl ); 151 SRA_EXTERN rc_t CC SRATableGetVTableUpdate ( SRATable *self, struct VTable **vtbl ); 152 153 /* GetKTable 154 * returns a new reference to underlying KTable 155 */ 156 SRA_EXTERN rc_t CC SRATableGetKTableRead ( const SRATable *self, struct KTable const **ktbl ); 157 SRA_EXTERN rc_t CC SRATableGetKTableUpdate ( SRATable *self, struct KTable **ktbl ); 158 159 160 /* MakeSingleFileArchive 161 * makes a single-file-archive file from an SRA table 162 * 163 * contents are ordered by frequency and necessity of access 164 * 165 * "lightweight" [ IN ] - when true, include only those components 166 * required for read and quality operations. 167 * 168 * "ext" [OUT,NULL] - optional file name extension to use for file 169 */ 170 SRA_EXTERN rc_t CC SRATableMakeSingleFileArchive ( const SRATable *self, 171 struct KFile const **sfa, bool lightweight, const char** ext ); 172 173 /* SingleFileArchiveExt 174 * retrieve archive extension based on object in the spec 175 */ 176 SRA_EXTERN rc_t CC SRAMgrSingleFileArchiveExt(const SRAMgr *self, 177 const char* spec, const bool lightweight, const char** ext); 178 /*-------------------------------------------------------------------------- 179 * SRAPath 180 */ 181 182 /* FindWithRepLen 183 * finds location of run within rep-server/volume matrix 184 * returns length of rep-server portion 185 */ 186 SRA_EXTERN rc_t CC SRAPathFindWithRepLen ( struct SRAPath const *self, 187 const char *accession, char *path, size_t path_max, size_t *rep_len ); 188 189 190 191 /*-------------------------------------------------------------------------- 192 * SRASchema - DEPRECATED 193 */ 194 SRA_EXTERN rc_t CC SRASchemaMake ( struct VSchema **schema, struct VDBManager const *mgr ); 195 196 197 #if 0 198 199 /*-------------------------------------------------------------------------- 200 * SRATableData - DEPRECATED 201 * a collection of spots with several data series, minimally including 202 * base or color calls and their quality ( confidence ) values, and 203 * optionally signal-related values ( signal, intensity, noise, ... ). 204 */ 205 union NucStrstr; 206 207 typedef struct SRASpotStructure SRASpotStructure; 208 struct SRASpotStructure 209 { 210 /* preformatted query expression 211 for fixed_seq when search is needed */ 212 union NucStrstr *q_str; 213 214 /* read of fixed len if != 0 215 either teminated by fixed_seq or by the end */ 216 uint16_t fixed_len; 217 218 /* SRAReadTypes */ 219 uint8_t read_type; 220 221 /* colorspace key */ 222 char cs_key; 223 224 char fixed_seq [ 1024 ]; 225 226 /* label for the read */ 227 char read_label [ 54 ]; 228 229 }; 230 231 typedef struct SRASpotCoord SRASpotCoord; 232 struct SRASpotCoord 233 { 234 uint32_t x, y, tile; 235 uint32_t lane; 236 spotid_t id; 237 238 /* prefix part of spotname */ 239 uint32_t platename_len; 240 char spotname [ 1024 ]; 241 242 }; 243 244 typedef struct SRATableData SRATableData; 245 struct SRATableData 246 { 247 uint64_t base_count; 248 uint64_t spot_count; 249 uint64_t bad_spot_count; 250 spotid_t max_spotid; 251 252 /* the spot is always fixed len read */ 253 uint32_t fixed_len; 254 255 /* number of reads per spot */ 256 uint32_t num_reads; 257 258 /* read mask containing bio reads */ 259 uint32_t read_mask_bio; 260 261 /* read description */ 262 SRASpotStructure read_descr [ 32 ]; 263 uint16_t read_len [ 32 ]; 264 265 /* platform type and name */ 266 uint8_t platform; 267 char platform_str [ 31 ]; 268 269 uint16_t prefix_len; 270 271 /* spot coordinates */ 272 SRASpotCoord coord; 273 274 }; 275 276 /* GetTableData 277 * returns a pointer to internal table data 278 * or NULL if "self" is invalid 279 * 280 * NB - THIS OBJECT IS NOT REFERENCE COUNTED 281 */ 282 SRA_EXTERN const SRATableData *CC SRATableGetTableData ( const SRATable *self ); 283 284 #endif 285 286 /*-------------------------------------------------------------------------- 287 * SRA Accession Cache 288 */ 289 struct SRACacheIndex; 290 291 typedef struct SRACacheMetrics 292 { 293 uint32_t elements; /* open accessions */ 294 295 /* not in use currently: */ 296 uint64_t bytes; /* expanded cache bytes, i.e. cursor */ 297 uint32_t threads; 298 uint32_t fds; 299 } SRACacheMetrics; 300 301 #define SRACacheThresholdSoftBytesDefault ((uint64_t)0) 302 #define SRACacheThresholdSoftElementsDefault ((uint32_t)1000) 303 #define SRACacheThresholdSoftThreadsDefault ((uint32_t)0) 304 #define SRACacheThresholdSoftFdsDefault ((uint32_t)0) 305 306 #define SRACacheThresholdHardBytesDefault ((uint64_t)0) 307 #define SRACacheThresholdHardElementsDefault ((uint32_t)10000) 308 #define SRACacheThresholdHardThreadsDefault ((uint32_t)0) 309 #define SRACacheThresholdHardFdsDefault ((uint32_t)0) 310 311 SRA_EXTERN bool CC SRACacheMetricsLessThan(const SRACacheMetrics* a, const SRACacheMetrics* b); 312 313 typedef struct SRACacheElement 314 { 315 DLNode dad; 316 317 SRATable* object; 318 319 KTime_t lastAccessed; 320 321 struct SRACacheIndex* index; 322 uint32_t key; 323 324 SRACacheMetrics metrics; 325 } SRACacheElement; 326 327 SRA_EXTERN rc_t CC SRACacheElementMake(SRACacheElement** self, 328 SRATable* object, 329 struct SRACacheIndex* index, 330 uint32_t key, 331 const SRACacheMetrics* metrics); 332 SRA_EXTERN rc_t CC SRACacheElementDestroy(SRACacheElement* self); 333 334 typedef struct SRACacheIndex 335 { 336 BSTNode dad; 337 338 String* prefix; 339 KVector* body; /* KVector<SRACacheElement*> */ 340 } SRACacheIndex; 341 342 SRA_EXTERN rc_t CC SRACacheIndexMake(SRACacheIndex** self, String* prefix); 343 SRA_EXTERN rc_t CC SRACacheIndexDestroy(SRACacheIndex* self); 344 345 typedef struct SRACacheUsage { 346 /* config */ 347 uint32_t soft_threshold; 348 uint32_t hard_threshold; 349 /* usage stats */ 350 uint32_t elements; 351 uint64_t requests; 352 uint64_t hits; 353 uint64_t misses; 354 uint64_t busy; 355 } SRACacheUsage; 356 357 typedef struct SRACache 358 { 359 BSTree indexes; /* grows as needed */ 360 361 DLList lru; /* DLList<SRACacheElement*>; head is the oldest */ 362 363 struct KLock* mutex; 364 365 SRACacheMetrics softThreshold; 366 SRACacheMetrics hardThreshold; 367 SRACacheMetrics current; 368 369 uint64_t requests; 370 uint64_t hits; 371 uint64_t misses; 372 uint64_t busy; 373 } SRACache; 374 375 SRA_EXTERN rc_t CC SRACacheInit(SRACache**, struct KConfig*); 376 377 SRA_EXTERN rc_t CC SRACacheGetSoftThreshold(SRACache* self, SRACacheMetrics* metrics); 378 SRA_EXTERN rc_t CC SRACacheSetSoftThreshold(SRACache* self, const SRACacheMetrics* metrics); 379 380 SRA_EXTERN rc_t CC SRACacheGetHardThreshold(SRACache* self, SRACacheMetrics* metrics); 381 SRA_EXTERN rc_t CC SRACacheSetHardThreshold(SRACache* self, const SRACacheMetrics* metrics); 382 383 SRA_EXTERN rc_t CC SRACacheGetUsage(SRACache* self, SRACacheUsage* usage); 384 385 /* flush tables until usage is lower than specified in self->softThreshold */ 386 SRA_EXTERN rc_t CC SRACacheFlush(SRACache* self); 387 388 /* 389 if found, moves element to the back of the list; return NULL object if not in the cache 390 if found but the refcount is not 1, returns RC( rcSRA, rcData, rcAccessing, rcParam, rcBusy) 391 */ 392 SRA_EXTERN rc_t CC SRACacheGetTable(SRACache* self, const char* acc, const SRATable** object); 393 394 /* 395 * fails if table is already in the cache. 396 * Does not affect usage stats. 397 */ 398 SRA_EXTERN rc_t CC SRACacheAddTable(SRACache* self, const char* acc, SRATable*); 399 400 SRA_EXTERN rc_t CC SRACacheWhack(SRACache* self); 401 402 #ifdef __cplusplus 403 } 404 #endif 405 406 #endif /* _h_sra_sradb_priv_ */ 407