1 /*  $Id: vdbread.cpp 632481 2021-06-02 11:13:44Z ivanov $
2  * ===========================================================================
3  *
4  *                            PUBLIC DOMAIN NOTICE
5  *               National Center for Biotechnology Information
6  *
7  *  This software/database is a "United States Government Work" under the
8  *  terms of the United States Copyright Act.  It was written as part of
9  *  the author's official duties as a United States Government employee and
10  *  thus cannot be copyrighted.  This software/database is freely available
11  *  to the public for use. The National Library of Medicine and the U.S.
12  *  Government have not placed any restriction on its use or reproduction.
13  *
14  *  Although all reasonable efforts have been taken to ensure the accuracy
15  *  and reliability of the software and data, the NLM and the U.S.
16  *  Government do not and cannot warrant the performance or results that
17  *  may be obtained by using this software or data. The NLM and the U.S.
18  *  Government disclaim all warranties, express or implied, including
19  *  warranties of performance, merchantability or fitness for any particular
20  *  purpose.
21  *
22  *  Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Authors:  Eugene Vasilchenko
27  *
28  * File Description:
29  *   Access to SRA files
30  *
31  */
32 
33 #include <ncbi_pch.hpp>
34 #include <common/ncbi_package_ver.h>
35 #include <common/ncbi_source_ver.h>
36 #include <sra/readers/sra/vdbread.hpp>
37 
38 #include <klib/rc.h>
39 #include <klib/log.h>
40 #include <klib/text.h>
41 #include <klib/sra-release-version.h>
42 #include <kfg/config.h>
43 #include <kdb/manager.h>
44 #include <kdb/kdb-priv.h>
45 #include <kns/manager.h>
46 #include <kns/http.h>
47 #include <kns/tls.h>
48 
49 #include <vfs/manager-priv.h>
50 #include <vfs/manager.h>
51 #include <vfs/path.h>
52 #include <vfs/resolver.h>
53 
54 #include <sra/sradb-priv.h>
55 
56 #include <vdb/vdb-priv.h>
57 #include <vdb/manager.h>
58 #include <vdb/database.h>
59 #include <vdb/schema.h>
60 #include <vdb/table.h>
61 #include <vdb/cursor.h>
62 
63 
64 #include <corelib/ncbimtx.hpp>
65 #include <corelib/ncbifile.hpp>
66 #include <corelib/ncbi_param.hpp>
67 #include <corelib/request_ctx.hpp>
68 #include <sra/readers/ncbi_traces_path.hpp>
69 #include <objects/general/general__.hpp>
70 #include <objects/seq/seq__.hpp>
71 #include <objects/seqset/seqset__.hpp>
72 #include <objects/seqres/seqres__.hpp>
73 #include <sra/error_codes.hpp>
74 
75 #include <cstring>
76 #include <algorithm>
77 
78 BEGIN_NCBI_SCOPE
79 
80 #define NCBI_USE_ERRCODE_X   VDBReader
81 NCBI_DEFINE_ERR_SUBCODE_X(2);
82 
83 BEGIN_SCOPE(objects)
84 
85 class CSeq_entry;
86 
87 
88 NCBI_PARAM_DECL(int, VDB, DIAG_HANDLER);
89 NCBI_PARAM_DEF(int, VDB, DIAG_HANDLER, 1);
90 
91 
s_GetDiagHandler(void)92 static int s_GetDiagHandler(void)
93 {
94     static CSafeStatic<NCBI_PARAM_TYPE(VDB, DIAG_HANDLER)> s_Value;
95     return s_Value->Get();
96 }
97 
98 
99 
100 NCBI_PARAM_DECL(int, VDB, DEBUG);
101 NCBI_PARAM_DEF_EX(int, VDB, DEBUG, 0, eParam_NoThread, VDB_DEBUG);
102 
103 
s_GetDebugLevel(void)104 static int s_GetDebugLevel(void)
105 {
106     static int value = NCBI_PARAM_TYPE(VDB, DEBUG)::GetDefault();
107     return value;
108 }
109 
110 
111 
112 DEFINE_SRA_REF_TRAITS(VDBManager, const);
113 DEFINE_SRA_REF_TRAITS(VDatabase, const);
114 DEFINE_SRA_REF_TRAITS(VTable, const);
115 DEFINE_SRA_REF_TRAITS(VCursor, const);
116 DEFINE_SRA_REF_TRAITS(KIndex, const);
117 DEFINE_SRA_REF_TRAITS(KConfig, );
118 DEFINE_SRA_REF_TRAITS(KDBManager, const);
119 DEFINE_SRA_REF_TRAITS(KNSManager, );
120 DEFINE_SRA_REF_TRAITS(VFSManager, );
121 DEFINE_SRA_REF_TRAITS(VPath, const);
122 DEFINE_SRA_REF_TRAITS(VResolver, );
123 
124 /////////////////////////////////////////////////////////////////////////////
125 // CKConfig
126 
CKConfig(void)127 CKConfig::CKConfig(void)
128 {
129     KConfig* cfg;
130     if ( rc_t rc = KConfigMake(&cfg, 0) ) {
131         *x_InitPtr() = 0;
132         NCBI_THROW2(CSraException, eInitFailed,
133                     "Cannot create KConfig", rc);
134     }
135     *x_InitPtr() = cfg;
136 }
137 
138 
CKConfig(const CVDBMgr & mgr)139 CKConfig::CKConfig(const CVDBMgr& mgr)
140 {
141     *x_InitPtr() = const_cast<KConfig*>(VFSManagerGetConfig(CVFSManager(mgr)));
142     if ( rc_t rc = KConfigAddRef(*this) ) {
143         *x_InitPtr() = 0;
144         NCBI_THROW2(CSraException, eInitFailed,
145                     "Cannot get reference to KConfig", rc);
146     }
147 }
148 
149 
CKConfig(EMake)150 CKConfig::CKConfig(EMake /*make*/)
151 {
152     if ( rc_t rc = KConfigMake(x_InitPtr(), NULL) ) {
153         NCBI_THROW2(CSraException, eInitFailed,
154                     "Cannot create KConfig singleton", rc);
155     }
156 }
157 
158 
Commit() const159 void CKConfig::Commit() const
160 {
161     if ( rc_t rc = KConfigCommit(const_cast<KConfig*>(GetPointer())) ) {
162         NCBI_THROW2(CSraException, eOtherError,
163                     "CKConfig: Cannot commit config changes", rc);
164     }
165 }
166 
167 
168 /////////////////////////////////////////////////////////////////////////////
169 // CVFSManager
170 
CVFSManager(void)171 CVFSManager::CVFSManager(void)
172 {
173     x_InitNew();
174 }
175 
176 
CVFSManager(ECreateNew)177 CVFSManager::CVFSManager(ECreateNew)
178 {
179     x_InitNew();
180 }
181 
182 
x_InitNew(void)183 void CVFSManager::x_InitNew(void)
184 {
185     if ( rc_t rc = VFSManagerMake(x_InitPtr()) ) {
186         *x_InitPtr() = 0;
187         NCBI_THROW2(CSraException, eInitFailed,
188                     "Cannot create VFSManager", rc);
189     }
190 }
191 
192 
CVFSManager(const CVDBMgr & mgr)193 CVFSManager::CVFSManager(const CVDBMgr& mgr)
194 {
195     if ( rc_t rc = KDBManagerGetVFSManager(CKDBManager(mgr), x_InitPtr()) ) {
196         *x_InitPtr() = 0;
197         NCBI_THROW2(CSraException, eInitFailed,
198                     "Cannot get VFSManager", rc);
199     }
200 }
201 
202 
203 /////////////////////////////////////////////////////////////////////////////
204 // CKDBManager
205 
206 
CKDBManager(const CVDBMgr & mgr)207 CKDBManager::CKDBManager(const CVDBMgr& mgr)
208 {
209     if ( rc_t rc = VDBManagerGetKDBManagerRead(mgr, x_InitPtr()) ) {
210         *x_InitPtr() = 0;
211         NCBI_THROW2(CSraException, eInitFailed,
212                     "Cannot get KDBManager", rc);
213     }
214 }
215 
216 
217 /////////////////////////////////////////////////////////////////////////////
218 // CKNSManager
219 
220 
CKNSManager(EMake)221 CKNSManager::CKNSManager(EMake /*make*/)
222 {
223     if ( rc_t rc = KNSManagerMake(x_InitPtr()) ) {
224         *x_InitPtr() = 0;
225         NCBI_THROW2(CSraException, eInitFailed,
226                     "Cannot make KNSManager", rc);
227     }
228 }
229 
230 
CKNSManager(const CVFSManager & mgr)231 CKNSManager::CKNSManager(const CVFSManager& mgr)
232 {
233     if ( rc_t rc = VFSManagerGetKNSMgr(mgr, x_InitPtr()) ) {
234         *x_InitPtr() = 0;
235         NCBI_THROW2(CSraException, eInitFailed,
236                     "Cannot get KNSManager", rc);
237     }
238 }
239 
240 
241 /////////////////////////////////////////////////////////////////////////////
242 // CVPath
243 
CVPath(const CVFSManager & mgr,const string & path,EType type)244 CVPath::CVPath(const CVFSManager& mgr, const string& path, EType type)
245 {
246     x_Init(mgr, path, type);
247 }
248 
249 
CVPath(const string & path,EType type)250 CVPath::CVPath(const string& path, EType type)
251 {
252     x_Init(CVFSManager(CVFSManager::eCreateNew), path, type);
253 }
254 
255 
x_Init(const CVFSManager & mgr,const string & path,EType type)256 void CVPath::x_Init(const CVFSManager& mgr, const string& path, EType type)
257 {
258     VPath* vpath = 0;
259     if ( type == eSys ) {
260         if ( rc_t rc = VFSManagerMakeSysPath(mgr, &vpath, path.c_str()) ) {
261             *x_InitPtr() = 0;
262             NCBI_THROW2_FMT(CSraException, eInitFailed,
263                             "Cannot create sys VPath: "<<path, rc);
264         }
265     }
266     else if ( type == eAcc ) {
267         if ( rc_t rc = VFSManagerMakeAccPath(mgr, &vpath, path.c_str()) ) {
268             *x_InitPtr() = 0;
269             NCBI_THROW2_FMT(CSraException, eInitFailed,
270                             "Cannot create acc VPath: "<<path, rc);
271         }
272     }
273     else {
274         if ( rc_t rc = VFSManagerMakePath(mgr, &vpath, path.c_str()) ) {
275             *x_InitPtr() = 0;
276             NCBI_THROW2_FMT(CSraException, eInitFailed,
277                             "Cannot create VPath: "<<path, rc);
278         }
279     }
280     *x_InitPtr() = vpath;
281 }
282 
283 
IsLocalFile() const284 bool CVPath::IsLocalFile() const
285 {
286     char buffer[32];
287     size_t size;
288     if ( VPathReadScheme(*this, buffer, sizeof(buffer), &size) != 0 ) {
289         return false;
290     }
291     if ( size == 4 && memcmp(buffer, "file", 4) == 0 ) {
292         return true;
293     }
294     if ( size == 9 && memcmp(buffer, "ncbi-file", 9) == 0 ) {
295         return true;
296     }
297     return false;
298 }
299 
300 
ToString(EType type) const301 string CVPath::ToString(EType type) const
302 {
303     const String* str = 0;
304     if (type == eSys && IsLocalFile()) {
305         if (rc_t rc = VPathMakeSysPath(*this, &str)) {
306             NCBI_THROW2(CSraException, eOtherError,
307                 "Cannot get path from VPath", rc);
308         }
309     }
310     else {
311         if (rc_t rc = VPathMakeString(*this, &str)) {
312             NCBI_THROW2(CSraException, eOtherError,
313                 "Cannot get path from VPath", rc);
314         }
315     }
316     string ret(str->addr, str->size);
317     StringWhack(str);
318     return ret;
319 }
320 
321 
322 #ifdef NCBI_OS_MSWIN
323 static inline
s_HasWindowsDriveLetter(const string & s)324 bool s_HasWindowsDriveLetter(const string& s)
325 {
326     // first symbol is letter, and second symbol is colon (':')
327     return s.length() >= 2 && isalpha(s[0]&0xff) && s[1] == ':';
328 }
329 #endif
330 
331 
IsPlainAccession(const string & acc_or_path)332 bool CVPath::IsPlainAccession(const string& acc_or_path)
333 {
334 #ifdef NCBI_OS_MSWIN
335     return !s_HasWindowsDriveLetter(acc_or_path) &&
336         acc_or_path.find_first_of("/\\") == NPOS;
337 #else
338     return acc_or_path.find('/') == NPOS;
339 #endif
340 }
341 
342 
343 #ifdef NCBI_OS_MSWIN
ConvertSysPathToPOSIX(const string & sys_path)344 string CVPath::ConvertSysPathToPOSIX(const string& sys_path)
345 {
346     // Convert Windows path with drive letter
347     // C:\Users\Public -> /C/Users/Public
348     if ( sys_path[0] == 'h' &&
349          (NStr::StartsWith(sys_path, "http://") ||
350           NStr::StartsWith(sys_path, "https://")) ) {
351         return sys_path;
352     }
353     try {
354         string path = CDirEntry::CreateAbsolutePath(sys_path);
355         replace(path.begin(), path.end(), '\\', '/');
356         if (s_HasWindowsDriveLetter(path)) {
357             // move drive letter from first symbol to second (in place of ':')
358             path[1] = toupper(path[0] & 0xff);
359             // add leading slash
360             path[0] = '/';
361         }
362         return path;
363     }
364     catch (exception&) {
365         // CDirEntry::CreateAbsolutePath() can fail on URL for remote access
366         return sys_path;
367     }
368 }
369 
370 
ConvertAccOrSysPathToPOSIX(const string & acc_or_path)371 string CVPath::ConvertAccOrSysPathToPOSIX(const string& acc_or_path)
372 {
373     return IsPlainAccession(acc_or_path) ?
374         acc_or_path :
375         ConvertSysPathToPOSIX(acc_or_path);
376 }
377 #endif
378 
379 
380 /////////////////////////////////////////////////////////////////////////////
381 // CVResolver
382 
CVResolver(const CVFSManager & mgr)383 CVResolver::CVResolver(const CVFSManager& mgr)
384     : m_Mgr(mgr)
385 {
386     if ( rc_t rc = VFSManagerGetResolver(mgr, x_InitPtr()) ) {
387         *x_InitPtr() = 0;
388         NCBI_THROW2(CSraException, eInitFailed,
389                     "Cannot get VResolver", rc);
390     }
391 }
392 
393 
CVResolver(const CVFSManager & mgr,const CKConfig & cfg)394 CVResolver::CVResolver(const CVFSManager& mgr, const CKConfig& cfg)
395     : m_Mgr(mgr)
396 {
397     if ( rc_t rc = VFSManagerMakeResolver(mgr, x_InitPtr(), cfg) ) {
398         *x_InitPtr() = 0;
399         NCBI_THROW2(CSraException, eInitFailed,
400                     "Cannot create VResolver", rc);
401     }
402 }
403 
404 
Resolve(const string & acc_or_path) const405 string CVResolver::Resolve(const string& acc_or_path) const
406 {
407     if ( !CVPath::IsPlainAccession(acc_or_path) ) {
408         // already a path
409         return acc_or_path;
410     }
411     CVPath acc(m_Mgr, acc_or_path, CVPath::eAcc);
412     const VPath* path;
413     rc_t rc = VResolverLocal(*this, acc, &path);
414     if ( rc ) {
415         rc = VResolverRemote(*this, eProtocolNone, acc, &path);
416     }
417     if ( rc ) {
418         if ( CDirEntry(acc_or_path).Exists() ) {
419             // local file
420             return acc_or_path;
421         }
422         NCBI_THROW2_FMT(CSraException, eNotFound,
423                         "Cannot find acc path: "<<acc_or_path, rc);
424     }
425     return CVPath(path).ToString();
426 }
427 
428 
429 /////////////////////////////////////////////////////////////////////////////
430 // CVDBMgr
431 
CVDBMgr(void)432 CVDBMgr::CVDBMgr(void)
433     : m_Resolver(null)
434 {
435     x_Init();
436 }
437 
438 
FindAccPath(const string & acc) const439 string CVDBMgr::FindAccPath(const string& acc) const
440 {
441     if ( !m_Resolver ) {
442         m_Resolver = CVResolver(CVFSManager(*this));
443     }
444     return m_Resolver.Resolve(acc);
445 }
446 
447 
448 //#define GUARD_SDK
449 #ifdef NCBI_COMPILER_MSVC
450 //# define GUARD_SDK_GET
451 #endif
452 
453 #ifdef GUARD_SDK
454 # define DECLARE_SDK_GUARD() CFastMutexGuard guard(sx_SDKMutex)
455 #else
456 # define DECLARE_SDK_GUARD()
457 #endif
458 
459 #ifdef GUARD_SDK_GET
460 # define DECLARE_SDK_GET_GUARD() CFastMutexGuard guard(sx_SDKMutex)
461 #else
462 # define DECLARE_SDK_GET_GUARD()
463 #endif
464 
465 
466 /////////////////////////////////////////////////////////////////////////////
467 // VDB library initialization code
468 // similar code is located in bamread.cpp
469 /////////////////////////////////////////////////////////////////////////////
470 
471 DEFINE_STATIC_FAST_MUTEX(sx_SDKMutex);
472 
473 static char s_VDBVersion[32]; // enough for 255.255.65535-dev4000000000
474 
475 static
s_InitVDBVersion()476 void s_InitVDBVersion()
477 {
478     if ( !s_VDBVersion[0] ) {
479         ostringstream s;
480         {{ // format VDB version string
481             SraReleaseVersion release_version;
482             SraReleaseVersionGet(&release_version);
483             s << (release_version.version>>24) << '.'
484               << ((release_version.version>>16)&0xff) << '.'
485               << (release_version.version&0xffff);
486             if ( release_version.revision != 0 ||
487                  release_version.type != SraReleaseVersion::eSraReleaseVersionTypeFinal ) {
488                 const char* type = "";
489                 switch ( release_version.type ) {
490                 case SraReleaseVersion::eSraReleaseVersionTypeDev:   type = "dev"; break;
491                 case SraReleaseVersion::eSraReleaseVersionTypeAlpha: type = "a"; break;
492                 case SraReleaseVersion::eSraReleaseVersionTypeBeta:  type = "b"; break;
493                 case SraReleaseVersion::eSraReleaseVersionTypeRC:    type = "RC"; break;
494                 default:                                             type = ""; break;
495                 }
496                 s << '-' << type << release_version.revision;
497             }
498         }}
499         string v = s.str();
500         if ( !v.empty() ) {
501             if ( v.size() >= sizeof(s_VDBVersion) ) {
502                 v.resize(sizeof(s_VDBVersion)-1);
503             }
504             copy(v.begin()+1, v.end(), s_VDBVersion+1);
505             s_VDBVersion[0] = v[0];
506         }
507     }
508 }
509 
510 struct SVDBSeverityTag {
511     const char* tag;
512     CNcbiDiag::FManip manip;
513 };
514 static const SVDBSeverityTag kSeverityTags[] = {
515     { "err:", Error },
516     { "int:", Error },
517     { "sys:", Error },
518     { "info:", Info },
519     { "warn:", Warning },
520     { "debug:", Trace },
521     { "fatal:", Fatal },
522 };
s_GetVDBSeverityTag(CTempString token)523 static const SVDBSeverityTag* s_GetVDBSeverityTag(CTempString token)
524 {
525     if ( !token.empty() && token[token.size()-1] == ':' ) {
526         for ( auto& tag : kSeverityTags ) {
527             if ( token == tag.tag ) {
528                 return &tag;
529             }
530         }
531     }
532     return 0;
533 }
534 
535 static
VDBLogWriter(void *,const char * buffer,size_t size,size_t * written)536 rc_t VDBLogWriter(void* /*data*/, const char* buffer, size_t size, size_t* written)
537 {
538     CTempString msg(buffer, size);
539     NStr::TruncateSpacesInPlace(msg);
540     CNcbiDiag::FManip sev_manip = Error;
541 
542     for ( SIZE_TYPE token_pos = 0, token_end; token_pos < msg.size(); token_pos = token_end + 1 ) {
543         token_end = msg.find(' ', token_pos);
544         if ( token_end == NPOS ) {
545             token_end = msg.size();
546         }
547         if ( auto tag = s_GetVDBSeverityTag(CTempString(msg, token_pos, token_end-token_pos)) ) {
548             sev_manip = tag->manip;
549             break;
550         }
551     }
552     if ( sev_manip == Trace ) {
553         _TRACE("VDB "<<s_VDBVersion<<": "<<msg);
554     }
555     else {
556         ERR_POST_X(2, sev_manip<<"VDB "<<s_VDBVersion<<": "<<msg);
557     }
558     *written = size;
559     return 0;
560 }
561 
562 
s_InitProxyConfig()563 static CKConfig s_InitProxyConfig()
564 {
565     CKConfig config(null);
566     if ( CNcbiApplicationGuard app = CNcbiApplication::InstanceGuard() ) {
567         string host = app->GetConfig().GetString("CONN", "HTTP_PROXY_HOST", kEmptyStr);
568         int port = app->GetConfig().GetInt("CONN", "HTTP_PROXY_PORT", 0);
569         if ( !host.empty() && port != 0 ) {
570             config = CKConfig(CKConfig::eMake);
571             string path = host + ':' + NStr::IntToString(port);
572             if ( rc_t rc = KConfigWriteString(config,
573                                               "/http/proxy/path", path.c_str()) ) {
574                 NCBI_THROW2(CSraException, eInitFailed,
575                             "Cannot set KConfig proxy path", rc);
576             }
577             if ( rc_t rc = KConfigWriteBool(config,
578                                             "/http/proxy/enabled", true) ) {
579                 NCBI_THROW2(CSraException, eInitFailed,
580                             "Cannot set KConfig proxy enabled", rc);
581             }
582         }
583     }
584     return config;
585 }
586 
587 
588 static DECLARE_TLS_VAR(const CRequestContext*, s_LastRequestContext);
589 static DECLARE_TLS_VAR(CRequestContext::TVersion, s_LastRequestContextVersion);
590 
s_UpdateVDBRequestContext(void)591 static void s_UpdateVDBRequestContext(void)
592 {
593     CRequestContext& req_ctx = CDiagContext::GetRequestContext();
594     auto req_ctx_version = req_ctx.GetVersion();
595     if ( &req_ctx == s_LastRequestContext && req_ctx_version == s_LastRequestContextVersion ) {
596         return;
597     }
598     _TRACE("CVDBMgr: Updating request context with version: "<<req_ctx_version);
599     s_LastRequestContext = &req_ctx;
600     s_LastRequestContextVersion = req_ctx_version;
601     CKNSManager kns_mgr(CKNSManager::eMake);
602     if ( req_ctx.IsSetSessionID() ) {
603         _TRACE("CVDBMgr: Updating session ID: "<<req_ctx.GetSessionID());
604         KNSManagerSetSessionID(kns_mgr, req_ctx.GetSessionID().c_str());
605     }
606     if ( req_ctx.IsSetClientIP() ) {
607         _TRACE("CVDBMgr: Updating client IP: "<<req_ctx.GetClientIP());
608         KNSManagerSetClientIP(kns_mgr, req_ctx.GetClientIP().c_str());
609     }
610     if ( req_ctx.IsSetHitID() ) {
611         _TRACE("CVDBMgr: Updating hit ID: "<<req_ctx.GetHitID());
612         KNSManagerSetPageHitID(kns_mgr, req_ctx.GetHitID().c_str());
613     }
614 }
615 
616 
s_InitAllKNS(KNSManager * kns_mgr)617 static void s_InitAllKNS(KNSManager* kns_mgr)
618 {
619     CNcbiApplicationGuard app = CNcbiApplication::InstanceGuard();
620     if ( app && app->GetConfig().GetBool("VDB", "ALLOW_ALL_CERTS", false) ) {
621         if ( rc_t rc = KNSManagerSetAllowAllCerts(kns_mgr, true) ) {
622             NCBI_THROW2(CSraException, eInitFailed,
623                         "Cannot enable all HTTPS certificates in KNSManager", rc);
624         }
625     }
626     {{ // set user agent
627         CNcbiOstrstream str;
628         if ( app ) {
629             str << app->GetAppName() << ": " << app->GetVersion().Print() << "; ";
630         }
631 #if NCBI_PACKAGE
632         str << "Package: " << NCBI_PACKAGE_NAME << ' ' <<
633             NCBI_PACKAGE_VERSION << "; ";
634 #endif
635         str << "C++ ";
636 #ifdef NCBI_PRODUCTION_VER
637         str << NCBI_PRODUCTION_VER << "/";
638 #endif
639 #ifdef NCBI_DEVELOPMENT_VER
640         str << NCBI_DEVELOPMENT_VER;
641 #endif
642         string prefix = CNcbiOstrstreamToString(str);
643         KNSManagerSetUserAgent(kns_mgr, "%s; VDB %s",
644                                prefix.c_str(),
645                                s_VDBVersion);
646     }}
647 }
648 
649 
s_InitStaticKNS(KNSManager * kns_mgr)650 static void s_InitStaticKNS(KNSManager* kns_mgr)
651 {
652     s_InitAllKNS(kns_mgr);
653 }
654 
655 
s_InitLocalKNS(KNSManager * kns_mgr)656 static void s_InitLocalKNS(KNSManager* kns_mgr)
657 {
658     s_InitAllKNS(kns_mgr);
659 }
660 
661 
s_VDBInit()662 static void s_VDBInit()
663 {
664     CFastMutexGuard guard(sx_SDKMutex);
665     static bool initialized = false;
666     if ( !initialized ) {
667         s_InitVDBVersion();
668         // redirect VDB log to C++ Toolkit
669         if ( s_GetDiagHandler() ) {
670             KLogInit();
671             KLogLevel ask_level;
672 #ifdef _DEBUG
673             ask_level = klogDebug;
674 #else
675             ask_level = klogInfo;
676 #endif
677             KLogLevelSet(ask_level);
678             KLogHandlerSet(VDBLogWriter, 0);
679             KLogLibHandlerSet(VDBLogWriter, 0);
680             if ( s_GetDebugLevel() >= 2 ) {
681                 const char* msg = "info: VDB initialized";
682                 size_t written;
683                 VDBLogWriter(0, msg, strlen(msg), &written);
684             }
685         }
686         CKConfig config = s_InitProxyConfig();
687         CKNSManager kns_mgr(CKNSManager::eMake);
688         s_InitStaticKNS(kns_mgr);
689         initialized = true;
690     }
691 }
692 
693 /////////////////////////////////////////////////////////////////////////////
694 // end of VDB library initialization code
695 /////////////////////////////////////////////////////////////////////////////
696 
x_Init(void)697 void CVDBMgr::x_Init(void)
698 {
699     s_VDBInit();
700     if ( rc_t rc = VDBManagerMakeRead(x_InitPtr(), 0) ) {
701         *x_InitPtr() = 0;
702         NCBI_THROW2(CSraException, eInitFailed,
703                     "Cannot open VDBManager", rc);
704     }
705     CVFSManager vfs_mgr(*this);
706     VFSManagerLogNamesServiceErrors(vfs_mgr, false);
707     s_InitLocalKNS(CKNSManager(vfs_mgr));
708 }
709 
710 
GetCacheRoot() const711 string CVDBMgr::GetCacheRoot() const
712 {
713     const VPath* ret;
714     if ( rc_t rc = VDBManagerGetCacheRoot(*this, &ret) ) {
715         if ( GetRCObject(rc) == RCObject(rcPath) &&
716              GetRCState(rc) == rcNotFound ) {
717             return kEmptyStr;
718         }
719         NCBI_THROW2(CSraException, eOtherError,
720                     "CVDBMgr: Cannot get cache root", rc);
721     }
722     return CVPath(ret).ToString(CVPath::eSys);
723 }
724 
725 
SetCacheRoot(const string & path)726 void CVDBMgr::SetCacheRoot(const string& path)
727 {
728     CVPath vpath(CVFSManager(*this), path, CVPath::eSys);
729     if ( rc_t rc = VDBManagerSetCacheRoot(*this, vpath) ) {
730         NCBI_THROW2(CSraException, eOtherError,
731                     "CVDBMgr: Cannot set cache root", rc);
732     }
733 }
734 
735 
DeleteCacheOlderThan(Uint4 days)736 void CVDBMgr::DeleteCacheOlderThan(Uint4 days)
737 {
738     if ( rc_t rc = VDBManagerDeleteCacheOlderThan(*this, days) ) {
739         NCBI_THROW2(CSraException, eOtherError,
740                     "CVDBMgr: Cannot delete old cache files", rc);
741     }
742 }
743 
744 
CommitConfig() const745 void CVDBMgr::CommitConfig() const
746 {
747     CKConfig(*this).Commit();
748 }
749 
750 
751 /////////////////////////////////////////////////////////////////////////////
752 // CVDB
753 
CVDB(const CVDBMgr & mgr,const string & acc_or_path)754 CVDB::CVDB(const CVDBMgr& mgr, const string& acc_or_path)
755     : m_Name(acc_or_path)
756 {
757     DECLARE_SDK_GUARD();
758     s_UpdateVDBRequestContext();
759     string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
760     if ( rc_t rc = VDBManagerOpenDBRead(mgr, x_InitPtr(), 0, "%.*s",
761                                         int(path.size()), path.data()) ) {
762         *x_InitPtr() = 0;
763         if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
764               GetRCObject(rc) == RCObject(rcPath) ||
765               GetRCObject(rc) == RCObject(rcFile)) &&
766              GetRCState(rc) == rcNotFound ) {
767             // no SRA accession
768             NCBI_THROW2_FMT(CSraException, eNotFoundDb,
769                             "Cannot open VDB: "<<acc_or_path, rc);
770         }
771         else if ( GetRCObject(rc) == rcName &&
772                   GetRCState(rc) == rcNotFound &&
773                   GetRCContext(rc) == rcResolving ) {
774             // invalid SRA database
775             NCBI_THROW2_FMT(CSraException, eNotFoundDb,
776                             "Cannot open VDB: "<<acc_or_path, rc);
777         }
778         else if ( GetRCObject(rc) == RCObject(rcFile) &&
779                   GetRCState(rc) == rcUnauthorized ) {
780             // invalid SRA database
781             NCBI_THROW2_FMT(CSraException, eProtectedDb,
782                             "Cannot open VDB: "<<acc_or_path, rc);
783         }
784         else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
785                   GetRCState(rc) == rcIncorrect ) {
786             // invalid SRA database
787             NCBI_THROW2_FMT(CSraException, eDataError,
788                             "Cannot open VDB: "<<acc_or_path, rc);
789         }
790         else {
791             // other errors
792             NCBI_THROW2_FMT(CSraException, eOtherError,
793                             "Cannot open VDB: "<<acc_or_path, rc);
794         }
795     }
796 }
797 
798 
PrintFullName(CNcbiOstream & out) const799 CNcbiOstream& CVDB::PrintFullName(CNcbiOstream& out) const
800 {
801     return out << GetName();
802 }
803 
804 
805 /////////////////////////////////////////////////////////////////////////////
806 // CVDBTable
807 
808 static inline
operator <<(CNcbiOstream & out,const CVDBTable & obj)809 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBTable& obj)
810 {
811     return obj.PrintFullName(out);
812 }
813 
814 
CVDBTable(const CVDB & db,const char * table_name,EMissing missing)815 CVDBTable::CVDBTable(const CVDB& db,
816                      const char* table_name,
817                      EMissing missing)
818     : m_Db(db),
819       m_Name(table_name)
820 {
821     DECLARE_SDK_GUARD();
822     s_UpdateVDBRequestContext();
823     if ( rc_t rc = VDatabaseOpenTableRead(db, x_InitPtr(), table_name) ) {
824         *x_InitPtr() = 0;
825         RCState rc_state = GetRCState(rc);
826         int rc_object = GetRCObject(rc);
827         if ( rc_state == rcNotFound &&
828              (rc_object == rcParam ||
829               rc_object == rcPath) ) {
830             // missing table in the DB
831             if ( missing != eMissing_Throw ) {
832                 return;
833             }
834             NCBI_THROW2_FMT(CSraException, eNotFoundTable,
835                             "Cannot open VDB table: "<<*this, rc);
836         }
837         else {
838             // other errors
839             NCBI_THROW2_FMT(CSraException, eOtherError,
840                             "Cannot open VDB table: "<<*this, rc);
841         }
842     }
843 }
844 
845 
CVDBTable(const CVDBMgr & mgr,const string & acc_or_path,EMissing missing)846 CVDBTable::CVDBTable(const CVDBMgr& mgr,
847                      const string& acc_or_path,
848                      EMissing missing)
849     : m_Name(acc_or_path)
850 {
851     *x_InitPtr() = 0;
852     DECLARE_SDK_GUARD();
853     s_UpdateVDBRequestContext();
854     string path = CVPath::ConvertAccOrSysPathToPOSIX(acc_or_path);
855     if ( rc_t rc = VDBManagerOpenTableRead(mgr, x_InitPtr(), 0, "%.*s",
856                                            int(path.size()), path.data()) ) {
857         *x_InitPtr() = 0;
858         if ( (GetRCObject(rc) == RCObject(rcDirectory) ||
859               GetRCObject(rc) == RCObject(rcPath)) &&
860              GetRCState(rc) == rcNotFound ) {
861             // no SRA accession
862             if ( missing != eMissing_Throw ) {
863                 return;
864             }
865             NCBI_THROW2_FMT(CSraException, eNotFoundTable,
866                             "Cannot open SRA table: "<<acc_or_path, rc);
867         }
868         else if ( GetRCObject(rc) == RCObject(rcDatabase) &&
869                   GetRCState(rc) == rcIncorrect ) {
870             // invalid SRA database
871             NCBI_THROW2_FMT(CSraException, eDataError,
872                             "Cannot open SRA table: "<<acc_or_path, rc);
873         }
874         else {
875             // other errors
876             NCBI_THROW2_FMT(CSraException, eOtherError,
877                             "Cannot open SRA table: "<<acc_or_path, rc);
878         }
879     }
880 }
881 
882 
GetFullName(void) const883 string CVDBTable::GetFullName(void) const
884 {
885     string ret;
886     if ( GetDb() ) {
887         ret = GetDb().GetFullName();
888         ret += '.';
889     }
890     ret += GetName();
891     return ret;
892 }
893 
894 
PrintFullName(CNcbiOstream & out) const895 CNcbiOstream& CVDBTable::PrintFullName(CNcbiOstream& out) const
896 {
897     if ( GetDb() ) {
898         GetDb().PrintFullName(out) << '.';
899     }
900     return out << GetName();
901 }
902 
903 
904 /////////////////////////////////////////////////////////////////////////////
905 // CVDBTableIndex
906 
907 static inline
operator <<(CNcbiOstream & out,const CVDBTableIndex & obj)908 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBTableIndex& obj)
909 {
910     return obj.PrintFullName(out);
911 }
912 
913 
CVDBTableIndex(const CVDBTable & table,const char * index_name,EMissing missing)914 CVDBTableIndex::CVDBTableIndex(const CVDBTable& table,
915                                const char* index_name,
916                                EMissing missing)
917     : m_Table(table),
918       m_Name(index_name)
919 {
920     s_UpdateVDBRequestContext();
921     if ( rc_t rc = VTableOpenIndexRead(table, x_InitPtr(), index_name) ) {
922         *x_InitPtr() = 0;
923         if ( GetRCObject(rc) == RCObject(rcIndex) &&
924              GetRCState(rc) == rcNotFound ) {
925             // no such index
926             if ( missing != eMissing_Throw ) {
927                 return;
928             }
929             NCBI_THROW2_FMT(CSraException, eNotFoundIndex,
930                             "Cannot open VDB table index: "<<*this, rc);
931         }
932         else {
933             NCBI_THROW2_FMT(CSraException, eOtherError,
934                             "Cannot open VDB table index: "<<*this, rc);
935         }
936     }
937 }
938 
939 
GetFullName(void) const940 string CVDBTableIndex::GetFullName(void) const
941 {
942     return GetTable().GetFullName()+'.'+GetName();
943 }
944 
945 
PrintFullName(CNcbiOstream & out) const946 CNcbiOstream& CVDBTableIndex::PrintFullName(CNcbiOstream& out) const
947 {
948     return GetTable().PrintFullName(out) << '.' << GetName();
949 }
950 
951 
Find(const string & value) const952 TVDBRowIdRange CVDBTableIndex::Find(const string& value) const
953 {
954     TVDBRowIdRange range;
955     if ( rc_t rc = KIndexFindText(*this, value.c_str(),
956                                   &range.first, &range.second, 0, 0) ) {
957         if ( GetRCObject(rc) == RCObject(rcString) &&
958              GetRCState(rc) == rcNotFound ) {
959             // no such value
960             range.first = range.second = 0;
961         }
962         else {
963             NCBI_THROW2_FMT(CSraException, eOtherError,
964                             "Cannot find value in index: "<<*this<<": "<<value,
965                             rc);
966         }
967     }
968     return range;
969 }
970 
971 
972 /////////////////////////////////////////////////////////////////////////////
973 // CVDBCursor
974 
975 static inline
operator <<(CNcbiOstream & out,const CVDBCursor & obj)976 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBCursor& obj)
977 {
978     return out << obj.GetTable();
979 }
980 
981 
Init(const CVDBTable & table)982 void CVDBCursor::Init(const CVDBTable& table)
983 {
984     s_UpdateVDBRequestContext();
985     if ( *this ) {
986         NCBI_THROW2(CSraException, eInvalidState,
987                     "Cannot init VDB cursor again",
988                     RC(rcApp, rcCursor, rcConstructing, rcSelf, rcOpen));
989     }
990     if ( rc_t rc = VTableCreateCursorRead(table, x_InitPtr()) ) {
991         *x_InitPtr() = 0;
992         NCBI_THROW2(CSraException, eInitFailed,
993                     "Cannot create VDB cursor", rc);
994     }
995     if ( rc_t rc = VCursorPermitPostOpenAdd(*this) ) {
996         NCBI_THROW2(CSraException, eInitFailed,
997                     "Cannot allow VDB cursor post open column add", rc);
998     }
999     if ( rc_t rc = VCursorOpen(*this) ) {
1000         NCBI_THROW2(CSraException, eInitFailed,
1001                     "Cannot open VDB cursor", rc);
1002     }
1003     m_Table = table;
1004 }
1005 
1006 
CloseRow(void)1007 void CVDBCursor::CloseRow(void)
1008 {
1009     if ( !RowIsOpened() ) {
1010         return;
1011     }
1012     if ( rc_t rc = VCursorCloseRow(*this) ) {
1013         NCBI_THROW2(CSraException, eInitFailed,
1014                     "Cannot close VDB cursor row", rc);
1015     }
1016     m_RowOpened = false;
1017 }
1018 
1019 
OpenRowRc(TVDBRowId row_id)1020 rc_t CVDBCursor::OpenRowRc(TVDBRowId row_id)
1021 {
1022     CloseRow();
1023     s_UpdateVDBRequestContext();
1024     if ( rc_t rc = VCursorSetRowId(*this, row_id) ) {
1025         return rc;
1026     }
1027     if ( rc_t rc = VCursorOpenRow(*this) ) {
1028         return rc;
1029     }
1030     m_RowOpened = true;
1031     return 0;
1032 }
1033 
1034 
OpenRow(TVDBRowId row_id)1035 void CVDBCursor::OpenRow(TVDBRowId row_id)
1036 {
1037     if ( rc_t rc = OpenRowRc(row_id) ) {
1038         NCBI_THROW2_FMT(CSraException, eInitFailed,
1039                         "Cannot open VDB cursor row: "<<*this<<": "<<row_id,
1040                         rc);
1041     }
1042 }
1043 
1044 
GetRowIdRange(TVDBColumnIdx column) const1045 TVDBRowIdRange CVDBCursor::GetRowIdRange(TVDBColumnIdx column) const
1046 {
1047     TVDBRowIdRange ret;
1048     if ( rc_t rc = VCursorIdRange(*this, column, &ret.first, &ret.second) ) {
1049         NCBI_THROW2_FMT(CSraException, eInitFailed,
1050                         "Cannot get VDB cursor row range: "<<*this<<": "<<column,
1051                         rc);
1052     }
1053     return ret;
1054 }
1055 
1056 
GetMaxRowId(void) const1057 TVDBRowId CVDBCursor::GetMaxRowId(void) const
1058 {
1059     TVDBRowIdRange range = GetRowIdRange();
1060     return range.first+range.second-1;
1061 }
1062 
1063 
SetParam(const char * name,const CTempString & value) const1064 void CVDBCursor::SetParam(const char* name, const CTempString& value) const
1065 {
1066     s_UpdateVDBRequestContext();
1067     if ( rc_t rc = VCursorParamsSet
1068          ((struct VCursorParams *)GetPointer(),
1069           name, "%.*s", value.size(), value.data()) ) {
1070         NCBI_THROW2_FMT(CSraException, eNotFound,
1071                         "Cannot set VDB cursor param: "<<*this<<": "<<name,
1072                         rc);
1073     }
1074 }
1075 
1076 
GetElementCount(TVDBRowId row,const CVDBColumn & column,uint32_t elem_bits) const1077 uint32_t CVDBCursor::GetElementCount(TVDBRowId row, const CVDBColumn& column,
1078                                      uint32_t elem_bits) const
1079 {
1080     DECLARE_SDK_GET_GUARD();
1081     s_UpdateVDBRequestContext();
1082     uint32_t read_count, remaining_count;
1083     if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1084                                          elem_bits, 0, 0, 0, 0,
1085                                          &read_count, &remaining_count) ) {
1086         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1087                         "Cannot read VDB value array size: "<<*this<<column<<
1088                         '['<<row<<']', rc);
1089     }
1090     return remaining_count;
1091 }
1092 
1093 
ReadElements(TVDBRowId row,const CVDBColumn & column,uint32_t elem_bits,uint32_t start,uint32_t count,void * buffer) const1094 void CVDBCursor::ReadElements(TVDBRowId row, const CVDBColumn& column,
1095                               uint32_t elem_bits,
1096                               uint32_t start, uint32_t count,
1097                               void* buffer) const
1098 {
1099     DECLARE_SDK_GET_GUARD();
1100     s_UpdateVDBRequestContext();
1101     uint32_t read_count, remaining_count;
1102     if ( rc_t rc = VCursorReadBitsDirect(*this, row, column.GetIndex(),
1103                                          elem_bits, start, buffer, 0, count,
1104                                          &read_count, &remaining_count) ) {
1105         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1106                         "Cannot read VDB value array: "<<*this<<column<<
1107                         '['<<row<<"]["<<start<<".."<<(start+count-1)<<']', rc);
1108     }
1109     if ( read_count != count ) {
1110         NCBI_THROW_FMT(CSraException, eNotFoundValue,
1111                        "Cannot read VDB value array: "<<*this<<column<<
1112                        '['<<row<<"]["<<start<<".."<<(start+count-1)<<
1113                        "] only "<<read_count<<" elements are read");
1114     }
1115 }
1116 
1117 
1118 /////////////////////////////////////////////////////////////////////////////
1119 // CVDBObjectCache
1120 
1121 static const size_t kCacheSize = 7;
1122 
1123 
CVDBObjectCacheBase(void)1124 CVDBObjectCacheBase::CVDBObjectCacheBase(void)
1125 {
1126     m_Objects.reserve(kCacheSize);
1127 }
1128 
1129 
~CVDBObjectCacheBase(void)1130 CVDBObjectCacheBase::~CVDBObjectCacheBase(void)
1131 {
1132 }
1133 
1134 
1135 DEFINE_STATIC_FAST_MUTEX(sm_CacheMutex);
1136 
1137 
Clear(void)1138 void CVDBObjectCacheBase::Clear(void)
1139 {
1140     CFastMutexGuard guard(sm_CacheMutex);
1141     m_Objects.clear();
1142 }
1143 
1144 
Get(TVDBRowId row)1145 CObject* CVDBObjectCacheBase::Get(TVDBRowId row)
1146 {
1147     CFastMutexGuard guard(sm_CacheMutex);
1148     if ( m_Objects.empty() ) {
1149         return 0;
1150     }
1151     TObjects::iterator best_it;
1152     TVDBRowId best_d = numeric_limits<TVDBRowId>::max();
1153     NON_CONST_ITERATE ( TObjects, it, m_Objects ) {
1154         TVDBRowId slot_row = it->first;
1155         if ( slot_row >= row ) {
1156             TVDBRowId d = slot_row - row;
1157             if ( d <= best_d ) {
1158                 best_d = d;
1159                 best_it = it;
1160             }
1161         }
1162         else {
1163             TVDBRowId d = row - slot_row;
1164             if ( d < best_d ) {
1165                 best_d = d;
1166                 best_it = it;
1167             }
1168         }
1169     }
1170     CObject* obj = best_it->second.Release();
1171     *best_it = m_Objects.back();
1172     m_Objects.pop_back();
1173     _ASSERT(!obj->Referenced());
1174     return obj;
1175 }
1176 
1177 
Put(CObject * obj,TVDBRowId row)1178 void CVDBObjectCacheBase::Put(CObject* obj, TVDBRowId row)
1179 {
1180     if ( obj->Referenced() ) {
1181         return;
1182     }
1183     //row = 0;
1184     CFastMutexGuard guard(sm_CacheMutex);
1185     if ( m_Objects.size() < kCacheSize ) {
1186         m_Objects.push_back(TSlot());
1187         m_Objects.back().first = row;
1188         m_Objects.back().second = obj;
1189     }
1190     else {
1191         CRef<CObject> ref(obj); // delete the object
1192     }
1193 }
1194 
1195 
1196 /////////////////////////////////////////////////////////////////////////////
1197 // CVDBColumn
1198 
1199 static inline
operator <<(CNcbiOstream & out,const CVDBColumn & column)1200 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBColumn& column)
1201 {
1202     return out << '.' << column.GetName();
1203 }
1204 
1205 
Init(const CVDBCursor & cursor,size_t element_bit_size,const char * name,const char * backup_name,EMissing missing)1206 void CVDBColumn::Init(const CVDBCursor& cursor,
1207                       size_t element_bit_size,
1208                       const char* name,
1209                       const char* backup_name,
1210                       EMissing missing)
1211 {
1212     DECLARE_SDK_GUARD();
1213     s_UpdateVDBRequestContext();
1214     m_Name = name;
1215     if ( rc_t rc = VCursorAddColumn(cursor, &m_Index, name) ) {
1216         if ( backup_name &&
1217              (rc = VCursorAddColumn(cursor, &m_Index, backup_name)) == 0 ) {
1218             m_Name = backup_name;
1219         }
1220         else {
1221             m_Index = kInvalidIndex;
1222             if ( missing == eMissing_Throw ) {
1223                 NCBI_THROW2_FMT(CSraException, eNotFoundColumn,
1224                                 "Cannot get VDB column: "<<cursor<<*this,rc);
1225             }
1226             else {
1227                 return;
1228             }
1229         }
1230     }
1231     if ( element_bit_size ) {
1232         VTypedesc type;
1233         if ( rc_t rc = VCursorDatatype(cursor, m_Index, 0, &type) ) {
1234             NCBI_THROW2_FMT(CSraException, eInvalidState,
1235                             "Cannot get VDB column type: "<<cursor<<*this,rc);
1236         }
1237         size_t size = type.intrinsic_bits*type.intrinsic_dim;
1238         if ( size != element_bit_size ) {
1239             ERR_POST_X(1, "Wrong VDB column size "<<cursor<<*this<<
1240                        " expected "<<element_bit_size<<" bits != "<<
1241                        type.intrinsic_dim<<"*"<<type.intrinsic_bits<<" bits");
1242             NCBI_THROW2_FMT(CSraException, eInvalidState,
1243                             "Wrong VDB column size: "<<cursor<<*this<<": "<<size,
1244                             RC(rcApp, rcColumn, rcConstructing, rcSelf, rcIncorrect));
1245         }
1246     }
1247 }
1248 
1249 
ResetIfAlwaysEmpty(const CVDBCursor & cursor)1250 void CVDBColumn::ResetIfAlwaysEmpty(const CVDBCursor& cursor)
1251 {
1252     if ( !*this ) {
1253         return;
1254     }
1255     bool static_value;
1256     if ( VCursorIsStaticColumn(cursor, GetIndex(), &static_value) ) {
1257         // cannot get 'static' value -> do nothing
1258         return;
1259     }
1260     try {
1261         if ( static_value && CVDBValue(cursor, 1, *this).empty() ) {
1262             // value is always empty
1263             Reset();
1264         }
1265     }
1266     catch ( CException& /*ignored*/ ) {
1267     }
1268 }
1269 
1270 
1271 /////////////////////////////////////////////////////////////////////////////
1272 // CVDBValue
1273 
PrintFullName(CNcbiOstream & out) const1274 CNcbiOstream& CVDBValue::SSaveRef::PrintFullName(CNcbiOstream& out) const
1275 {
1276     if ( m_Table ) {
1277         m_Table->PrintFullName(out);
1278     }
1279     if ( m_Row ) {
1280         out << '[' << m_Row << ']';
1281     }
1282     if ( m_ColumnName ) {
1283         out << '.' << m_ColumnName;
1284     }
1285     return out;
1286 }
1287 
1288 
1289 static inline
operator <<(CNcbiOstream & out,const CVDBValue & obj)1290 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBValue& obj)
1291 {
1292     return obj.PrintFullName(out);
1293 }
1294 
1295 
x_Get(const CVDBCursor & cursor,const CVDBColumn & column)1296 void CVDBValue::x_Get(const CVDBCursor& cursor, const CVDBColumn& column)
1297 {
1298     DECLARE_SDK_GET_GUARD();
1299     s_UpdateVDBRequestContext();
1300     uint32_t bit_offset, bit_length;
1301     if ( rc_t rc = VCursorCellData(cursor, column.GetIndex(),
1302                                    &bit_length, &m_Data, &bit_offset,
1303                                    &m_ElemCount) ) {
1304         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1305                         "Cannot read VDB value: "<<cursor<<column, rc);
1306     }
1307     if ( bit_offset ) {
1308         NCBI_THROW2_FMT(CSraException, eInvalidState,
1309                         "Cannot read VDB value with non-zero bit offset: "
1310                         <<cursor<<column<<": "<<bit_offset,
1311                         RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1312     }
1313     m_Ref.Set(cursor, 0, column);
1314 }
1315 
1316 
x_Get(const CVDBCursor & cursor,TVDBRowId row,const CVDBColumn & column,EMissing missing)1317 void CVDBValue::x_Get(const CVDBCursor& cursor,
1318                       TVDBRowId row,
1319                       const CVDBColumn& column,
1320                       EMissing missing)
1321 {
1322     DECLARE_SDK_GET_GUARD();
1323     s_UpdateVDBRequestContext();
1324     uint32_t bit_offset, bit_length;
1325     if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1326                                          &bit_length, &m_Data, &bit_offset,
1327                                          &m_ElemCount) ) {
1328         if ( missing != eMissing_Throw ) {
1329             m_Data = 0;
1330             m_ElemCount = 0;
1331             return;
1332         }
1333         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1334                         "Cannot read VDB value: "<<cursor<<column<<'['<<row<<']', rc);
1335     }
1336     if ( bit_offset ) {
1337         NCBI_THROW2_FMT(CSraException, eInvalidState,
1338                         "Cannot read VDB value with non-zero bit offset: "<<
1339                         cursor<<column<<'['<<row<<"]: "<<bit_offset,
1340                         RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1341     }
1342     if ( s_GetDebugLevel() >= 9 ) {
1343         CNcbiOstrstream s;
1344         if ( bit_length == 8 ) {
1345             s << '"' << NStr::PrintableString(CTempString((const char*)m_Data, m_ElemCount)) << '"';
1346         }
1347         else if ( bit_length == 16 ) {
1348             for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1349                 if ( i ) {
1350                     s << ", ";
1351                 }
1352                 s << ((const int16_t*)m_Data)[i];
1353             }
1354         }
1355         else if ( bit_length == 32 ) {
1356             for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1357                 if ( i ) {
1358                     s << ", ";
1359                 }
1360                 s << ((const int32_t*)m_Data)[i];
1361             }
1362         }
1363         else if ( bit_length == 64 ) {
1364             for ( uint32_t i = 0; i < m_ElemCount; ++i ) {
1365                 if ( i ) {
1366                     s << ", ";
1367                 }
1368                 s << ((const int64_t*)m_Data)[i];
1369             }
1370         }
1371         else {
1372             s << "*** bad bit_length="<<bit_length;
1373         }
1374         LOG_POST(Info<<"VDB "<<cursor<<column<<'['<<row<<"]: "<<CNcbiOstrstreamToString(s));
1375     }
1376     m_Ref.Set(cursor, row, column);
1377 }
1378 
1379 
x_ReportIndexOutOfBounds(size_t index) const1380 void CVDBValue::x_ReportIndexOutOfBounds(size_t index) const
1381 {
1382     if ( index >= size() ) {
1383         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1384                         "Invalid index for VDB value array: "<<
1385                         *this<<'['<<index<<']',
1386                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1387     }
1388 }
1389 
1390 
x_ReportNotOneValue(void) const1391 void CVDBValue::x_ReportNotOneValue(void) const
1392 {
1393     if ( size() != 1 ) {
1394         NCBI_THROW2_FMT(CSraException, eDataError,
1395                         "VDB value array doen't have single value: "<<
1396                         *this<<'['<<size()<<']',
1397                         RC(rcApp, rcData, rcRetrieving, rcSize, rcIncorrect));
1398     }
1399 }
1400 
1401 
x_CheckRange(size_t pos,size_t len) const1402 void CVDBValue::x_CheckRange(size_t pos, size_t len) const
1403 {
1404     if ( pos > size() ) {
1405         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1406                         "Invalid index for VDB value array: "<<
1407                         *this<<'['<<pos<<']',
1408                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1409     }
1410     if ( pos+len < pos ) {
1411         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1412                         "Invalid length for VDB value sub-array: "<<
1413                         *this<<'['<<pos<<','<<len<<']',
1414                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1415     }
1416     if ( pos+len > size() ) {
1417         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1418                         "Invalid end of VDB value sub-array: "<<
1419                         *this<<'['<<pos<<','<<len<<']',
1420                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1421     }
1422 }
1423 
1424 
1425 static inline
operator <<(CNcbiOstream & out,const CVDBValueFor4Bits & obj)1426 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBValueFor4Bits& obj)
1427 {
1428     return obj.PrintFullName(out);
1429 }
1430 
1431 
x_Get(const CVDBCursor & cursor,TVDBRowId row,const CVDBColumn & column)1432 void CVDBValueFor4Bits::x_Get(const CVDBCursor& cursor,
1433                               TVDBRowId row,
1434                               const CVDBColumn& column)
1435 {
1436     DECLARE_SDK_GET_GUARD();
1437     s_UpdateVDBRequestContext();
1438     uint32_t bit_offset, bit_length, elem_count;
1439     const void* data;
1440     if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1441                                          &bit_length, &data, &bit_offset,
1442                                          &elem_count) ) {
1443         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1444                         "Cannot read VDB 4-bits value array: "<<
1445                         cursor<<column<<'['<<row<<']', rc);
1446     }
1447     if ( bit_offset >= 8 || (bit_offset&3) ) {
1448         NCBI_THROW2_FMT(CSraException, eInvalidState,
1449                         "Cannot read VDB 4-bits value array with odd bit offset"<<
1450                         cursor<<column<<'['<<row<<"]: "<<bit_offset,
1451                         RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1452     }
1453     m_RawData = static_cast<const char*>(data);
1454     m_ElemOffset = bit_offset >> 2;
1455     m_ElemCount = elem_count;
1456     m_Ref.Set(cursor, row, column);
1457 }
1458 
1459 
x_ReportIndexOutOfBounds(size_t index) const1460 void CVDBValueFor4Bits::x_ReportIndexOutOfBounds(size_t index) const
1461 {
1462     if ( index >= size() ) {
1463         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1464                         "Invalid index for VDB 4-bits value array: "<<
1465                         *this<<'['<<index<<']',
1466                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1467     }
1468 }
1469 
1470 
1471 inline
x_CheckRange(size_t pos,size_t len) const1472 void CVDBValueFor4Bits::x_CheckRange(size_t pos, size_t len) const
1473 {
1474     if ( pos > size() ) {
1475         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1476                         "Invalid index for VDB 4-bits value array: "<<
1477                         *this<<'['<<pos<<']',
1478                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1479     }
1480     if ( pos+len < pos ) {
1481         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1482                         "Invalid length for VDB 4-bits value sub-array: "<<
1483                         *this<<'['<<pos<<','<<len<<']',
1484                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1485     }
1486     if ( pos+len > size() ) {
1487         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1488                         "Invalid end of VDB 4-bits value sub-array: "<<
1489                         *this<<'['<<pos<<','<<len<<']',
1490                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1491     }
1492 }
1493 
1494 
substr(size_t pos,size_t len) const1495 CVDBValueFor4Bits CVDBValueFor4Bits::substr(size_t pos, size_t len) const
1496 {
1497     x_CheckRange(pos, len);
1498     size_t offset = m_ElemOffset + pos;
1499     const char* raw_data = m_RawData + offset/2;
1500     offset %= 2;
1501     // limits are checked above
1502     return CVDBValueFor4Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
1503 }
1504 
1505 
1506 static inline
operator <<(CNcbiOstream & out,const CVDBValueFor2Bits & obj)1507 CNcbiOstream& operator<<(CNcbiOstream& out, const CVDBValueFor2Bits& obj)
1508 {
1509     return obj.PrintFullName(out);
1510 }
1511 
1512 
x_Get(const CVDBCursor & cursor,TVDBRowId row,const CVDBColumn & column)1513 void CVDBValueFor2Bits::x_Get(const CVDBCursor& cursor,
1514                               TVDBRowId row,
1515                               const CVDBColumn& column)
1516 {
1517     DECLARE_SDK_GET_GUARD();
1518     s_UpdateVDBRequestContext();
1519     uint32_t bit_offset, bit_length, elem_count;
1520     const void* data;
1521     if ( rc_t rc = VCursorCellDataDirect(cursor, row, column.GetIndex(),
1522                                          &bit_length, &data, &bit_offset,
1523                                          &elem_count) ) {
1524         NCBI_THROW2_FMT(CSraException, eNotFoundValue,
1525                         "Cannot read VDB 2-bits value array: "<<
1526                         cursor<<column<<'['<<row<<']', rc);
1527     }
1528     if ( bit_offset >= 8 || (bit_offset&1) ) {
1529         NCBI_THROW2_FMT(CSraException, eInvalidState,
1530                         "Cannot read VDB 2-bits value array with odd bit offset"<<
1531                         cursor<<column<<'['<<row<<"]: "<<bit_offset,
1532                         RC(rcApp, rcColumn, rcDecoding, rcOffset, rcUnsupported));
1533     }
1534     m_RawData = static_cast<const char*>(data);
1535     m_ElemOffset = bit_offset >> 1;
1536     m_ElemCount = elem_count;
1537     m_Ref.Set(cursor, row, column);
1538 }
1539 
1540 
x_ReportIndexOutOfBounds(size_t index) const1541 void CVDBValueFor2Bits::x_ReportIndexOutOfBounds(size_t index) const
1542 {
1543     if ( index >= size() ) {
1544         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1545                         "Invalid index for VDB 2-bits value array: "<<
1546                         *this<<'['<<index<<']',
1547                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1548     }
1549 }
1550 
1551 
1552 inline
x_CheckRange(size_t pos,size_t len) const1553 void CVDBValueFor2Bits::x_CheckRange(size_t pos, size_t len) const
1554 {
1555     if ( pos > size() ) {
1556         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1557                         "Invalid index for VDB 2-bits value array: "<<
1558                         *this<<'['<<pos<<']',
1559                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1560     }
1561     if ( pos+len < pos ) {
1562         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1563                         "Invalid length for VDB 2-bits value sub-array: "<<
1564                         *this<<'['<<pos<<','<<len<<']',
1565                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1566     }
1567     if ( pos+len > size() ) {
1568         NCBI_THROW2_FMT(CSraException, eInvalidIndex,
1569                         "Invalid end of VDB 2-bits value sub-array: "<<
1570                         *this<<'['<<pos<<','<<len<<']',
1571                         RC(rcApp, rcData, rcRetrieving, rcOffset, rcTooBig));
1572     }
1573 }
1574 
1575 
substr(size_t pos,size_t len) const1576 CVDBValueFor2Bits CVDBValueFor2Bits::substr(size_t pos, size_t len) const
1577 {
1578     x_CheckRange(pos, len);
1579     size_t offset = m_ElemOffset + pos;
1580     const char* raw_data = m_RawData + offset/4;
1581     offset %= 4;
1582     // limits are checked above
1583     return CVDBValueFor2Bits(m_Ref, raw_data, uint32_t(offset), uint32_t(len));
1584 }
1585 
1586 
1587 END_SCOPE(objects)
1588 END_NCBI_SCOPE
1589