1 /* $Id: netstorageobjectloc.cpp 617407 2020-09-30 19:12:51Z sadyrovr $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Author: Dmitry Kazimirov
27 *
28 * File Description:
29 * Implementation of the unified network blob storage API.
30 *
31 */
32
33 #include <ncbi_pch.hpp>
34
35 #include <connect/services/netstorage.hpp>
36 #include <connect/services/error_codes.hpp>
37
38 #include <connect/ncbi_socket.h>
39
40 #include <corelib/ncbi_base64.h>
41 #include <corelib/ncbiargs.hpp>
42
43 #include <time.h>
44 #include <string.h>
45
46
47 #define NCBI_USE_ERRCODE_X NetStorage_Common
48
49 #define FILETRACK_STORAGE_CODE "FT"
50 #define NETCACHE_STORAGE_CODE "NC"
51
52 #define STORAGE_INFO_CUE 0
53
54 BEGIN_NCBI_SCOPE
55
CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,TNetStorageAttrFlags flags,const string & app_domain,Uint8 random_number,EFileTrackSite ft_site)56 CNetStorageObjectLoc::CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,
57 TNetStorageAttrFlags flags,
58 const string& app_domain,
59 Uint8 random_number,
60 EFileTrackSite ft_site) :
61 m_CompoundIDPool(cid_pool),
62 m_LocatorFlags(x_StorageFlagsToLocatorFlags(flags, ft_site)),
63 m_AppDomain(app_domain),
64 m_Timestamp(time(NULL)),
65 m_Random(random_number),
66 m_ShortUniqueKey(MakeShortUniqueKey()),
67 m_UniqueKey(MakeUniqueKey()),
68 m_Dirty(true)
69 {
70
71 }
72
CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,TNetStorageAttrFlags flags,const string & app_domain,const string & unique_key,EFileTrackSite ft_site)73 CNetStorageObjectLoc::CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,
74 TNetStorageAttrFlags flags,
75 const string& app_domain,
76 const string& unique_key,
77 EFileTrackSite ft_site) :
78 m_CompoundIDPool(cid_pool),
79 m_LocatorFlags(x_StorageFlagsToLocatorFlags(flags, ft_site) | fLF_HasUserKey),
80 m_AppDomain(app_domain),
81 m_ShortUniqueKey(unique_key),
82 m_UniqueKey(MakeUniqueKey()),
83 m_Dirty(true)
84 {
85 }
86
87 #define INVALID_LOC_ERROR_MSG "Invalid NetStorage object locator"
88
89 #define THROW_INVALID_LOC_ERROR(cid, msg) \
90 NCBI_THROW_FMT(CNetStorageException, eInvalidArg, \
91 msg " '" << cid.ToString() << '\'')
92
93 #define VERIFY_FIELD_EXISTS(field) \
94 if (!(field)) { \
95 THROW_INVALID_LOC_ERROR(cid, INVALID_LOC_ERROR_MSG); \
96 }
97
s_LocationCodeToLocation(const string & location)98 ENetStorageObjectLocation s_LocationCodeToLocation(const string& location)
99 {
100 if (location.length() == 2) {
101 if (location.data()[0] == FILETRACK_STORAGE_CODE[0] &&
102 location.data()[1] == FILETRACK_STORAGE_CODE[1])
103 return eNFL_FileTrack;
104 if (location.data()[0] == NETCACHE_STORAGE_CODE[0] &&
105 location.data()[1] == NETCACHE_STORAGE_CODE[1])
106 return eNFL_NetCache;
107 }
108 return eNFL_Unknown;
109 }
110
CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,const string & object_loc)111 CNetStorageObjectLoc::CNetStorageObjectLoc(CCompoundIDPool::TInstance cid_pool,
112 const string& object_loc) :
113 m_CompoundIDPool(cid_pool),
114 m_Dirty(false),
115 m_Locator(object_loc)
116 {
117 auto cid = m_CompoundIDPool.FromString(object_loc);
118 Parse(cid, false);
119 }
120
SetServiceName(const string & service_name)121 void CNetStorageObjectLoc::SetServiceName(const string& service_name)
122 {
123 if (service_name.empty() ||
124 strchr(service_name.c_str(), ':') != NULL)
125 ClearLocatorFlags(fLF_NetStorageService);
126 else {
127 m_ServiceName = service_name;
128 SetLocatorFlags(fLF_NetStorageService);
129 }
130 m_Dirty = true;
131 }
132
GetServiceName(CCompoundID cid)133 string CNetStorageObjectLoc::GetServiceName(CCompoundID cid)
134 {
135 CNetStorageObjectLoc loc;
136 loc.Parse(cid, true);
137 return loc.m_ServiceName;
138 }
139
Parse(CCompoundID cid,bool service_name_only)140 void CNetStorageObjectLoc::Parse(CCompoundID cid, bool service_name_only)
141 {
142 // Check the ID class.
143 switch (cid.GetClass()) {
144 case eCIC_NetStorageObjectLocV1:
145 THROW_INVALID_LOC_ERROR(cid,
146 "Unsupported NetStorage object locator version");
147 case eCIC_NetStorageObjectLoc:
148 break;
149 default:
150 THROW_INVALID_LOC_ERROR(cid, INVALID_LOC_ERROR_MSG);
151 }
152
153 // Get locator flags.
154 CCompoundIDField field = cid.GetFirst(eCIT_Flags);
155 VERIFY_FIELD_EXISTS(field);
156 m_LocatorFlags = (TLocatorFlags) field.GetFlags();
157
158 // Restore NetStorage service name.
159 if (m_LocatorFlags & fLF_NetStorageService) {
160 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
161 m_ServiceName = field.GetServiceName();
162 }
163
164 if (service_name_only) return;
165
166 // Restore object ID.
167 if (m_LocatorFlags & fLF_HasObjectID) {
168 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
169 m_ObjectID = field.GetID();
170 }
171
172 // Get the domain name.
173 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
174 m_AppDomain = field.GetString();
175
176 m_SubKey.clear();
177 m_Version = 0;
178
179 // Restore object identification.
180 if (m_LocatorFlags & fLF_HasUserKey) {
181 // Get the unique object key.
182 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
183 m_ShortUniqueKey = field.GetString();
184
185 if (m_LocatorFlags & fLF_HasSubKey) {
186 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
187 m_SubKey = field.GetString();
188
189 if (m_LocatorFlags & fLF_HasVersion) {
190 // null is stored as 0 (0 is not stored as it's the default)
191 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
192 auto version = static_cast<int>(field.GetInteger());
193 m_Version = version ? TVersion(version) : TVersion(null);
194 }
195 }
196 } else {
197 // Get object creation timestamp.
198 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
199 m_Timestamp = field.GetTimestamp();
200 // Get the random ID.
201 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
202 m_Random = (Uint8) field.GetRandom() << (sizeof(Uint4) * 8);
203 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
204 m_Random |= field.GetRandom();
205 m_ShortUniqueKey = MakeShortUniqueKey();
206 }
207
208 m_UniqueKey = MakeUniqueKey();
209
210 // Not used, though has to be read to be backward-compatible
211 if (m_LocatorFlags & fLF_Cacheable) {
212 field = field.GetNextNeighbor();
213 }
214
215 // Find storage info (optional).
216 for (field = cid.GetFirst(eCIT_Cue); field; field = field.GetNextHomogeneous()) {
217 if (field.GetCue() == STORAGE_INFO_CUE) {
218 // Restore object location.
219 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
220 m_LocationCode = field.GetDatabaseName();
221 m_Location = s_LocationCodeToLocation(m_LocationCode);
222
223 // Restore storage-specific info.
224 if (m_Location == eNFL_NetCache) {
225 // Not used, though has to be read to be backward-compatible
226 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
227
228 // Get the service name.
229 VERIFY_FIELD_EXISTS(field = field.GetNextNeighbor());
230 m_NCServiceName = field.GetServiceName();
231 }
232
233 break;
234 }
235 }
236 }
237
SetLocation(const string & nc_service_name)238 void CNetStorageObjectLoc::SetLocation(const string& nc_service_name)
239 {
240 // If NetCache
241 if (!nc_service_name.empty()) {
242 if (m_Location == eNFL_NetCache) return;
243
244 m_LocationCode = NETCACHE_STORAGE_CODE;
245 m_Location = eNFL_NetCache;
246 } else {
247 if (m_Location == eNFL_FileTrack) return;
248
249 m_LocationCode = FILETRACK_STORAGE_CODE;
250 m_Location = eNFL_FileTrack;
251 }
252
253 m_Dirty = true;
254 m_NCServiceName = nc_service_name;
255 }
256
GetFileTrackSite() const257 CNetStorageObjectLoc::EFileTrackSite CNetStorageObjectLoc::GetFileTrackSite() const
258 {
259 return m_LocatorFlags & fLF_DevEnv ? eFileTrack_DevSite :
260 m_LocatorFlags & fLF_QAEnv ? eFileTrack_QASite :
261 eFileTrack_ProdSite;
262 }
263
MakeShortUniqueKey() const264 string CNetStorageObjectLoc::MakeShortUniqueKey() const
265 {
266 string result = NStr::NumericToString(m_Timestamp);
267 result += '-';
268 result += NStr::NumericToString(m_Random);
269
270 if (m_LocatorFlags & fLF_HasObjectID) {
271 result += '-';
272 result += NStr::NumericToString(m_ObjectID);
273 }
274
275 return result;
276 }
277
x_Pack() const278 void CNetStorageObjectLoc::x_Pack() const
279 {
280 // Allocate a new CompoundID object.
281 CCompoundID cid = m_CompoundIDPool.NewID(eCIC_NetStorageObjectLoc);
282
283 // Save locator flags.
284 // If its location is not set yet, add "Movable", so the locator can be used
285 cid.AppendFlags(m_Location != eNFL_Unknown ?
286 m_LocatorFlags : (m_LocatorFlags | fLF_Movable));
287
288 // Save NetStorage service name.
289 if (m_LocatorFlags & fLF_NetStorageService)
290 cid.AppendServiceName(m_ServiceName);
291
292 // Save object ID.
293 if (m_LocatorFlags & fLF_HasObjectID)
294 cid.AppendID(m_ObjectID);
295
296 // Save the domain name.
297 cid.AppendString(m_AppDomain);
298
299 // Save object identification
300 if (m_LocatorFlags & fLF_HasUserKey) {
301 // Save the unique object key.
302 cid.AppendString(m_ShortUniqueKey);
303
304 if (m_LocatorFlags & fLF_HasSubKey) {
305 cid.AppendString(m_SubKey);
306
307 if (m_LocatorFlags & fLF_HasVersion) {
308 // null is stored as 0 (0 is not stored as it's the default)
309 cid.AppendInteger(m_Version.IsNull() ? 0 : m_Version.GetValue());
310 }
311 }
312 } else {
313 // Save object creation timestamp.
314 cid.AppendTimestamp(m_Timestamp);
315 // Save the random ID.
316 cid.AppendRandom((Uint4)(m_Random >> (sizeof(Uint4) * 8)));
317 cid.AppendRandom((Uint4) m_Random);
318 }
319
320 // Not used, though has to be written to be backward-compatible
321 if (m_LocatorFlags & fLF_Cacheable)
322 cid.AppendInteger(0);
323
324 // Save storage info (optional).
325 if (m_Location != eNFL_Unknown) {
326 cid.AppendCue(STORAGE_INFO_CUE);
327
328 // Save object location.
329 cid.AppendDatabaseName(m_LocationCode);
330
331 switch (m_Location) {
332 case eNFL_NetCache:
333 // Not used, though has to be written to be backward-compatible
334 cid.AppendFlags(0);
335 // Save the service name.
336 cid.AppendServiceName(m_NCServiceName);
337 break;
338 default:
339 break;
340 }
341 }
342
343 // Now pack it all up.
344 m_Locator = cid.ToString();
345
346 m_Dirty = false;
347 }
348
GetStorageAttrFlags() const349 TNetStorageAttrFlags CNetStorageObjectLoc::GetStorageAttrFlags() const
350 {
351 TNetStorageAttrFlags flags = 0;
352
353 if (m_LocatorFlags & fLF_Movable)
354 flags |= fNST_Movable;
355 if (m_LocatorFlags & fLF_Cacheable)
356 flags |= fNST_Cacheable;
357 if (m_LocatorFlags & fLF_NoMetaData)
358 flags |= fNST_NoMetaData;
359
360 return flags;
361 }
362
SetStorageAttrFlags(TNetStorageAttrFlags flags)363 void CNetStorageObjectLoc::SetStorageAttrFlags(TNetStorageAttrFlags flags)
364 {
365 const auto new_locator_flags = (m_LocatorFlags & eLF_FieldFlags) | x_StorageFlagsToLocatorFlags(flags);
366
367 if (new_locator_flags != m_LocatorFlags) {
368 m_Dirty = true;
369 m_LocatorFlags = new_locator_flags;
370 }
371 }
372
373 CNetStorageObjectLoc::TLocatorFlags
x_StorageFlagsToLocatorFlags(TNetStorageAttrFlags storage_flags,EFileTrackSite ft_site)374 CNetStorageObjectLoc::x_StorageFlagsToLocatorFlags(
375 TNetStorageAttrFlags storage_flags,
376 EFileTrackSite ft_site)
377 {
378 TLocatorFlags locator_flags = 0;
379
380 if (storage_flags & fNST_Movable)
381 locator_flags |= fLF_Movable;
382 if (storage_flags & fNST_Cacheable)
383 locator_flags |= fLF_Cacheable;
384 if (storage_flags & fNST_NoMetaData)
385 locator_flags |= fLF_NoMetaData;
386
387 if (ft_site == eFileTrack_DevSite)
388 locator_flags |= fLF_DevEnv;
389 else if (ft_site == eFileTrack_QASite)
390 locator_flags |= fLF_QAEnv;
391
392 return locator_flags;
393 }
394
ToJSON() const395 CJsonNode CNetStorageObjectLoc::ToJSON() const
396 {
397 CJsonNode root(CJsonNode::NewObjectNode());
398 ToJSON(root);
399 return root;
400 }
401
ToJSON(CJsonNode & root) const402 void CNetStorageObjectLoc::ToJSON(CJsonNode& root) const
403 {
404 root.SetInteger("Version", 1 +
405 eCIC_NetStorageObjectLoc - eCIC_NetStorageObjectLocV1);
406
407 root.SetString("Environment",
408 m_LocatorFlags & fLF_DevEnv ? "dev/test" :
409 m_LocatorFlags & fLF_QAEnv ? "QA" : "production");
410
411 if (m_LocatorFlags & fLF_NetStorageService)
412 root.SetString("ServiceName", m_ServiceName);
413
414 if (m_LocatorFlags & fLF_HasSubKey) {
415 root.SetString("Cache", m_AppDomain);
416 root.SetString("ObjectKey", m_ShortUniqueKey);
417 root.SetString("ObjectSubKey", m_SubKey);
418
419 if (m_Version.IsNull()) {
420 root.SetNull("ObjectVersion");
421 } else {
422 root.SetInteger("ObjectVersion", m_Version);
423 }
424 } else {
425 root.SetString("ObjectKey", m_UniqueKey);
426 }
427
428 CJsonNode storage_flags(CJsonNode::NewObjectNode());
429
430 storage_flags.SetBoolean("Movable",
431 (m_LocatorFlags & fLF_Movable) != 0);
432 storage_flags.SetBoolean("Cacheable",
433 (m_LocatorFlags & fLF_Cacheable) != 0);
434 storage_flags.SetBoolean("NoMetaData",
435 (m_LocatorFlags & fLF_NoMetaData) != 0);
436
437 root.SetByKey("StorageFlags", storage_flags);
438
439 if (!m_LocationCode.empty())
440 root.SetString("DefaultLocation", m_LocationCode);
441
442 CJsonNode storage_info(CJsonNode::NewObjectNode());
443
444 switch (m_Location) {
445 case eNFL_NetCache:
446 storage_info.SetString("ServiceName", m_NCServiceName);
447 root.SetByKey("NetCache", storage_info);
448 break;
449 default:
450 break;
451 }
452 }
453
ParseFileTrackSite(const string & ft_site_name)454 CNetStorageObjectLoc::EFileTrackSite CNetStorageObjectLoc::ParseFileTrackSite(const string& ft_site_name)
455 {
456 static map<CTempString, EFileTrackSite, PNocase> p =
457 {
458 { "production", eFileTrack_ProdSite },
459 { "prod", eFileTrack_ProdSite },
460 { "submit", eFileTrack_ProdSite },
461 { "development", eFileTrack_DevSite },
462 { "dev", eFileTrack_DevSite },
463 { "dsubmit", eFileTrack_DevSite },
464 { "qa", eFileTrack_QASite },
465 { "qsubmit", eFileTrack_QASite },
466 };
467
468 auto i = p.find(ft_site_name);
469
470 if (i == p.end()) {
471 NCBI_THROW_FMT(CArgException, eInvalidArg, "unrecognized FileTrack site '" << ft_site_name << '\'');
472 }
473
474 return i->second;
475 }
476
Create(const string & service_name,const string & cache_name,const string & key,const string & subkey,const TVersion & version)477 string CNetStorageObjectLoc::Create(const string& service_name, const string& cache_name,
478 const string& key, const string& subkey, const TVersion& version)
479 {
480 CCompoundIDPool id_pool;
481 auto ft_site = ParseFileTrackSite(CDiagContext::GetHostRole());
482 CNetStorageObjectLoc loc(id_pool, fNST_NoMetaData, cache_name, key, ft_site);
483 loc.SetLocation(service_name);
484 loc.m_SubKey = subkey;
485 loc.m_Version = version;
486 loc.m_LocatorFlags |= (version == TVersion(0) ? 0 : fLF_HasVersion) | fLF_HasSubKey;
487 return loc.GetLocator();
488 }
489
490 END_NCBI_SCOPE
491