1 #ifndef NETCACHE__NC_STORAGE_BLOB__HPP
2 #define NETCACHE__NC_STORAGE_BLOB__HPP
3 /* $Id: nc_storage_blob.hpp 535739 2017-05-11 13:15:22Z gouriano $
4 * ===========================================================================
5 *
6 * PUBLIC DOMAIN NOTICE
7 * National Center for Biotechnology Information
8 *
9 * This software/database is a "United States Government Work" under the
10 * terms of the United States Copyright Act. It was written as part of
11 * the author's official duties as a United States Government employee and
12 * thus cannot be copyrighted. This software/database is freely available
13 * to the public for use. The National Library of Medicine and the U.S.
14 * Government have not placed any restriction on its use or reproduction.
15 *
16 * Although all reasonable efforts have been taken to ensure the accuracy
17 * and reliability of the software and data, the NLM and the U.S.
18 * Government do not and cannot warrant the performance or results that
19 * may be obtained by using this software or data. The NLM and the U.S.
20 * Government disclaim all warranties, express or implied, including
21 * warranties of performance, merchantability or fitness for any particular
22 * purpose.
23 *
24 * Please cite the author in any work or product based on this material.
25 *
26 * ===========================================================================
27 *
28 * Authors: Pavel Ivanov
29 */
30
31
32 #include "nc_db_info.hpp"
33
34
35 BEGIN_NCBI_SCOPE
36
37
38 class CNCBlobStorage;
39 class CNCBlobAccessor;
40 class SNCCacheData;
41 class CCurVerReader;
42 struct SNCStateStat;
43
44
45 class CNCBlobVerManager : public CObject, public CSrvTask
46 {
47 public:
48 static CNCBlobVerManager* Get(Uint2 time_bucket,
49 const string& key,
50 SNCCacheData* cache_data,
51 bool for_new_version);
52 void ObtainReference(void);
53 void Release(void);
54
55 void RequestCurVersion(CSrvTransConsumer* consumer);
56 CSrvRef<SNCBlobVerData> GetCurVersion(void);
57 CSrvRef<SNCBlobVerData> CreateNewVersion(void);
58
59 void FinalizeWriting(SNCBlobVerData* ver_data);
60 void DeadTimeChanged(SNCBlobVerData* ver_data);
61 void DeleteVersion(const SNCBlobVerData* ver_data);
62 void DeleteDeadVersion(int cut_time);
63
64 SNCCacheData* GetCacheData(void);
65
66 public:
67 void RevokeDataWrite();
68 void DataWritten(void);
69 void RequestMemRelease(void);
70 const string& GetKey(void);
71
72 private:
73 CNCBlobVerManager(const CNCBlobVerManager&);
74 CNCBlobVerManager& operator= (const CNCBlobVerManager&);
75 CNCBlobVerManager(void);
76
77 CNCBlobVerManager(Uint2 time_bucket,
78 const string& key,
79 SNCCacheData* cache_data);
80 virtual ~CNCBlobVerManager(void);
81
82 virtual void DeleteThis(void);
83 virtual void ExecuteSlice(TSrvThreadNum thr_num);
84
85 void x_DeleteCurVersion(void);
86 void x_ReleaseMgr(void);
87
88
89 friend class CCurVerReader;
90
91
92 Uint2 m_TimeBucket;
93 bool m_NeedReleaseMem;
94 bool m_NeedAbort;
95 SNCCacheData* m_CacheData;
96 CCurVerReader* m_CurVerReader;
97 string m_Key;
98 CSrvRef<SNCBlobVerData> m_CurVersion;
99 };
100
101
102 class CNCBlobAccessor : public CSrvTransConsumer
103 {
104 public:
105 void RequestMetaInfo(CSrvTask* owner);
106 bool IsMetaInfoReady(void);
107 /// Check if blob exists.
108 /// Method can be called only after lock is acquired.
109 bool IsBlobExists(void) const;
110 /// Check if password provided for accessing the blob was correct
111 bool IsAuthorized(void) const;
112 bool HasError(void) const;
113 /// Get key of the blob.
114 /// Method can be called only after lock is acquired.
115 const string& GetBlobKey (void) const;
116 Uint2 GetTimeBucket (void) const;
117 /// Get size of the blob.
118 /// Method can be called only after lock is acquired.
119 /// If blob doesn't exist then value is undefined.
120 Uint8 GetCurBlobSize (void) const;
121 Uint8 GetNewBlobSize (void) const;
122 Uint8 GetSizeRead (void) const;
123 /// Check if blob is already expired but not yet deleted by GC.
124 /// Method can be called only after lock is acquired.
125 /// If blob doesn't exist then value is undefined.
126 bool IsCurBlobExpired (void) const;
127 bool IsCurVerExpired (void) const;
128 bool IsCurBlobDead (void) const;
129 /// Get type of access this holder was created for
130 ENCAccessType GetAccessType (void) const;
131
132 /// Initially set current position in the blob to start reading from
133 void SetPosition(Uint8 pos);
134 Uint8 GetPosition(void);
135 Uint4 GetReadMemSize(void);
136 const void* GetReadMemPtr(void);
137 void MoveReadPos(Uint4 move_size);
138 unsigned int GetCurBlobTTL(void) const;
139 unsigned int GetNewBlobTTL(void) const;
140 /// Set blob's timeout after last access before it will be deleted.
141 /// Method can be called only after lock is acquired, if blob exists and
142 /// lock is acquired for eNCDelete or eNCCreate access.
143 void SetBlobTTL(unsigned int ttl);
144 unsigned int GetCurVersionTTL(void) const;
145 unsigned int GetNewVersionTTL(void) const;
146 void SetVersionTTL(int ttl);
147 int GetCurBlobVersion(void) const;
148 void SetBlobVersion(int ver);
149 Uint8 GetCurBlobCreateTime(void) const;
150 Uint8 GetNewBlobCreateTime(void) const;
151 void SetBlobCreateTime(Uint8 create_time);
152 int GetCurBlobDeadTime(void) const;
153 int GetCurBlobExpire(void) const;
154 int GetNewBlobExpire(void) const;
155 int GetCurVerExpire(void) const;
156 void SetCurBlobExpire(int expire, int dead_time = 0);
157 void SetNewBlobExpire(int expire, int dead_time = 0);
158 void SetCurVerExpire(int dead_time);
159 void SetNewVerExpire(int dead_time);
160 Uint8 GetCurCreateServer(void) const;
161 Uint8 GetNewCreateServer(void) const;
162 Uint4 GetCurCreateId(void) const;
163 void SetCreateServer(Uint8 create_server, Uint4 create_id);
164 void UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time);
165 bool IsValid(void) const;
166 Uint8 GetValidServer(void) const;
167 string GetCurPassword(void) const;
168 void SetPassword(CTempString password);
169 bool ReplaceBlobInfo(const SNCBlobVerData& new_info);
170
171 size_t GetWriteMemSize(void);
172 void* GetWriteMemPtr(void);
173 void MoveWritePos(Uint4 move_size);
174 void Finalize(void);
175
176 /// Release blob lock.
177 /// No other method can be called after call to this one.
178 void Release(void);
179
180 public:
181 // For internal use only
182
183 /// Create holder of blob lock bound to the given NetCache storage.
184 /// Lock holder is yet not usable after construction until Prepare()
185 /// and InitializeLock() are called.
186 CNCBlobAccessor(void);
187 virtual ~CNCBlobAccessor(void);
188
189 bool IsPurged(const CNCBlobKeyLight& nc_key) const;
190 static Uint8 GetPurgeCount();
191 static bool Purge(const CNCBlobKeyLight& nc_key, Uint8 when);
192 static string GetPurgeData(char separator='\n');
193 static bool UpdatePurgeData(const string& data, char separator='\n');
194
195 static void SetFailedWriteCount(int failed_write);
196 static int GetFailedWriteCount(void);
197 static void PutFailed(const string& blob_key);
198 static void PutSucceeded(const string& blob_key);
199 static bool HasPutSucceeded(const string& blob_key);
200
201 /// Prepare lock for the blob identified by key, subkey and version.
202 /// Method only initializes necessary variables, actual acquiring of the
203 /// lock happens in InitializeLock() method.
204 /// If access is eNCRead and storage's IsChangeTimeOnRead() returns TRUE
205 /// then holder will change blob's access time when lock is released. If
206 /// access is eNCCreate then blob will be automatically deleted
207 /// when lock is released if blob wasn't properly finalized.
208 ///
209 /// @param key
210 /// Key of the blob
211 /// @param access
212 /// Required access to the blob
213 void Prepare(const string& key,
214 const string& password,
215 Uint2 time_bucket,
216 ENCAccessType access_type);
217 /// Initialize and acquire the lock.
218 /// Should be called only after Prepare().
219 void Initialize(SNCCacheData* cache_data);
220 void Deinitialize(void);
221
222 private:
223 CNCBlobAccessor(const CNCBlobAccessor&);
224 CNCBlobAccessor& operator= (const CNCBlobAccessor&);
225
226 virtual void ExecuteSlice(TSrvThreadNum thr_num);
227
228 void x_CreateNewData(void);
229 void x_DelCorruptedVersion(void);
230
231
232 /// Type of access requested for the blob
233 ENCAccessType m_AccessType;
234 string m_BlobKey;
235 /// Password that was used for access to the blob
236 string m_Password;
237 CNCBlobVerManager* m_VerManager;
238 CSrvRef<SNCBlobVerData> m_CurData;
239 CSrvRef<SNCBlobVerData> m_NewData;
240 SNCChunkMaps* m_ChunkMaps;
241 bool m_HasError;
242 bool m_MetaInfoReady;
243 bool m_WriteMemRequested;
244 Uint2 m_TimeBucket;
245 Uint8 m_CurChunk;
246 /// Current position of reading/writing inside blob's chunk
247 Uint4 m_ChunkPos;
248 Uint4 m_ChunkSize;
249 Uint8 m_SizeRead;
250 char* m_Buffer;
251 CSrvTask* m_Owner;
252 };
253
254
255 class CCurVerReader : public CSrvTransitionTask
256 {
257 public:
258 CCurVerReader(CNCBlobVerManager* mgr);
259 virtual ~CCurVerReader(void);
260
261 private:
262 virtual void ExecuteSlice(TSrvThreadNum thr_num);
263
264
265 CNCBlobVerManager* m_VerMgr;
266 };
267
268
269 class CWriteBackControl : public CSrvTask
270 {
271 public:
272 static void Initialize(void);
273 static void ReadState(SNCStateStat& state);
274
275 // statistics
276 static void AnotherServerMain(void);
277 static void StartSyncBlob(Uint8 create_time);
278 static void RecordNotifyUpdateBlob(Uint8 update_received);
279 static void ResetStatCounters(void);
280
281 private:
282 CWriteBackControl(void);
283 virtual ~CWriteBackControl(void);
284
285 virtual void ExecuteSlice(TSrvThreadNum thr_num);
286 };
287
288
289 Uint8 GetWBSoftSizeLimit(void);
290 Uint8 GetWBHardSizeLimit(void);
291 int GetWBWriteTimeout(void);
292 int GetWBFailedWriteDelay(void);
293
294 void SetWBSoftSizeLimit(Uint8 limit);
295 void SetWBHardSizeLimit(Uint8 limit);
296 void SetWBWriteTimeout(int timeout1, int timeout2);
297 void SetWBFailedWriteDelay(int delay);
298 void SetWBInitialSyncComplete(void);
299
300
301 class CWBMemDeleter : public CSrvRCUUser
302 {
303 public:
304 CWBMemDeleter(char* mem, Uint4 mem_size);
305 virtual ~CWBMemDeleter(void);
306
307 private:
308 virtual void ExecuteRCU(void);
309
310
311 char* m_Mem;
312 Uint4 m_MemSize;
313 };
314
315
316
317 //////////////////////////////////////////////////////////////////////////
318 // Inline methods
319 //////////////////////////////////////////////////////////////////////////
320
321 inline SNCCacheData*
GetCacheData(void)322 CNCBlobVerManager::GetCacheData(void)
323 {
324 return m_CacheData;
325 }
326
327 inline const string&
GetKey(void)328 CNCBlobVerManager::GetKey(void)
329 {
330 return m_Key;
331 }
332
333 inline void
RequestCurVersion(CSrvTransConsumer * consumer)334 CNCBlobVerManager::RequestCurVersion(CSrvTransConsumer* consumer)
335 {
336 m_CurVerReader->RequestTransition(consumer);
337 }
338
339
340 inline bool
IsMetaInfoReady(void)341 CNCBlobAccessor::IsMetaInfoReady(void)
342 {
343 return m_MetaInfoReady;
344 }
345
346 inline bool
IsBlobExists(void) const347 CNCBlobAccessor::IsBlobExists(void) const
348 {
349 return m_CurData.NotNull();
350 }
351
352 inline bool
IsAuthorized(void) const353 CNCBlobAccessor::IsAuthorized(void) const
354 {
355 return !IsBlobExists() || IsCurBlobExpired()
356 || m_CurData->password == m_Password;
357 }
358
359 inline bool
HasError(void) const360 CNCBlobAccessor::HasError(void) const
361 {
362 return m_HasError;
363 }
364
365 inline const string&
GetBlobKey(void) const366 CNCBlobAccessor::GetBlobKey(void) const
367 {
368 return m_BlobKey;
369 }
370
371 inline Uint2
GetTimeBucket(void) const372 CNCBlobAccessor::GetTimeBucket(void) const
373 {
374 return m_TimeBucket;
375 }
376
377 inline Uint8
GetCurBlobSize(void) const378 CNCBlobAccessor::GetCurBlobSize(void) const
379 {
380 return m_CurData->size;
381 }
382
383 inline Uint8
GetNewBlobSize(void) const384 CNCBlobAccessor::GetNewBlobSize(void) const
385 {
386 return m_NewData? m_NewData->size: 0;
387 }
388
389 inline Uint8
GetSizeRead(void) const390 CNCBlobAccessor::GetSizeRead(void) const
391 {
392 return m_SizeRead;
393 }
394
395 inline int
GetCurBlobVersion(void) const396 CNCBlobAccessor::GetCurBlobVersion(void) const
397 {
398 return m_CurData->blob_ver;
399 }
400
401 inline unsigned int
GetCurBlobTTL(void) const402 CNCBlobAccessor::GetCurBlobTTL(void) const
403 {
404 return m_CurData->ttl;
405 }
406
407 inline unsigned int
GetNewBlobTTL(void) const408 CNCBlobAccessor::GetNewBlobTTL(void) const
409 {
410 return m_NewData->ttl;
411 }
412
413 inline bool
IsCurBlobExpired(void) const414 CNCBlobAccessor::IsCurBlobExpired(void) const
415 {
416 return m_CurData->expire <= CSrvTime::CurSecs();
417 }
418
419 inline bool
IsCurVerExpired(void) const420 CNCBlobAccessor::IsCurVerExpired(void) const
421 {
422 return m_CurData->ver_expire <= CSrvTime::CurSecs();
423 }
424 inline bool
IsCurBlobDead(void) const425 CNCBlobAccessor::IsCurBlobDead(void) const
426 {
427 return m_CurData->dead_time < CSrvTime::CurSecs();
428 }
429
430 inline void
SetBlobCreateTime(Uint8 create_time)431 CNCBlobAccessor::SetBlobCreateTime(Uint8 create_time)
432 {
433 x_CreateNewData();
434 Uint8 prev = GetCurBlobCreateTime();
435 if (create_time < prev) {
436 // new data cannot be created earlier than the current one.
437 // due to time differences between servers, or who knows what else..
438 // anyway, check here
439 create_time = prev + 1;
440 }
441 m_NewData->create_time = create_time;
442 }
443
444 inline Uint8
GetCurBlobCreateTime(void) const445 CNCBlobAccessor::GetCurBlobCreateTime(void) const
446 {
447 return m_CurData.NotNull()? m_CurData->create_time: 0;
448 }
449
450 inline Uint8
GetNewBlobCreateTime(void) const451 CNCBlobAccessor::GetNewBlobCreateTime(void) const
452 {
453 return m_NewData->create_time;
454 }
455
456 inline int
GetCurBlobDeadTime(void) const457 CNCBlobAccessor::GetCurBlobDeadTime(void) const
458 {
459 return m_CurData->dead_time;
460 }
461
462 inline int
GetCurBlobExpire(void) const463 CNCBlobAccessor::GetCurBlobExpire(void) const
464 {
465 return m_CurData->expire;
466 }
467
468 inline int
GetNewBlobExpire(void) const469 CNCBlobAccessor::GetNewBlobExpire(void) const
470 {
471 return m_NewData->expire;
472 }
473
474 inline int
GetCurVerExpire(void) const475 CNCBlobAccessor::GetCurVerExpire(void) const
476 {
477 return m_CurData->ver_expire;
478 }
479
480 inline void
SetNewBlobExpire(int expire,int dead_time)481 CNCBlobAccessor::SetNewBlobExpire(int expire, int dead_time /* = 0 */)
482 {
483 x_CreateNewData();
484 m_NewData->expire = expire;
485 int d_time = max(dead_time, expire + 40);
486 if (dead_time == 0 && m_CurData) {
487 d_time = max(m_CurData->dead_time, d_time);
488 }
489 m_NewData->dead_time = d_time;
490 }
491
492 inline unsigned int
GetCurVersionTTL(void) const493 CNCBlobAccessor::GetCurVersionTTL(void) const
494 {
495 return m_CurData->ver_ttl;
496 }
497
498 inline unsigned int
GetNewVersionTTL(void) const499 CNCBlobAccessor::GetNewVersionTTL(void) const
500 {
501 return m_NewData->ver_ttl;
502 }
503
504 inline void
SetVersionTTL(int ttl)505 CNCBlobAccessor::SetVersionTTL(int ttl)
506 {
507 x_CreateNewData();
508 m_NewData->ver_ttl = ttl;
509 }
510
511 inline void
SetBlobTTL(unsigned int ttl)512 CNCBlobAccessor::SetBlobTTL(unsigned int ttl)
513 {
514 x_CreateNewData();
515 m_NewData->ttl = ttl;
516 }
517
518 inline void
SetBlobVersion(int ver)519 CNCBlobAccessor::SetBlobVersion(int ver)
520 {
521 x_CreateNewData();
522 m_NewData->blob_ver = ver;
523 }
524
525 inline void
SetCurBlobExpire(int expire,int dead_time)526 CNCBlobAccessor::SetCurBlobExpire(int expire, int dead_time /* = 0 */)
527 {
528 m_CurData->expire = expire;
529 m_CurData->dead_time = max(expire + 40, max(m_CurData->dead_time, dead_time));
530 m_VerManager->DeadTimeChanged(m_CurData);
531 }
532
533 inline void
SetCurVerExpire(int expire)534 CNCBlobAccessor::SetCurVerExpire(int expire)
535 {
536 m_CurData->ver_expire = expire;
537 m_VerManager->DeadTimeChanged(m_CurData);
538 }
539
540 inline void
SetNewVerExpire(int expire)541 CNCBlobAccessor::SetNewVerExpire(int expire)
542 {
543 x_CreateNewData();
544 m_NewData->ver_expire = expire;
545 }
546
547 inline void
SetCreateServer(Uint8 create_server,Uint4 create_id)548 CNCBlobAccessor::SetCreateServer(Uint8 create_server,
549 Uint4 create_id)
550 {
551 x_CreateNewData();
552 m_NewData->create_server = create_server;
553 m_NewData->create_id = create_id;
554 }
555 inline void
UpdateMetaInfo(Uint8 upd_server,Uint8 upd_time)556 CNCBlobAccessor::UpdateMetaInfo(Uint8 upd_server, Uint8 upd_time)
557 {
558 if (m_CurData->create_time < upd_time) {
559 m_CurData->updated_on_server = upd_server;
560 m_CurData->updated_at_time = upd_time;
561 m_CurData->update_received = CSrvTime().Current().AsUSec();
562 }
563 }
564
565 inline bool
IsValid(void) const566 CNCBlobAccessor::IsValid(void) const
567 {
568 return m_CurData.NotNull()? (m_CurData->updated_on_server == 0) : true;
569 }
570
571 inline Uint8
GetValidServer(void) const572 CNCBlobAccessor::GetValidServer(void) const
573 {
574 return m_CurData.NotNull()? m_CurData->updated_on_server : 0;
575 }
576
577
578 inline Uint8
GetCurCreateServer(void) const579 CNCBlobAccessor::GetCurCreateServer(void) const
580 {
581 return m_CurData.NotNull()? m_CurData->create_server: 0;
582 }
583
584 inline Uint8
GetNewCreateServer(void) const585 CNCBlobAccessor::GetNewCreateServer(void) const
586 {
587 return m_NewData->create_server;
588 }
589
590 inline Uint4
GetCurCreateId(void) const591 CNCBlobAccessor::GetCurCreateId(void) const
592 {
593 return m_CurData.NotNull()? m_CurData->create_id: 0;
594 }
595
596 inline void
SetPassword(CTempString password)597 CNCBlobAccessor::SetPassword(CTempString password)
598 {
599 x_CreateNewData();
600 m_NewData->password = password;
601 }
602
603 inline ENCAccessType
GetAccessType(void) const604 CNCBlobAccessor::GetAccessType(void) const
605 {
606 return m_AccessType;
607 }
608
609 inline void
SetPosition(Uint8 pos)610 CNCBlobAccessor::SetPosition(Uint8 pos)
611 {
612 m_CurChunk = pos / m_CurData->chunk_size;
613 m_ChunkPos = size_t(pos % m_CurData->chunk_size);
614 }
615
616 inline Uint8
GetPosition(void)617 CNCBlobAccessor::GetPosition(void)
618 {
619 return m_CurChunk * m_CurData->chunk_size + m_ChunkPos;
620 }
621
622 inline const void*
GetReadMemPtr(void)623 CNCBlobAccessor::GetReadMemPtr(void)
624 {
625 return m_Buffer + m_ChunkPos;
626 }
627
628 inline void*
GetWriteMemPtr(void)629 CNCBlobAccessor::GetWriteMemPtr(void)
630 {
631 return m_Buffer + m_ChunkPos;
632 }
633
634 inline void
MoveWritePos(Uint4 move_size)635 CNCBlobAccessor::MoveWritePos(Uint4 move_size)
636 {
637 m_ChunkPos += move_size;
638 m_NewData->size += move_size;
639 }
640
641 END_NCBI_SCOPE
642
643 #endif /* NETCACHE__NC_STORAGE_BLOB__HPP */
644