1 #ifndef NETCACHE__NC_DB_INFO__HPP
2 #define NETCACHE__NC_DB_INFO__HPP
3 /* $Id: nc_db_info.hpp 632770 2021-06-07 19:05:12Z ivanov $
4 * ===========================================================================
5 *
6 * PUBLIC DOMAIN NOTICE
7 * National Center for Biotechnology Information
8 *
9 * This software/database is a "United States Government Work" under the
10 * terms of the United States Copyright Act. It was written as part of
11 * the author's official duties as a United States Government employee and
12 * thus cannot be copyrighted. This software/database is freely available
13 * to the public for use. The National Library of Medicine and the U.S.
14 * Government have not placed any restriction on its use or reproduction.
15 *
16 * Although all reasonable efforts have been taken to ensure the accuracy
17 * and reliability of the software and data, the NLM and the U.S.
18 * Government do not and cannot warrant the performance or results that
19 * may be obtained by using this software or data. The NLM and the U.S.
20 * Government disclaim all warranties, express or implied, including
21 * warranties of performance, merchantability or fitness for any particular
22 * purpose.
23 *
24 * Please cite the author in any work or product based on this material.
25 *
26 * ===========================================================================
27 *
28 * Authors: Pavel Ivanov
29 */
30
31
32 #include "nc_utils.hpp"
33 #include "memory_man.hpp"
34
35
36 namespace intr = boost::intrusive;
37
38
39 BEGIN_NCBI_SCOPE
40
41
42 static const Uint4 kNCMaxDBFileId = 0xFFFFFFFF;
43 /// Maximum size of blob chunk stored in database.
44 /// It's a little bit uneven to be efficient in the current memory manager.
45 /// I.e. memory manager in TaskServer allocates memory with pages of 65536 bytes,
46 /// uses some of that memory for page header and then splits the rest in half to
47 /// obtain the maximum block size it can allocate this way. kNCMaxBlobChunkSize
48 /// should be equal or less than that size (which is kMMMaxBlockSize in
49 /// memory_man.cpp in task_server library).
50 #if __NC_MEMMAN_NEWALLOC && __NC_MEMMAN_USEREALPTR
51 static const size_t kNCMaxBlobChunkSize = 32736;
52 #else
53 static const size_t kNCMaxBlobChunkSize = 32740;
54 #endif
55 static const Uint2 kNCMaxChunksInMap = 128;
56
57 static const Uint1 kNCMaxBlobMapsDepth = 3;
58
59 // 32740 * 128^3 - 32740 = 68'660'723'740
60 const Uint8 kNCLargestBlobSize =
61 Uint8(kNCMaxChunksInMap) * Uint8(kNCMaxChunksInMap) * Uint8(kNCMaxChunksInMap) * Uint8(kNCMaxBlobChunkSize) - Uint8(kNCMaxBlobChunkSize);
62
63
64
65 class CNCBlobVerManager;
66 struct SNCChunkMaps;
67
68
69 struct ATTR_PACKED SNCDataCoord
70 {
71 Uint4 file_id;
72 Uint4 rec_num;
73
74
SNCDataCoordSNCDataCoord75 SNCDataCoord(void)
76 : file_id(0), rec_num(0) {
77 }
78 bool empty(void) const;
79 void clear(void);
80 };
81
82 const CSrvDiagMsg& operator<< (const CSrvDiagMsg& msg, SNCDataCoord coord);
83 bool operator== (SNCDataCoord left, SNCDataCoord right);
84 bool operator!= (SNCDataCoord left, SNCDataCoord right);
85
86
87 struct SVersMap_tag;
88 typedef intr::set_base_hook<intr::tag<SVersMap_tag>,
89 intr::optimize_size<true> > TVerDataMapHook;
90
91
92 /// Full information about NetCache blob (excluding key)
93 struct SNCBlobVerData : public CObject,
94 public CSrvTask,
95 public TVerDataMapHook
96 {
97 public:
98 SNCDataCoord coord;
99 SNCDataCoord data_coord;
100 Uint8 size;
101 string password;
102 Uint8 create_time;
103 Uint8 create_server;
104 Uint4 create_id;
105 Uint8 updated_on_server;
106 Uint8 updated_at_time;
107 Uint8 update_received;
108 unsigned int ttl;
109 int expire;
110 int dead_time;
111 int blob_ver;
112 int ver_ttl;
113 int ver_expire;
114 Uint4 chunk_size;
115 Uint2 map_size;
116 Uint1 map_depth;
117 bool has_error;
118
119 bool is_cur_version;
120 bool meta_has_changed;
121 bool move_or_rewrite;
122 bool is_releasable;
123 bool request_data_write;
124 bool need_write_all;
125 bool need_stop_write;
126 bool need_mem_release;
127 bool delete_scheduled;
128 Uint4 map_move_counter;
129 int last_access_time;
130 int need_write_time;
131 Uint8 cnt_chunks;
132 Uint8 cur_chunk_num;
133 SNCChunkMaps* chunk_maps;
134
135 CNCBlobVerManager* ver_manager;
136
137 int saved_access_time;
138 CMiniMutex wb_mem_lock;
139 size_t meta_mem;
140 size_t data_mem;
141 size_t releasable_mem;
142 size_t releasing_mem;
143 vector<char*> chunks;
144
145
146 SNCBlobVerData(CNCBlobVerManager* mgr);
147 virtual ~SNCBlobVerData(void);
148
149 public:
150 void AddChunkMem(char* mem, Uint4 mem_size);
151 void RequestDataWrite(void);
152 size_t RequestMemRelease(void);
153 void SetNotCurrent(void);
154 void SetCurrent(void);
155 void SetReleasable(void);
156 void SetNonReleasable(void);
157
158 private:
159 SNCBlobVerData(const SNCBlobVerData&);
160 SNCBlobVerData& operator= (const SNCBlobVerData&);
161
162 virtual void DeleteThis(void);
163 virtual void ExecuteSlice(TSrvThreadNum thr_num);
164
165 void x_FreeChunkMaps(void);
166 bool x_WriteBlobInfo(void);
167 bool x_WriteCurChunk(char* write_mem, Uint4 write_size);
168 bool x_ExecuteWriteAll(void);
169 void x_DeleteVersion(void);
170 };
171
172
173 struct SCompareVerAccessTime
174 {
operator ()SCompareVerAccessTime175 bool operator() (const SNCBlobVerData& left, const SNCBlobVerData& right) const
176 {
177 return left.saved_access_time < right.saved_access_time;
178 }
179 };
180
181 typedef intr::rbtree<SNCBlobVerData,
182 intr::base_hook<TVerDataMapHook>,
183 intr::constant_time_size<false>,
184 intr::compare<SCompareVerAccessTime> > TVerDataMap;
185
186 struct SNCChunkMapInfo
187 {
188 Uint4 map_counter;
189 Uint2 map_idx;
190 SNCDataCoord coords[1] ATTR_ALIGNED_8;
191 };
192
193 struct SNCChunkMaps
194 {
195 SNCChunkMapInfo* maps[kNCMaxBlobMapsDepth + 1];
196
197
198 SNCChunkMaps(Uint2 map_size);
199 ~SNCChunkMaps(void);
200 };
201
202 struct SNCBlobSummary
203 {
204 Uint8 size;
205 Uint8 create_time;
206 Uint8 create_server;
207 Uint4 create_id;
208 int dead_time;
209 int expire;
210 int ver_expire;
211
SNCBlobSummarySNCBlobSummary212 SNCBlobSummary(void)
213 : size(0), create_time(0), create_server(0), create_id(0),
214 dead_time(0), expire(0), ver_expire(0)
215 {
216 }
resetSNCBlobSummary217 void reset(void) {
218 memset(this, 0, sizeof(SNCBlobSummary));
219 }
operator =SNCBlobSummary220 SNCBlobSummary& operator=( const SNCBlobSummary& other) {
221 if (this != &other) {
222 memcpy(this, &other, sizeof(SNCBlobSummary));
223 }
224 return *this;
225 }
isOlderSNCBlobSummary226 bool isOlder(const SNCBlobSummary& other) const
227 {
228 if (create_time != other.create_time)
229 return create_time < other.create_time;
230 else if (create_server != other.create_server)
231 return create_server < other.create_server;
232 else if (create_id != other.create_id)
233 return create_id < other.create_id;
234 else if (expire != other.expire)
235 return expire < other.expire;
236 else if (ver_expire != other.ver_expire)
237 return ver_expire < other.ver_expire;
238 else
239 return dead_time < other.dead_time;
240 }
241
isSameDataSNCBlobSummary242 bool isSameData(const SNCBlobSummary& other) const
243 {
244 return create_time == other.create_time
245 && create_server == other.create_server
246 && create_id == other.create_id;
247 }
248
isEqualSNCBlobSummary249 bool isEqual(const SNCBlobSummary& other) const
250 {
251 return create_time == other.create_time
252 && create_server == other.create_server
253 && create_id == other.create_id
254 && dead_time == other.dead_time
255 && expire == other.expire
256 && ver_expire == other.ver_expire;
257 }
isExpiredSNCBlobSummary258 bool isExpired(void) const
259 {
260 return expire < CSrvTime::CurSecs();
261 }
262 };
263
264 struct SNCBlobFilter {
SNCBlobFilterSNCBlobFilter265 SNCBlobFilter(void) {
266 memset(this, 0, sizeof(SNCBlobFilter));
267 }
268 Uint8 cr_ago_ge;
269 Uint8 cr_ago_lt;
270 Uint8 cr_epoch_ge;
271 Uint8 cr_epoch_lt;
272 Uint8 exp_now_ge;
273 Uint8 exp_now_lt;
274 Uint8 exp_epoch_ge;
275 Uint8 exp_epoch_lt;
276 Uint8 vexp_now_ge;
277 Uint8 vexp_now_lt;
278 Uint8 vexp_epoch_ge;
279 Uint8 vexp_epoch_lt;
280 Uint8 cr_srv;
281 Uint8 size_ge;
282 Uint8 size_lt;
283 };
284
285
286 class CNCBlobVerManager;
287
288 struct SNCBlobSummary;
289 typedef map<string, SNCBlobSummary*> TNCBlobSumList;
290
291
292
293 enum ENCDBFileType {
294 eDBFileMeta = 0x01,
295 eDBFileData = 0x02,
296 eDBFileMaps = 0x03
297 //fDBFileForMove = 0x80,
298 //eDBFileMoveMeta = eDBFileMeta + fDBFileForMove,
299 //eDBFileMoveData = eDBFileData + fDBFileForMove,
300 //eDBFileMoveMaps = eDBFileMaps + fDBFileForMove
301 };
302
303 enum EDBFileIndex {
304 eFileIndexMeta = 0,
305 eFileIndexData = 1,
306 eFileIndexMaps = 2
307 //eFileIndexMoveShift = 3,
308 //eFileIndexMoveMeta = eFileIndexMeta + eFileIndexMoveShift,
309 //eFileIndexMoveData = eFileIndexData + eFileIndexMoveShift,
310 //eFileIndexMoveMaps = eFileIndexMaps + eFileIndexMoveShift
311 };
312
313
314 #if defined(NCBI_OS_MSWIN)
315 typedef HANDLE TFileHandle;
316 #else
317 typedef int TFileHandle;
318 #endif
319
320
321 struct SFileIndexRec;
322
323 /// Information about database part in NetCache storage
324 struct SNCDBFileInfo : public CObject
325 {
326 char* file_map;
327 Uint4 file_id;
328 Uint4 file_size;
329 Uint4 garb_size;
330 Uint4 used_size;
331 SFileIndexRec* index_head;
332 CMiniMutex info_lock;
333 bool is_releasing;
334 TFileHandle fd;
335 int create_time;
336 int next_shrink_time;
337 CAtomicCounter cnt_unfinished;
338 ENCDBFileType file_type;
339 EDBFileIndex type_index;
340 string file_name;
341
342
343 SNCDBFileInfo(void);
344 virtual ~SNCDBFileInfo(void);
345 };
346 /// Information about all database parts in NetCache storage
347 typedef map<Uint4, CSrvRef<SNCDBFileInfo> > TNCDBFilesMap;
348
349
350
351 inline bool
empty(void) const352 SNCDataCoord::empty(void) const
353 {
354 return file_id == 0 && rec_num == 0;
355 }
356
357 inline void
clear(void)358 SNCDataCoord::clear(void)
359 {
360 file_id = rec_num = 0;
361 }
362
363 inline const CSrvDiagMsg&
operator <<(const CSrvDiagMsg & msg,SNCDataCoord coord)364 operator<< (const CSrvDiagMsg& msg, SNCDataCoord coord)
365 {
366 msg << "(" << coord.file_id << ", " << coord.rec_num << ")";
367 return msg;
368 }
369
370 inline bool
operator ==(SNCDataCoord left,SNCDataCoord right)371 operator== (SNCDataCoord left, SNCDataCoord right)
372 {
373 return left.file_id == right.file_id && left.rec_num == right.rec_num;
374 }
375
376 inline bool
operator !=(SNCDataCoord left,SNCDataCoord right)377 operator!= (SNCDataCoord left, SNCDataCoord right)
378 {
379 return !(left == right);
380 }
381
382 END_NCBI_SCOPE
383
384 #endif /* NETCACHE__NC_DB_INFO__HPP */
385