1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2015-2015 Planets Communications B.V.
5 Copyright (C) 2015-2016 Bareos GmbH & Co. KG
6
7 This program is Free Software; you can redistribute it and/or
8 modify it under the terms of version three of the GNU Affero General Public
9 License as published by the Free Software Foundation and included
10 in the file LICENSE.
11
12 This program is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Affero General Public License for more details.
16
17 You should have received a copy of the GNU Affero General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 02110-1301, USA.
21 */
22 /*
23 * Marco van Wieringen & Philipp Storz, May 2015
24 */
25 /*
26 * @file
27 * FHDB using LMDB for NDMP Data Management Application (DMA)
28 */
29
30 #include "include/bareos.h"
31 #include "dird.h"
32
33 #if defined(HAVE_NDMP) && defined(HAVE_LMDB)
34
35 #include "ndmp/ndmagents.h"
36 #include "ndmp_dma_priv.h"
37 #include "lmdb/lmdb.h"
38
39 namespace directordaemon {
40
41 /*
42 * What is actually stored in the LMDB
43 */
44 struct fhdb_payload {
45 ndmp9_u_quad node;
46 ndmp9_u_quad dir_node;
47 ndmp9_file_stat ndmp_fstat;
48 char namebuffer[1];
49 };
50
51 struct fhdb_state {
52 uint64_t root_node;
53 POOLMEM* pay_load;
54 POOLMEM* lmdb_name;
55 POOLMEM* path;
56 MDB_env* db_env;
57 MDB_dbi db_dbi;
58 MDB_txn* db_rw_txn;
59 MDB_txn* db_ro_txn;
60 };
61
62 static int debuglevel = 100;
63
64 /*
65 * Payload = 8 + 8 + 96 = 112 bytes + namelength.
66 */
67 #define AVG_NR_BYTES_PER_ENTRY 256
68 #define B_PAGE_SIZE 4096
69
bndmp_fhdb_lmdb_add_dir(struct ndmlog * ixlog,int tagc,char * raw_name,ndmp9_u_quad dir_node,ndmp9_u_quad node)70 extern "C" int bndmp_fhdb_lmdb_add_dir(struct ndmlog* ixlog,
71 int tagc,
72 char* raw_name,
73 ndmp9_u_quad dir_node,
74 ndmp9_u_quad node)
75 {
76 NIS* nis = (NIS*)ixlog->ctx;
77
78 /*
79 * Ignore . and .. directory entries.
80 */
81 if (bstrcmp(raw_name, ".") || bstrcmp(raw_name, "..")) { return 0; }
82
83 if (nis->save_filehist) {
84 int result, length;
85 int total_length;
86 MDB_val key, data;
87 struct fhdb_payload* payload;
88 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
89
90 Dmsg3(100, "{ \"%s\", %llu , %llu},\n", raw_name, dir_node, node);
91 Dmsg3(100,
92 "bndmp_fhdb_lmdb_add_dir New dir \"%s\" - dirnode:[%llu] - "
93 "node:[%llu]\n",
94 raw_name, dir_node, node);
95
96 /*
97 * Make sure fhdb_state->pay_load is large enough.
98 */
99 length = strlen(raw_name);
100 total_length = sizeof(struct fhdb_payload) + length + 2;
101 fhdb_state->pay_load =
102 CheckPoolMemorySize(fhdb_state->pay_load, total_length);
103 payload = (struct fhdb_payload*)fhdb_state->pay_load;
104 payload->node = node;
105 payload->dir_node = dir_node;
106 memset(&payload->ndmp_fstat, 0, sizeof(ndmp9_file_stat));
107 memcpy(payload->namebuffer, raw_name, length + 1);
108
109 key.mv_data = &payload->node;
110 key.mv_size = sizeof(payload->node);
111
112 data.mv_data = payload;
113 data.mv_size = total_length;
114
115 retry:
116 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data, 0);
117 switch (result) {
118 case 0:
119 Dmsg3(debuglevel, "added file \"%s\", node=%llu, dir_node=%llu\n",
120 raw_name, payload->node, payload->dir_node);
121 break;
122 case MDB_TXN_FULL:
123 /*
124 * Seems we filled the transaction.
125 * Flush the current transaction start a new one and retry the put.
126 */
127 result = mdb_txn_commit(fhdb_state->db_rw_txn);
128 if (result == 0) {
129 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
130 &fhdb_state->db_rw_txn);
131 if (result == 0) {
132 goto retry;
133 } else {
134 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
135 mdb_strerror(result));
136 Jmsg1(nis->jcr, M_FATAL, 0,
137 _("Unable create new transaction: %s\n"),
138 mdb_strerror(result));
139 goto bail_out;
140 }
141 } else {
142 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
143 mdb_strerror(result));
144 Jmsg1(nis->jcr, M_FATAL, 0,
145 _("Unable to commit full transaction: %s\n"),
146 mdb_strerror(result));
147 goto bail_out;
148 }
149 break;
150 default:
151 Dmsg2(debuglevel, _("Unable to insert new data at %llu: %s\n"),
152 payload->node, mdb_strerror(result));
153 Jmsg2(nis->jcr, M_FATAL, 0, _("Unable insert new data at %llu: %s\n"),
154 payload->node, mdb_strerror(result));
155 goto bail_out;
156 }
157 }
158
159 return 0;
160
161 bail_out:
162 return 1;
163 }
164
bndmp_fhdb_lmdb_add_node(struct ndmlog * ixlog,int tagc,ndmp9_u_quad node,ndmp9_file_stat * ndmp_fstat)165 extern "C" int bndmp_fhdb_lmdb_add_node(struct ndmlog* ixlog,
166 int tagc,
167 ndmp9_u_quad node,
168 ndmp9_file_stat* ndmp_fstat)
169 {
170 NIS* nis;
171 nis = (NIS*)ixlog->ctx;
172
173 nis->jcr->lock();
174 nis->jcr->JobFiles++;
175 nis->jcr->unlock();
176
177 if (nis->save_filehist) {
178 int result;
179 MDB_val key, data;
180 struct fhdb_payload* payload;
181 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
182
183 Dmsg1(100, "{ NULL, %llu , 0},\n", node);
184 Dmsg1(debuglevel, "bndmp_fhdb_lmdb_add_node node:[%llu]\n", node);
185
186 /*
187 * Need to update which means we first get the existing data
188 * and update the fields and write back.
189 */
190 key.mv_data = &node;
191 key.mv_size = sizeof(node);
192
193 retry_get:
194 result = mdb_get(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data);
195 switch (result) {
196 case 0:
197 /*
198 * Make a copy of the current pay_load.
199 */
200 fhdb_state->pay_load =
201 CheckPoolMemorySize(fhdb_state->pay_load, data.mv_size);
202 memcpy(fhdb_state->pay_load, data.mv_data, data.mv_size);
203 payload = (struct fhdb_payload*)fhdb_state->pay_load;
204
205 /*
206 * Copy the new file statistics,
207 */
208 memcpy(&payload->ndmp_fstat, ndmp_fstat, sizeof(ndmp9_file_stat));
209
210 /*
211 * Keys and length don't change only content.
212 */
213 data.mv_data = payload;
214
215 retry_del:
216 result = mdb_del(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, NULL);
217 switch (result) {
218 case 0:
219 break;
220 case MDB_TXN_FULL:
221 /*
222 * Seems we filled the transaction.
223 * Flush the current transaction start a new one and retry the
224 * delete.
225 */
226 result = mdb_txn_commit(fhdb_state->db_rw_txn);
227 if (result == 0) {
228 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
229 &fhdb_state->db_rw_txn);
230 if (result == 0) {
231 goto retry_del;
232 } else {
233 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
234 mdb_strerror(result));
235 Jmsg1(nis->jcr, M_FATAL, 0,
236 _("Unable create new transaction: %s\n"),
237 mdb_strerror(result));
238 goto bail_out;
239 }
240 } else {
241 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
242 mdb_strerror(result));
243 Jmsg1(nis->jcr, M_FATAL, 0,
244 _("Unable to commit full transaction: %s\n"),
245 mdb_strerror(result));
246 goto bail_out;
247 }
248 break;
249 default:
250 Dmsg1(debuglevel, _("Unable delete old data: %s\n"),
251 mdb_strerror(result));
252 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable delete old data: %s\n"),
253 mdb_strerror(result));
254 goto bail_out;
255 }
256
257 retry_put:
258 /*
259 * Overwrite existing key
260 */
261 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data,
262 MDB_NODUPDATA);
263 switch (result) {
264 case 0:
265 break;
266 case MDB_TXN_FULL:
267 /*
268 * Seems we filled the transaction.
269 * Flush the current transaction start a new one and retry the put.
270 */
271 result = mdb_txn_commit(fhdb_state->db_rw_txn);
272 if (result == 0) {
273 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
274 &fhdb_state->db_rw_txn);
275 if (result == 0) {
276 goto retry_put;
277 } else {
278 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
279 mdb_strerror(result));
280 Jmsg1(nis->jcr, M_FATAL, 0,
281 _("Unable create new transaction: %s\n"),
282 mdb_strerror(result));
283 goto bail_out;
284 }
285 } else {
286 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
287 mdb_strerror(result));
288 Jmsg1(nis->jcr, M_FATAL, 0,
289 _("Unable to commit full transaction: %s\n"),
290 mdb_strerror(result));
291 goto bail_out;
292 }
293 break;
294 default:
295 Dmsg1(debuglevel, _("Unable put new data: %s\n"),
296 mdb_strerror(result));
297 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable put new data: %s\n"),
298 mdb_strerror(result));
299 goto bail_out;
300 }
301 break;
302 case MDB_TXN_FULL:
303 /*
304 * Seems we filled the transaction.
305 * Flush the current transaction start a new one and retry the put.
306 */
307 result = mdb_txn_commit(fhdb_state->db_rw_txn);
308 if (result == 0) {
309 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
310 &fhdb_state->db_rw_txn);
311 if (result == 0) {
312 goto retry_get;
313 } else {
314 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
315 mdb_strerror(result));
316 Jmsg1(nis->jcr, M_FATAL, 0,
317 _("Unable to create new transaction: %s\n"),
318 mdb_strerror(result));
319 goto bail_out;
320 }
321 } else {
322 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
323 mdb_strerror(result));
324 Jmsg1(nis->jcr, M_FATAL, 0,
325 _("Unable to commit full transaction: %s\n"),
326 mdb_strerror(result));
327 goto bail_out;
328 }
329 break;
330 default:
331 Dmsg1(debuglevel, _("Unable get old data: %s\n"), mdb_strerror(result));
332 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable get old data: %s\n"),
333 mdb_strerror(result));
334 goto bail_out;
335 }
336 }
337
338 return 0;
339
340 bail_out:
341 return 1;
342 }
343
bndmp_fhdb_lmdb_add_dirnode_root(struct ndmlog * ixlog,int tagc,ndmp9_u_quad root_node)344 extern "C" int bndmp_fhdb_lmdb_add_dirnode_root(struct ndmlog* ixlog,
345 int tagc,
346 ndmp9_u_quad root_node)
347 {
348 NIS* nis = (NIS*)ixlog->ctx;
349
350 if (nis->save_filehist) {
351 int result;
352 int total_length;
353 MDB_val key, data;
354 struct fhdb_payload* payload;
355 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
356
357 Dmsg1(100, "{ NULL, 0, %llu },\n", root_node);
358 Dmsg1(100, "bndmp_fhdb_lmdb_add_dirnode_root: New root node [%llu]\n",
359 root_node);
360
361 /*
362 * Make sure fhdb_state->pay_load is large enough.
363 */
364 total_length = sizeof(struct fhdb_payload);
365 fhdb_state->pay_load =
366 CheckPoolMemorySize(fhdb_state->pay_load, total_length);
367 payload = (struct fhdb_payload*)fhdb_state->pay_load;
368
369 fhdb_state->root_node = root_node;
370 payload->node = root_node;
371 payload->dir_node = root_node;
372 payload->namebuffer[0] = '\0';
373 memset(&payload->ndmp_fstat, 0, sizeof(payload->ndmp_fstat));
374
375 key.mv_data = &payload->node;
376 key.mv_size = sizeof(payload->node);
377
378 data.mv_data = payload;
379 data.mv_size = total_length;
380
381 retry:
382 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data,
383 MDB_NOOVERWRITE);
384 switch (result) {
385 case 0:
386 Dmsg1(100, "new rootnode=%llu\n", root_node);
387 break;
388 case MDB_TXN_FULL:
389 /*
390 * Seems we filled the transaction.
391 * Flush the current transaction start a new one and retry the put.
392 */
393 result = mdb_txn_commit(fhdb_state->db_rw_txn);
394 if (result == 0) {
395 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
396 &fhdb_state->db_rw_txn);
397 if (result == 0) {
398 goto retry;
399 } else {
400 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
401 mdb_strerror(result));
402 Jmsg1(nis->jcr, M_FATAL, 0,
403 _("Unable to create new transaction: %s\n"),
404 mdb_strerror(result));
405 goto bail_out;
406 }
407 } else {
408 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
409 mdb_strerror(result));
410 Jmsg1(nis->jcr, M_FATAL, 0,
411 _("Unable to commit full transaction: %s\n"),
412 mdb_strerror(result));
413 goto bail_out;
414 }
415 break;
416 default:
417 Dmsg1(debuglevel, _("Unable insert new data: %s\n"),
418 mdb_strerror(result));
419 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable insert new data: %s\n"),
420 mdb_strerror(result));
421 goto bail_out;
422 }
423 }
424
425 return 0;
426
427 bail_out:
428 return 1;
429 }
430
431 /**
432 * This glues the NDMP File Handle DB with internal code.
433 */
NdmpFhdbLmdbRegister(struct ndmlog * ixlog)434 void NdmpFhdbLmdbRegister(struct ndmlog* ixlog)
435 {
436 NIS* nis = (NIS*)ixlog->ctx;
437 struct ndm_fhdb_callbacks fhdb_callbacks;
438
439 /*
440 * Register the FileHandleDB callbacks.
441 */
442 fhdb_callbacks.add_file = BndmpFhdbAddFile;
443 fhdb_callbacks.add_dir = bndmp_fhdb_lmdb_add_dir;
444 fhdb_callbacks.add_node = bndmp_fhdb_lmdb_add_node;
445 fhdb_callbacks.add_dirnode_root = bndmp_fhdb_lmdb_add_dirnode_root;
446 ndmfhdb_register_callbacks(ixlog, &fhdb_callbacks);
447
448 Dmsg0(100, "NdmpFhdbLmdbRegister\n");
449
450 if (nis->save_filehist) {
451 int result;
452 ssize_t mapsize = 10485760;
453 NIS* nis = (NIS*)ixlog->ctx;
454 struct fhdb_state* fhdb_state;
455
456 /*
457 * Initiate LMDB environment
458 */
459 fhdb_state = (struct fhdb_state*)malloc(sizeof(struct fhdb_state));
460 memset(fhdb_state, 0, sizeof(struct fhdb_state));
461
462 fhdb_state->lmdb_name = GetPoolMemory(PM_FNAME);
463 fhdb_state->pay_load = GetPoolMemory(PM_MESSAGE);
464 fhdb_state->path = GetPoolMemory(PM_FNAME);
465
466 result = mdb_env_create(&fhdb_state->db_env);
467 if (result) {
468 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"),
469 mdb_strerror(result));
470 Dmsg1(debuglevel, _("Unable to create MDB environment: %s\n"),
471 mdb_strerror(result));
472 goto bail_out;
473 }
474
475 if ((nis->filehist_size * AVG_NR_BYTES_PER_ENTRY) > mapsize) {
476 size_t pagesize;
477
478 #ifdef HAVE_GETPAGESIZE
479 pagesize = getpagesize();
480 #else
481 pagesize = B_PAGE_SIZE;
482 #endif
483
484 mapsize =
485 (((nis->filehist_size * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1) *
486 pagesize;
487 }
488
489 result = mdb_env_set_mapsize(fhdb_state->db_env, mapsize);
490 if (result) {
491 Dmsg1(debuglevel, _("Unable to set MDB mapsize: %s\n"),
492 mdb_strerror(result));
493 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"),
494 mdb_strerror(result));
495 goto bail_out;
496 }
497
498 /*
499 * Explicitly set the number of readers to 1.
500 */
501 result = mdb_env_set_maxreaders(fhdb_state->db_env, 1);
502 if (result) {
503 Dmsg1(debuglevel, _("Unable to set MDB maxreaders: %s\n"),
504 mdb_strerror(result));
505 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"),
506 mdb_strerror(result));
507 goto bail_out;
508 }
509
510 Mmsg(fhdb_state->lmdb_name, "%s/.fhdb_lmdb.%d", working_directory,
511 nis->jcr->JobId);
512 result = mdb_env_open(fhdb_state->db_env, fhdb_state->lmdb_name,
513 MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600);
514 if (result != MDB_SUCCESS) {
515 Dmsg2(debuglevel,
516 _("Unable to create LMDB database %s: %s. Check OS ulimit settings "
517 "or adapt FileHistorySize\n"),
518 fhdb_state->lmdb_name, mdb_strerror(result));
519 Jmsg2(nis->jcr, M_FATAL, 0,
520 _("Unable to create LMDB database %s: %s. Check OS ulimit settings "
521 "or adapt FileHistorySize\n"),
522 fhdb_state->lmdb_name, mdb_strerror(result));
523 goto bail_out;
524 }
525
526 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &fhdb_state->db_rw_txn);
527 if (result != MDB_SUCCESS) {
528 Dmsg1(debuglevel, _("Unable to start a write transaction: %s\n"),
529 mdb_strerror(result));
530 Jmsg1(nis->jcr, M_FATAL, 0,
531 _("Unable to start a write transaction: %s\n"),
532 mdb_strerror(result));
533 goto bail_out;
534 }
535
536 result = mdb_dbi_open(fhdb_state->db_rw_txn, NULL,
537 MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT,
538 &fhdb_state->db_dbi);
539 if (result != MDB_SUCCESS) {
540 Dmsg1(debuglevel, _("Unable to open LMDB internal database: %s\n"),
541 mdb_strerror(result));
542 Jmsg1(nis->jcr, M_FATAL, 0,
543 _("Unable to open LMDB internal database: %s\n"),
544 mdb_strerror(result));
545 }
546
547 nis->fhdb_state = (void*)fhdb_state;
548
549 return;
550
551 bail_out:
552 if (fhdb_state->db_env) { mdb_env_close(fhdb_state->db_env); }
553
554 FreePoolMemory(fhdb_state->lmdb_name);
555 FreePoolMemory(fhdb_state->pay_load);
556 FreePoolMemory(fhdb_state->path);
557
558 free(fhdb_state);
559
560 return;
561 }
562 }
563
NdmpFhdbLmdbUnregister(struct ndmlog * ixlog)564 void NdmpFhdbLmdbUnregister(struct ndmlog* ixlog)
565 {
566 NIS* nis = (NIS*)ixlog->ctx;
567 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
568
569 ndmfhdb_unregister_callbacks(ixlog);
570
571 if (fhdb_state) {
572 /*
573 * Abort any pending write transaction.
574 */
575 if (fhdb_state->db_rw_txn) { mdb_txn_abort(fhdb_state->db_rw_txn); }
576
577 if (fhdb_state->db_env) {
578 /*
579 * Drop the contents of the LMDB.
580 */
581 if (fhdb_state->db_dbi) {
582 int result;
583 MDB_txn* txn;
584
585 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &txn);
586 if (result == 0) {
587 result = mdb_drop(txn, fhdb_state->db_dbi, 1);
588 if (result == 0) {
589 mdb_txn_commit(txn);
590 } else {
591 mdb_txn_abort(txn);
592 }
593 }
594 }
595
596 /*
597 * Close the environment.
598 */
599 mdb_env_close(fhdb_state->db_env);
600 }
601
602 FreePoolMemory(fhdb_state->pay_load);
603 FreePoolMemory(fhdb_state->path);
604 SecureErase(nis->jcr, fhdb_state->lmdb_name);
605 FreePoolMemory(fhdb_state->lmdb_name);
606
607 free(fhdb_state);
608 }
609 }
610
CalculatePath(uint64_t node,fhdb_state * fhdb_state)611 static inline void CalculatePath(uint64_t node, fhdb_state* fhdb_state)
612 {
613 PoolMem temp;
614 int result = 0;
615 MDB_val rkey, rdata;
616 struct fhdb_payload* payload;
617 bool root_node_reached = false;
618
619 Dmsg1(100, "CalculatePath for node %llu\n", node);
620
621 PmStrcpy(fhdb_state->path, "");
622 while (!result && !root_node_reached) {
623 rkey.mv_data = &node;
624 rkey.mv_size = sizeof(node);
625
626 result = mdb_get(fhdb_state->db_ro_txn, fhdb_state->db_dbi, &rkey, &rdata);
627 switch (result) {
628 case 0:
629 payload = (struct fhdb_payload*)rdata.mv_data;
630 if (node != fhdb_state->root_node) {
631 PmStrcpy(temp, "/");
632 PmStrcat(temp, payload->namebuffer);
633 PmStrcat(temp, fhdb_state->path);
634 PmStrcpy(fhdb_state->path, temp.c_str());
635 node = payload->dir_node;
636 } else {
637 /*
638 * Root reached
639 */
640 root_node_reached = true;
641 }
642 break;
643 case MDB_NOTFOUND:
644 break;
645 case EINVAL:
646 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
647 break;
648 default:
649 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
650 break;
651 }
652 }
653 }
654
ProcessLmdb(NIS * nis,struct fhdb_state * fhdb_state)655 static inline void ProcessLmdb(NIS* nis, struct fhdb_state* fhdb_state)
656 {
657 int result;
658 uint64_t node;
659 MDB_cursor* cursor;
660 int8_t FileType = 0;
661 MDB_val rkey, rdata;
662 PoolMem attribs(PM_FNAME);
663 ndmp9_file_stat ndmp_fstat;
664 PoolMem full_path(PM_FNAME);
665 struct fhdb_payload* payload;
666
667 result = mdb_cursor_open(fhdb_state->db_ro_txn, fhdb_state->db_dbi, &cursor);
668 if (result) { Dmsg1(debuglevel, "%s\n", mdb_strerror(result)); }
669
670 rkey.mv_data = &node;
671 rkey.mv_size = sizeof(node);
672 result = mdb_cursor_get(cursor, &rkey, &rdata, MDB_FIRST);
673 if (result) { Dmsg1(debuglevel, "%s\n", mdb_strerror(result)); }
674
675 while (!result) {
676 switch (result) {
677 case 0:
678 payload = (struct fhdb_payload*)rdata.mv_data;
679 ndmp_fstat = payload->ndmp_fstat;
680 node = *(uint64_t*)rkey.mv_data;
681
682 if (ndmp_fstat.node.valid == NDMP9_VALIDITY_VALID) {
683 CalculatePath(payload->dir_node, fhdb_state);
684 NdmpConvertFstat(&ndmp_fstat, nis->FileIndex, &FileType, attribs);
685
686 PmStrcpy(full_path, nis->filesystem);
687 PmStrcat(full_path, fhdb_state->path);
688 PmStrcat(full_path, "/");
689 PmStrcat(full_path, payload->namebuffer);
690
691 if (FileType == FT_DIREND) {
692 /*
693 * SplitPathAndFilename() expects directories to end with a '/'
694 * so append '/' if full_path does not already end with '/'
695 */
696 if (full_path.c_str()[strlen(full_path.c_str()) - 1] != '/') {
697 Dmsg1(100, ("appending / to %s \n"), full_path.c_str());
698 PmStrcat(full_path, "/");
699 }
700 }
701 NdmpStoreAttributeRecord(
702 nis->jcr, full_path.c_str(), nis->virtual_filename,
703 attribs.c_str(), FileType, 0,
704 (ndmp_fstat.fh_info.valid == NDMP9_VALIDITY_VALID)
705 ? ndmp_fstat.fh_info.value
706 : 0);
707 } else {
708 Dmsg1(100, "skipping node %lu because it has no valid node data\n",
709 node);
710 }
711 result = mdb_cursor_get(cursor, &rkey, &rdata, MDB_NEXT);
712 break;
713 default:
714 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
715 break;
716 }
717 }
718 }
719
NdmpFhdbLmdbProcessDb(struct ndmlog * ixlog)720 void NdmpFhdbLmdbProcessDb(struct ndmlog* ixlog)
721 {
722 NIS* nis = (NIS*)ixlog->ctx;
723 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
724
725 if (!fhdb_state) { return; }
726
727 Dmsg0(100, "NdmpFhdbLmdbProcessDb called\n");
728
729 /*
730 * Commit any pending write transactions.
731 */
732 if (fhdb_state->db_rw_txn) {
733 int result = mdb_txn_commit(fhdb_state->db_rw_txn);
734 if (result != MDB_SUCCESS) {
735 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable close write transaction: %s\n"),
736 mdb_strerror(result));
737 return;
738 }
739 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &fhdb_state->db_rw_txn);
740 if (result != MDB_SUCCESS) {
741 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create write transaction: %s\n"),
742 mdb_strerror(result));
743 return;
744 }
745 }
746
747 /*
748 * From now on we also will be doing read transactions so create a read
749 * transaction context.
750 */
751 int result = mdb_txn_begin(fhdb_state->db_env, NULL, MDB_RDONLY,
752 &fhdb_state->db_ro_txn);
753 if (result != MDB_SUCCESS) {
754 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create read transaction: %s\n"),
755 mdb_strerror(result));
756 return;
757 }
758
759 Jmsg(nis->jcr, M_INFO, 0, "Now processing lmdb database\n");
760
761 ProcessLmdb(nis, fhdb_state);
762
763 Jmsg(nis->jcr, M_INFO, 0, "Processing lmdb database done\n");
764 }
765
766 } /* namespace directordaemon */
767 #endif /* HAVE_LMDB */
768