1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2015-2015 Planets Communications B.V.
5 Copyright (C) 2015-2019 Bareos GmbH & Co. KG
6
7 This program is Free Software; you can redistribute it and/or
8 modify it under the terms of version three of the GNU Affero General Public
9 License as published by the Free Software Foundation and included
10 in the file LICENSE.
11
12 This program is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Affero General Public License for more details.
16
17 You should have received a copy of the GNU Affero General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 02110-1301, USA.
21 */
22 /*
23 * Marco van Wieringen & Philipp Storz, May 2015
24 */
25 /*
26 * @file
27 * FHDB using LMDB for NDMP Data Management Application (DMA)
28 */
29
30 #include "include/bareos.h"
31 #include "dird.h"
32
33 #if defined(HAVE_NDMP) && defined(HAVE_LMDB)
34
35 # include "ndmp/ndmagents.h"
36 # include "ndmp_dma_priv.h"
37 # include "lmdb/lmdb.h"
38
39 namespace directordaemon {
40
41 /*
42 * What is actually stored in the LMDB
43 */
44 struct fhdb_payload {
45 ndmp9_u_quad node;
46 ndmp9_u_quad dir_node;
47 ndmp9_file_stat ndmp_fstat;
48 char namebuffer[1];
49 };
50
51 struct fhdb_state {
52 uint64_t root_node;
53 POOLMEM* pay_load;
54 POOLMEM* lmdb_name;
55 POOLMEM* path;
56 MDB_env* db_env;
57 MDB_dbi db_dbi;
58 MDB_txn* db_rw_txn;
59 MDB_txn* db_ro_txn;
60 };
61
62 static int debuglevel = 100;
63
64 /*
65 * Payload = 8 + 8 + 96 = 112 bytes + namelength.
66 */
67 # define AVG_NR_BYTES_PER_ENTRY 256
68 # define B_PAGE_SIZE 4096
69
bndmp_fhdb_lmdb_add_dir(struct ndmlog * ixlog,int tagc,char * raw_name,ndmp9_u_quad dir_node,ndmp9_u_quad node)70 extern "C" int bndmp_fhdb_lmdb_add_dir(struct ndmlog* ixlog,
71 int tagc,
72 char* raw_name,
73 ndmp9_u_quad dir_node,
74 ndmp9_u_quad node)
75 {
76 NIS* nis = (NIS*)ixlog->ctx;
77
78 /*
79 * Ignore . and .. directory entries.
80 */
81 if (bstrcmp(raw_name, ".") || bstrcmp(raw_name, "..")) { return 0; }
82
83 if (nis->save_filehist) {
84 int result, length;
85 int total_length;
86 MDB_val key, data;
87 struct fhdb_payload* payload;
88 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
89
90 Dmsg3(100, "{ \"%s\", %llu , %llu},\n", raw_name, dir_node, node);
91 Dmsg3(100,
92 "bndmp_fhdb_lmdb_add_dir New dir \"%s\" - dirnode:[%llu] - "
93 "node:[%llu]\n",
94 raw_name, dir_node, node);
95
96 /*
97 * Make sure fhdb_state->pay_load is large enough.
98 */
99 length = strlen(raw_name);
100 total_length = sizeof(struct fhdb_payload) + length + 2;
101 fhdb_state->pay_load
102 = CheckPoolMemorySize(fhdb_state->pay_load, total_length);
103 payload = (struct fhdb_payload*)fhdb_state->pay_load;
104 payload->node = node;
105 payload->dir_node = dir_node;
106 memset(&payload->ndmp_fstat, 0, sizeof(ndmp9_file_stat));
107 memcpy(payload->namebuffer, raw_name, length + 1);
108
109 key.mv_data = &payload->node;
110 key.mv_size = sizeof(payload->node);
111
112 data.mv_data = payload;
113 data.mv_size = total_length;
114
115 retry:
116 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data, 0);
117 switch (result) {
118 case 0:
119 Dmsg3(debuglevel, "added file \"%s\", node=%llu, dir_node=%llu\n",
120 raw_name, payload->node, payload->dir_node);
121 break;
122 case MDB_TXN_FULL:
123 /*
124 * Seems we filled the transaction.
125 * Flush the current transaction start a new one and retry the put.
126 */
127 result = mdb_txn_commit(fhdb_state->db_rw_txn);
128 if (result == 0) {
129 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
130 &fhdb_state->db_rw_txn);
131 if (result == 0) {
132 goto retry;
133 } else {
134 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
135 mdb_strerror(result));
136 Jmsg1(nis->jcr, M_FATAL, 0,
137 _("Unable create new transaction: %s\n"),
138 mdb_strerror(result));
139 goto bail_out;
140 }
141 } else {
142 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
143 mdb_strerror(result));
144 Jmsg1(nis->jcr, M_FATAL, 0,
145 _("Unable to commit full transaction: %s\n"),
146 mdb_strerror(result));
147 goto bail_out;
148 }
149 break;
150 default:
151 Dmsg2(debuglevel, _("Unable to insert new data at %llu: %s\n"),
152 payload->node, mdb_strerror(result));
153 Jmsg2(nis->jcr, M_FATAL, 0, _("Unable insert new data at %llu: %s\n"),
154 payload->node, mdb_strerror(result));
155 goto bail_out;
156 }
157 }
158
159 return 0;
160
161 bail_out:
162 return 1;
163 }
164
bndmp_fhdb_lmdb_add_node(struct ndmlog * ixlog,int tagc,ndmp9_u_quad node,ndmp9_file_stat * ndmp_fstat)165 extern "C" int bndmp_fhdb_lmdb_add_node(struct ndmlog* ixlog,
166 int tagc,
167 ndmp9_u_quad node,
168 ndmp9_file_stat* ndmp_fstat)
169 {
170 NIS* nis;
171 nis = (NIS*)ixlog->ctx;
172
173 nis->jcr->lock();
174 nis->jcr->JobFiles++;
175 nis->jcr->unlock();
176
177 if (nis->save_filehist) {
178 int result;
179 MDB_val key, data;
180 struct fhdb_payload* payload;
181 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
182
183 Dmsg1(100, "{ NULL, %llu , 0},\n", node);
184 Dmsg1(debuglevel, "bndmp_fhdb_lmdb_add_node node:[%llu]\n", node);
185
186 /*
187 * Need to update which means we first get the existing data
188 * and update the fields and write back.
189 */
190 key.mv_data = &node;
191 key.mv_size = sizeof(node);
192
193 retry_get:
194 result = mdb_get(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data);
195 switch (result) {
196 case 0:
197 /*
198 * Make a copy of the current pay_load.
199 */
200 fhdb_state->pay_load
201 = CheckPoolMemorySize(fhdb_state->pay_load, data.mv_size);
202 memcpy(fhdb_state->pay_load, data.mv_data, data.mv_size);
203 payload = (struct fhdb_payload*)fhdb_state->pay_load;
204
205 /*
206 * Copy the new file statistics,
207 */
208 memcpy(&payload->ndmp_fstat, ndmp_fstat, sizeof(ndmp9_file_stat));
209
210 /*
211 * Keys and length don't change only content.
212 */
213 data.mv_data = payload;
214
215 retry_del:
216 result = mdb_del(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, NULL);
217 switch (result) {
218 case 0:
219 break;
220 case MDB_TXN_FULL:
221 /*
222 * Seems we filled the transaction.
223 * Flush the current transaction start a new one and retry the
224 * delete.
225 */
226 result = mdb_txn_commit(fhdb_state->db_rw_txn);
227 if (result == 0) {
228 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
229 &fhdb_state->db_rw_txn);
230 if (result == 0) {
231 goto retry_del;
232 } else {
233 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
234 mdb_strerror(result));
235 Jmsg1(nis->jcr, M_FATAL, 0,
236 _("Unable create new transaction: %s\n"),
237 mdb_strerror(result));
238 goto bail_out;
239 }
240 } else {
241 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
242 mdb_strerror(result));
243 Jmsg1(nis->jcr, M_FATAL, 0,
244 _("Unable to commit full transaction: %s\n"),
245 mdb_strerror(result));
246 goto bail_out;
247 }
248 break;
249 default:
250 Dmsg1(debuglevel, _("Unable delete old data: %s\n"),
251 mdb_strerror(result));
252 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable delete old data: %s\n"),
253 mdb_strerror(result));
254 goto bail_out;
255 }
256
257 retry_put:
258 /*
259 * Overwrite existing key
260 */
261 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data,
262 MDB_NODUPDATA);
263 switch (result) {
264 case 0:
265 break;
266 case MDB_TXN_FULL:
267 /*
268 * Seems we filled the transaction.
269 * Flush the current transaction start a new one and retry the put.
270 */
271 result = mdb_txn_commit(fhdb_state->db_rw_txn);
272 if (result == 0) {
273 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
274 &fhdb_state->db_rw_txn);
275 if (result == 0) {
276 goto retry_put;
277 } else {
278 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
279 mdb_strerror(result));
280 Jmsg1(nis->jcr, M_FATAL, 0,
281 _("Unable create new transaction: %s\n"),
282 mdb_strerror(result));
283 goto bail_out;
284 }
285 } else {
286 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
287 mdb_strerror(result));
288 Jmsg1(nis->jcr, M_FATAL, 0,
289 _("Unable to commit full transaction: %s\n"),
290 mdb_strerror(result));
291 goto bail_out;
292 }
293 break;
294 default:
295 Dmsg1(debuglevel, _("Unable put new data: %s\n"),
296 mdb_strerror(result));
297 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable put new data: %s\n"),
298 mdb_strerror(result));
299 goto bail_out;
300 }
301 break;
302 case MDB_TXN_FULL:
303 /*
304 * Seems we filled the transaction.
305 * Flush the current transaction start a new one and retry the put.
306 */
307 result = mdb_txn_commit(fhdb_state->db_rw_txn);
308 if (result == 0) {
309 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
310 &fhdb_state->db_rw_txn);
311 if (result == 0) {
312 goto retry_get;
313 } else {
314 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
315 mdb_strerror(result));
316 Jmsg1(nis->jcr, M_FATAL, 0,
317 _("Unable to create new transaction: %s\n"),
318 mdb_strerror(result));
319 goto bail_out;
320 }
321 } else {
322 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
323 mdb_strerror(result));
324 Jmsg1(nis->jcr, M_FATAL, 0,
325 _("Unable to commit full transaction: %s\n"),
326 mdb_strerror(result));
327 goto bail_out;
328 }
329 break;
330 default:
331 Dmsg1(debuglevel, _("Unable get old data: %s\n"), mdb_strerror(result));
332 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable get old data: %s\n"),
333 mdb_strerror(result));
334 goto bail_out;
335 }
336 }
337
338 return 0;
339
340 bail_out:
341 return 1;
342 }
343
bndmp_fhdb_lmdb_add_dirnode_root(struct ndmlog * ixlog,int tagc,ndmp9_u_quad root_node)344 extern "C" int bndmp_fhdb_lmdb_add_dirnode_root(struct ndmlog* ixlog,
345 int tagc,
346 ndmp9_u_quad root_node)
347 {
348 NIS* nis = (NIS*)ixlog->ctx;
349
350 if (nis->save_filehist) {
351 int result;
352 int total_length;
353 MDB_val key, data;
354 struct fhdb_payload* payload;
355 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
356
357 Dmsg1(100, "{ NULL, 0, %llu },\n", root_node);
358 Dmsg1(100, "bndmp_fhdb_lmdb_add_dirnode_root: New root node [%llu]\n",
359 root_node);
360
361 /*
362 * Make sure fhdb_state->pay_load is large enough.
363 */
364 total_length = sizeof(struct fhdb_payload);
365 fhdb_state->pay_load
366 = CheckPoolMemorySize(fhdb_state->pay_load, total_length);
367 payload = (struct fhdb_payload*)fhdb_state->pay_load;
368
369 fhdb_state->root_node = root_node;
370 payload->node = root_node;
371 payload->dir_node = root_node;
372 payload->namebuffer[0] = '\0';
373 memset(&payload->ndmp_fstat, 0, sizeof(payload->ndmp_fstat));
374
375 key.mv_data = &payload->node;
376 key.mv_size = sizeof(payload->node);
377
378 data.mv_data = payload;
379 data.mv_size = total_length;
380
381 retry:
382 result = mdb_put(fhdb_state->db_rw_txn, fhdb_state->db_dbi, &key, &data,
383 MDB_NOOVERWRITE);
384 switch (result) {
385 case 0:
386 Dmsg1(100, "new rootnode=%llu\n", root_node);
387 break;
388 case MDB_TXN_FULL:
389 /*
390 * Seems we filled the transaction.
391 * Flush the current transaction start a new one and retry the put.
392 */
393 result = mdb_txn_commit(fhdb_state->db_rw_txn);
394 if (result == 0) {
395 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0,
396 &fhdb_state->db_rw_txn);
397 if (result == 0) {
398 goto retry;
399 } else {
400 Dmsg1(debuglevel, _("Unable to create new transaction: %s\n"),
401 mdb_strerror(result));
402 Jmsg1(nis->jcr, M_FATAL, 0,
403 _("Unable to create new transaction: %s\n"),
404 mdb_strerror(result));
405 goto bail_out;
406 }
407 } else {
408 Dmsg1(debuglevel, _("Unable to commit full transaction: %s\n"),
409 mdb_strerror(result));
410 Jmsg1(nis->jcr, M_FATAL, 0,
411 _("Unable to commit full transaction: %s\n"),
412 mdb_strerror(result));
413 goto bail_out;
414 }
415 break;
416 default:
417 Dmsg1(debuglevel, _("Unable insert new data: %s\n"),
418 mdb_strerror(result));
419 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable insert new data: %s\n"),
420 mdb_strerror(result));
421 goto bail_out;
422 }
423 }
424
425 return 0;
426
427 bail_out:
428 return 1;
429 }
430
431 /**
432 * This glues the NDMP File Handle DB with internal code.
433 */
NdmpFhdbLmdbRegister(struct ndmlog * ixlog)434 void NdmpFhdbLmdbRegister(struct ndmlog* ixlog)
435 {
436 NIS* nis = (NIS*)ixlog->ctx;
437 struct ndm_fhdb_callbacks fhdb_callbacks;
438
439 /*
440 * Register the FileHandleDB callbacks.
441 */
442 fhdb_callbacks.add_file = BndmpFhdbAddFile;
443 fhdb_callbacks.add_dir = bndmp_fhdb_lmdb_add_dir;
444 fhdb_callbacks.add_node = bndmp_fhdb_lmdb_add_node;
445 fhdb_callbacks.add_dirnode_root = bndmp_fhdb_lmdb_add_dirnode_root;
446 ndmfhdb_register_callbacks(ixlog, &fhdb_callbacks);
447
448 Dmsg0(100, "NdmpFhdbLmdbRegister\n");
449
450 if (nis->save_filehist) {
451 int result;
452 ssize_t mapsize = 10485760;
453 NIS* nis = (NIS*)ixlog->ctx;
454 struct fhdb_state* fhdb_state;
455
456 /*
457 * Initiate LMDB environment
458 */
459 fhdb_state = (struct fhdb_state*)malloc(sizeof(struct fhdb_state));
460 memset(fhdb_state, 0, sizeof(struct fhdb_state));
461
462 fhdb_state->lmdb_name = GetPoolMemory(PM_FNAME);
463 fhdb_state->pay_load = GetPoolMemory(PM_MESSAGE);
464 fhdb_state->path = GetPoolMemory(PM_FNAME);
465
466 result = mdb_env_create(&fhdb_state->db_env);
467 if (result) {
468 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create MDB environment: %s\n"),
469 mdb_strerror(result));
470 Dmsg1(debuglevel, _("Unable to create MDB environment: %s\n"),
471 mdb_strerror(result));
472 goto bail_out;
473 }
474
475 if ((nis->filehist_size * AVG_NR_BYTES_PER_ENTRY) > mapsize) {
476 size_t pagesize;
477
478 # ifdef HAVE_GETPAGESIZE
479 pagesize = getpagesize();
480 # else
481 pagesize = B_PAGE_SIZE;
482 # endif
483
484 mapsize = (((nis->filehist_size * AVG_NR_BYTES_PER_ENTRY) / pagesize) + 1)
485 * pagesize;
486 }
487
488 result = mdb_env_set_mapsize(fhdb_state->db_env, mapsize);
489 if (result) {
490 Dmsg1(debuglevel, _("Unable to set MDB mapsize: %s\n"),
491 mdb_strerror(result));
492 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to set MDB mapsize: %s\n"),
493 mdb_strerror(result));
494 goto bail_out;
495 }
496
497 /*
498 * Explicitly set the number of readers to 1.
499 */
500 result = mdb_env_set_maxreaders(fhdb_state->db_env, 1);
501 if (result) {
502 Dmsg1(debuglevel, _("Unable to set MDB maxreaders: %s\n"),
503 mdb_strerror(result));
504 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to set MDB maxreaders: %s\n"),
505 mdb_strerror(result));
506 goto bail_out;
507 }
508
509 Mmsg(fhdb_state->lmdb_name, "%s/.fhdb_lmdb.%d", working_directory,
510 nis->jcr->JobId);
511 result = mdb_env_open(fhdb_state->db_env, fhdb_state->lmdb_name,
512 MDB_NOSUBDIR | MDB_NOLOCK | MDB_NOSYNC, 0600);
513 if (result != MDB_SUCCESS) {
514 Dmsg2(debuglevel,
515 _("Unable to create LMDB database %s: %s. Check OS ulimit settings "
516 "or adapt FileHistorySize\n"),
517 fhdb_state->lmdb_name, mdb_strerror(result));
518 Jmsg2(nis->jcr, M_FATAL, 0,
519 _("Unable to create LMDB database %s: %s. Check OS ulimit settings "
520 "or adapt FileHistorySize\n"),
521 fhdb_state->lmdb_name, mdb_strerror(result));
522 goto bail_out;
523 }
524
525 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &fhdb_state->db_rw_txn);
526 if (result != MDB_SUCCESS) {
527 Dmsg1(debuglevel, _("Unable to start a write transaction: %s\n"),
528 mdb_strerror(result));
529 Jmsg1(nis->jcr, M_FATAL, 0,
530 _("Unable to start a write transaction: %s\n"),
531 mdb_strerror(result));
532 goto bail_out;
533 }
534
535 result = mdb_dbi_open(fhdb_state->db_rw_txn, NULL,
536 MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT,
537 &fhdb_state->db_dbi);
538 if (result != MDB_SUCCESS) {
539 Dmsg1(debuglevel, _("Unable to open LMDB internal database: %s\n"),
540 mdb_strerror(result));
541 Jmsg1(nis->jcr, M_FATAL, 0,
542 _("Unable to open LMDB internal database: %s\n"),
543 mdb_strerror(result));
544 }
545
546 nis->fhdb_state = (void*)fhdb_state;
547
548 return;
549
550 bail_out:
551 if (fhdb_state->db_env) { mdb_env_close(fhdb_state->db_env); }
552
553 FreePoolMemory(fhdb_state->lmdb_name);
554 FreePoolMemory(fhdb_state->pay_load);
555 FreePoolMemory(fhdb_state->path);
556
557 free(fhdb_state);
558
559 return;
560 }
561 }
562
NdmpFhdbLmdbUnregister(struct ndmlog * ixlog)563 void NdmpFhdbLmdbUnregister(struct ndmlog* ixlog)
564 {
565 NIS* nis = (NIS*)ixlog->ctx;
566 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
567
568 ndmfhdb_unregister_callbacks(ixlog);
569
570 if (fhdb_state) {
571 /*
572 * Abort any pending write transaction.
573 */
574 if (fhdb_state->db_rw_txn) { mdb_txn_abort(fhdb_state->db_rw_txn); }
575
576 if (fhdb_state->db_env) {
577 /*
578 * Drop the contents of the LMDB.
579 */
580 if (fhdb_state->db_dbi) {
581 int result;
582 MDB_txn* txn;
583
584 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &txn);
585 if (result == 0) {
586 result = mdb_drop(txn, fhdb_state->db_dbi, 1);
587 if (result == 0) {
588 mdb_txn_commit(txn);
589 } else {
590 mdb_txn_abort(txn);
591 }
592 }
593 }
594
595 /*
596 * Close the environment.
597 */
598 mdb_env_close(fhdb_state->db_env);
599 }
600
601 FreePoolMemory(fhdb_state->pay_load);
602 FreePoolMemory(fhdb_state->path);
603 SecureErase(nis->jcr, fhdb_state->lmdb_name);
604 FreePoolMemory(fhdb_state->lmdb_name);
605
606 free(fhdb_state);
607 }
608 }
609
CalculatePath(uint64_t node,fhdb_state * fhdb_state)610 static inline void CalculatePath(uint64_t node, fhdb_state* fhdb_state)
611 {
612 PoolMem temp;
613 int result = 0;
614 MDB_val rkey, rdata;
615 struct fhdb_payload* payload;
616 bool root_node_reached = false;
617
618 Dmsg1(100, "CalculatePath for node %llu\n", node);
619
620 PmStrcpy(fhdb_state->path, "");
621 while (!result && !root_node_reached) {
622 rkey.mv_data = &node;
623 rkey.mv_size = sizeof(node);
624
625 result = mdb_get(fhdb_state->db_ro_txn, fhdb_state->db_dbi, &rkey, &rdata);
626 switch (result) {
627 case 0:
628 payload = (struct fhdb_payload*)rdata.mv_data;
629 if (node != fhdb_state->root_node) {
630 PmStrcpy(temp, "/");
631 PmStrcat(temp, payload->namebuffer);
632 PmStrcat(temp, fhdb_state->path);
633 PmStrcpy(fhdb_state->path, temp.c_str());
634 node = payload->dir_node;
635 } else {
636 /*
637 * Root reached
638 */
639 root_node_reached = true;
640 }
641 break;
642 case MDB_NOTFOUND:
643 break;
644 case EINVAL:
645 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
646 break;
647 default:
648 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
649 break;
650 }
651 }
652 }
653
ProcessLmdb(NIS * nis,struct fhdb_state * fhdb_state)654 static inline void ProcessLmdb(NIS* nis, struct fhdb_state* fhdb_state)
655 {
656 int result;
657 uint64_t node;
658 MDB_cursor* cursor;
659 int8_t FileType = 0;
660 MDB_val rkey, rdata;
661 PoolMem attribs(PM_FNAME);
662 ndmp9_file_stat ndmp_fstat;
663 PoolMem full_path(PM_FNAME);
664 struct fhdb_payload* payload;
665
666 result = mdb_cursor_open(fhdb_state->db_ro_txn, fhdb_state->db_dbi, &cursor);
667 if (result) { Dmsg1(debuglevel, "%s\n", mdb_strerror(result)); }
668
669 rkey.mv_data = &node;
670 rkey.mv_size = sizeof(node);
671 result = mdb_cursor_get(cursor, &rkey, &rdata, MDB_FIRST);
672 if (result) { Dmsg1(debuglevel, "%s\n", mdb_strerror(result)); }
673
674 while (!result) {
675 switch (result) {
676 case 0:
677 payload = (struct fhdb_payload*)rdata.mv_data;
678 ndmp_fstat = payload->ndmp_fstat;
679 node = *(uint64_t*)rkey.mv_data;
680
681 if (ndmp_fstat.node.valid == NDMP9_VALIDITY_VALID) {
682 CalculatePath(payload->dir_node, fhdb_state);
683 NdmpConvertFstat(&ndmp_fstat, nis->FileIndex, &FileType, attribs);
684
685 PmStrcpy(full_path, nis->filesystem);
686 PmStrcat(full_path, fhdb_state->path);
687 PmStrcat(full_path, "/");
688 PmStrcat(full_path, payload->namebuffer);
689
690 if (FileType == FT_DIREND) {
691 /*
692 * SplitPathAndFilename() expects directories to end with a '/'
693 * so append '/' if full_path does not already end with '/'
694 */
695 if (full_path.c_str()[strlen(full_path.c_str()) - 1] != '/') {
696 Dmsg1(100, ("appending / to %s \n"), full_path.c_str());
697 PmStrcat(full_path, "/");
698 }
699 }
700 NdmpStoreAttributeRecord(
701 nis->jcr, full_path.c_str(), nis->virtual_filename,
702 attribs.c_str(), FileType, 0,
703 (ndmp_fstat.fh_info.valid == NDMP9_VALIDITY_VALID)
704 ? ndmp_fstat.fh_info.value
705 : 0);
706 } else {
707 Dmsg1(100, "skipping node %lu because it has no valid node data\n",
708 node);
709 }
710 result = mdb_cursor_get(cursor, &rkey, &rdata, MDB_NEXT);
711 break;
712 default:
713 Dmsg1(debuglevel, "%s\n", mdb_strerror(result));
714 break;
715 }
716 }
717 }
718
NdmpFhdbLmdbProcessDb(struct ndmlog * ixlog)719 void NdmpFhdbLmdbProcessDb(struct ndmlog* ixlog)
720 {
721 NIS* nis = (NIS*)ixlog->ctx;
722 struct fhdb_state* fhdb_state = (struct fhdb_state*)nis->fhdb_state;
723
724 if (!fhdb_state) { return; }
725
726 Dmsg0(100, "NdmpFhdbLmdbProcessDb called\n");
727
728 /*
729 * Commit any pending write transactions.
730 */
731 if (fhdb_state->db_rw_txn) {
732 int result = mdb_txn_commit(fhdb_state->db_rw_txn);
733 if (result != MDB_SUCCESS) {
734 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable close write transaction: %s\n"),
735 mdb_strerror(result));
736 return;
737 }
738 result = mdb_txn_begin(fhdb_state->db_env, NULL, 0, &fhdb_state->db_rw_txn);
739 if (result != MDB_SUCCESS) {
740 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create write transaction: %s\n"),
741 mdb_strerror(result));
742 return;
743 }
744 }
745
746 /*
747 * From now on we also will be doing read transactions so create a read
748 * transaction context.
749 */
750 int result = mdb_txn_begin(fhdb_state->db_env, NULL, MDB_RDONLY,
751 &fhdb_state->db_ro_txn);
752 if (result != MDB_SUCCESS) {
753 Jmsg1(nis->jcr, M_FATAL, 0, _("Unable to create read transaction: %s\n"),
754 mdb_strerror(result));
755 return;
756 }
757
758 Jmsg(nis->jcr, M_INFO, 0, "Now processing lmdb database\n");
759
760 ProcessLmdb(nis, fhdb_state);
761
762 Jmsg(nis->jcr, M_INFO, 0, "Processing lmdb database done\n");
763 }
764
765 } /* namespace directordaemon */
766 #endif /* HAVE_LMDB */
767