1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2011-2015 Planets Communications B.V.
5 Copyright (C) 2013-2019 Bareos GmbH & Co. KG
6
7 This program is Free Software; you can redistribute it and/or
8 modify it under the terms of version three of the GNU Affero General Public
9 License as published by the Free Software Foundation and included
10 in the file LICENSE.
11
12 This program is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Affero General Public License for more details.
16
17 You should have received a copy of the GNU Affero General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 02110-1301, USA.
21 */
22 /*
23 * Backup specific NDMP Data Management Application (DMA) routines
24 * for NDMP_NATIVE Backups
25 *
26 * Philipp Storz, April 2017
27 */
28
29 #include "include/bareos.h"
30 #include "dird.h"
31 #include "dird/jcr_private.h"
32 #include "dird/dird_globals.h"
33 #include "dird/job.h"
34 #include "dird/next_vol.h"
35 #include "dird/quota.h"
36 #include "dird/storage.h"
37
38 #include "lib/edit.h"
39
40 #if HAVE_NDMP
41 #include "dird/ndmp_dma_backup_common.h"
42 #include "dird/ndmp_dma_generic.h"
43 #include "dird/ndmp_dma_storage.h"
44
45 #define NDMP_NEED_ENV_KEYWORDS 1
46
47 #include "ndmp/ndmagents.h"
48 #include "ndmp_dma_priv.h"
49 #endif /* HAVE_NDMP */
50
51 namespace directordaemon {
52
53 #if HAVE_NDMP
54 /* Forward referenced functions */
55 static inline bool extract_post_backup_stats_ndmp_native(
56 JobControlRecord* jcr,
57 char* filesystem,
58 struct ndm_session* sess);
59
60
61 /**
62 * callback to check if job is cancelled and NDMP should stop
63 */
is_job_canceled_cb(struct ndm_session * sess)64 int is_job_canceled_cb(struct ndm_session* sess)
65 {
66 NIS* nis = (NIS*)sess->param->log.ctx;
67 JobControlRecord* jcr = nis->jcr;
68
69 return jcr->IsCanceled();
70 }
71
72
73 /**
74 * Load next medium, also used as "load first" callback
75 * find the next volume for append and add it to the
76 * media tab of the current ndmp job so that the next
77 * medium can be loaded.
78 *
79 * \return 0 if OK, -1 if NOT OK
80 */
81
NdmpLoadNext(struct ndm_session * sess)82 int NdmpLoadNext(struct ndm_session* sess)
83 {
84 NIS* nis = (NIS*)sess->param->log.ctx;
85 JobControlRecord* jcr = nis->jcr;
86 struct ndm_media_table* media_tab;
87 media_tab = &sess->control_acb->job.media_tab;
88 MediaDbRecord mr;
89 char* unwanted_volumes = (char*)"";
90 bool create = false;
91 bool prune = false;
92 struct ndmmedia* media;
93 int index = 1;
94 StorageResource* store = jcr->impl->res.write_storage;
95
96 /*
97 * get the poolid for pool name
98 */
99 mr.PoolId = jcr->impl->jr.PoolId;
100
101
102 if (FindNextVolumeForAppend(jcr, &mr, index, unwanted_volumes, create,
103 prune)) {
104 Jmsg(jcr, M_INFO, 0, _("Found volume for append: %s\n"), mr.VolumeName);
105
106 /*
107 * reserve medium so that it cannot be used by other job while we use it
108 */
109 bstrncpy(mr.VolStatus, NT_("Used"), sizeof(mr.VolStatus));
110
111
112 /*
113 * set FirstWritten Timestamp
114 */
115 mr.FirstWritten = (utime_t)time(NULL);
116
117 if (!jcr->db->UpdateMediaRecord(jcr, &mr)) {
118 Jmsg(jcr, M_FATAL, 0, _("Catalog error updating Media record. %s"),
119 jcr->db->strerror());
120 goto bail_out;
121 }
122
123 /*
124 * insert volume info in ndmmedia list
125 */
126 media = ndma_store_media(media_tab, mr.Slot);
127
128 bstrncpy(media->label, mr.VolumeName, NDMMEDIA_LABEL_MAX - 1);
129 media->valid_label = NDMP9_VALIDITY_VALID;
130
131
132 /*
133 * we want to append, so we need to skip what is already written
134 */
135 media->file_mark_offset = mr.VolFiles + 1;
136 media->valid_filemark = NDMP9_VALIDITY_VALID;
137
138 media->slot_addr = mr.Slot;
139
140 if (!NdmpUpdateStorageMappings(jcr, store)) {
141 Jmsg(jcr, M_ERROR, 0, _("ERROR in NdmpUpdateStorageMappings\n"));
142 goto bail_out;
143 }
144
145 slot_number_t slotnumber = GetElementAddressByBareosSlotNumber(
146 &store->runtime_storage_status->storage_mapping,
147 slot_type_t::kSlotTypeStorage, mr.Slot);
148 /* check for success */
149 if (!IsSlotNumberValid(slotnumber)) {
150 Jmsg(jcr, M_FATAL, 0, _("GetElementAddressByBareosSlotNumber failed\n"));
151 goto bail_out;
152 }
153
154 media->slot_addr = slotnumber;
155 media->valid_slot = NDMP9_VALIDITY_VALID;
156
157 ndmca_media_calculate_offsets(sess);
158
159 return 0;
160
161 } else {
162 bail_out:
163 Jmsg(jcr, M_INFO, 0, _("Error finding volume for append\n"));
164 return -1;
165 }
166 }
167
168 /*
169 * Run a NDMP backup session for NDMP_NATIVE backup
170 */
DoNdmpBackupNdmpNative(JobControlRecord * jcr)171 bool DoNdmpBackupNdmpNative(JobControlRecord* jcr)
172 {
173 int status;
174 char ed1[100];
175 NIS* nis = NULL;
176 FilesetResource* fileset;
177 struct ndm_job_param ndmp_job;
178 struct ndm_session ndmp_sess;
179 bool session_initialized = false;
180 bool retval = false;
181 uint32_t ndmp_log_level;
182
183 char* item;
184
185 ndmp_log_level =
186 std::max(jcr->impl->res.client->ndmp_loglevel, me->ndmp_loglevel);
187
188 struct ndmca_media_callbacks media_callbacks;
189
190 media_callbacks.load_first =
191 NdmpLoadNext; /* we use the same callback for first and next*/
192 media_callbacks.load_next = NdmpLoadNext;
193 media_callbacks.unload_current = NULL;
194
195 struct ndmca_jobcontrol_callbacks jobcontrol_callbacks;
196 jobcontrol_callbacks.is_job_canceled = is_job_canceled_cb;
197
198
199 /*
200 * Print Job Start message
201 */
202 Jmsg(jcr, M_INFO, 0, _("Start NDMP Backup JobId %s, Job=%s\n"),
203 edit_uint64(jcr->JobId, ed1), jcr->Job);
204
205 jcr->setJobStatus(JS_Running);
206 Dmsg2(100, "JobId=%d JobLevel=%c\n", jcr->impl->jr.JobId,
207 jcr->impl->jr.JobLevel);
208 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
209 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
210 return false;
211 }
212
213 status = 0;
214 StorageResource* store = jcr->impl->res.write_storage;
215 PoolMem virtual_filename(PM_FNAME);
216 IncludeExcludeItem* ie;
217
218
219 if (!NdmpBuildClientAndStorageJob(jcr, jcr->impl->res.write_storage,
220 jcr->impl->res.client,
221 true, /* init_tape */
222 true, /* init_robot */
223 NDM_JOB_OP_BACKUP, &ndmp_job)) {
224 goto bail_out;
225 }
226
227 if (!ndmp_native_setup_robot_and_tape_for_native_backup_job(jcr, store,
228 ndmp_job)) {
229 Jmsg(jcr, M_ERROR, 0,
230 _("ndmp_native_setup_robot_and_tape_for_native_backup_job failed\n"));
231 goto bail_out;
232 }
233
234 nis = (NIS*)malloc(sizeof(NIS));
235 memset(nis, 0, sizeof(NIS));
236
237 /*
238 * Only one include set of the fileset is allowed in NATIVE mode as
239 * in NDMP also per job only one filesystem can be backed up
240 */
241 fileset = jcr->impl->res.fileset;
242
243 if (fileset->include_items.size() > 1) {
244 Jmsg(jcr, M_ERROR, 0,
245 "Exactly one include set is supported in NDMP NATIVE mode\n");
246 return retval;
247 }
248
249 ie = fileset->include_items[0];
250
251 /*
252 * only one file = entry is allowed
253 * and it is the ndmp filesystem to be backed up
254 */
255 if (ie->name_list.size() != 1) {
256 Jmsg(jcr, M_ERROR, 0,
257 "Exactly one File specification is supported in NDMP NATIVE mode\n");
258 return retval;
259 }
260
261 item = (char*)ie->name_list.first();
262
263 /*
264 * Perform the actual NDMP job.
265 * Initialize a new NDMP session
266 */
267 memset(&ndmp_sess, 0, sizeof(ndmp_sess));
268 ndmp_sess.conn_snooping = (me->ndmp_snooping) ? 1 : 0;
269 ndmp_sess.control_agent_enabled = 1;
270
271 ndmp_sess.param =
272 (struct ndm_session_param*)malloc(sizeof(struct ndm_session_param));
273 memset(ndmp_sess.param, 0, sizeof(struct ndm_session_param));
274 ndmp_sess.param->log.deliver = NdmpLoghandler;
275 ndmp_sess.param->log_level =
276 NativeToNdmpLoglevel(ndmp_log_level, debug_level, nis);
277 nis->filesystem = item;
278 nis->FileIndex = 1;
279 nis->jcr = jcr;
280 nis->save_filehist = jcr->impl->res.job->SaveFileHist;
281 nis->filehist_size = jcr->impl->res.job->FileHistSize;
282
283 ndmp_sess.param->log.ctx = nis;
284 ndmp_sess.param->log_tag = strdup("DIR-NDMP");
285 ndmp_sess.dump_media_info = 1;
286
287 /*
288 * Initialize the session structure.
289 */
290 if (ndma_session_initialize(&ndmp_sess)) { goto cleanup; }
291 session_initialized = true;
292
293 /*
294 * Copy the actual job to perform.
295 */
296 memcpy(&ndmp_sess.control_acb->job, &ndmp_job, sizeof(struct ndm_job_param));
297
298 /*
299 * We can use the same private pointer used in the logging with the JCR in
300 * the file index generation. We don't setup a index_log.deliver
301 * function as we catch the index information via callbacks.
302 */
303 ndmp_sess.control_acb->job.index_log.ctx = ndmp_sess.param->log.ctx;
304
305 if (!FillBackupEnvironment(jcr, ie, nis->filesystem,
306 &ndmp_sess.control_acb->job)) {
307 goto cleanup;
308 }
309 /* register the callbacks */
310 ndmca_media_register_callbacks(&ndmp_sess, &media_callbacks);
311 ndmca_jobcontrol_register_callbacks(&ndmp_sess, &jobcontrol_callbacks);
312
313 /*
314 * The full ndmp archive has a virtual filename, we need it to hardlink the
315 * individual file records to it. So we allocate it here once so its available
316 * during the whole NDMP session.
317 */
318 if (Bstrcasecmp(jcr->impl->backup_format, "dump")) {
319 Mmsg(virtual_filename, "/@NDMP%s%%%d", nis->filesystem,
320 jcr->impl->DumpLevel);
321 } else {
322 Mmsg(virtual_filename, "/@NDMP%s", nis->filesystem);
323 }
324
325 if (nis->virtual_filename) { free(nis->virtual_filename); }
326 nis->virtual_filename = strdup(virtual_filename.c_str());
327
328 // FIXME: disabled because of "missing media entry" error
329 // if (!ndmp_validate_job(jcr, &ndmp_sess.control_acb->job)) {
330 // goto cleanup;
331 //}
332
333 /*
334 * Commission the session for a run.
335 */
336 if (ndma_session_commission(&ndmp_sess)) { goto cleanup; }
337
338 /*
339 * Setup the DMA.
340 */
341 if (ndmca_connect_control_agent(&ndmp_sess)) { goto cleanup; }
342
343 ndmp_sess.conn_open = 1;
344 ndmp_sess.conn_authorized = 1;
345
346 RegisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);
347
348 /*
349 * Let the DMA perform its magic.
350 */
351 if (ndmca_control_agent(&ndmp_sess) != 0) { goto cleanup; }
352
353 if (!unreserve_ndmp_tapedevice_for_job(store, jcr)) {
354 Jmsg(jcr, M_ERROR, 0, "could not free ndmp tape device %s from job %d",
355 ndmp_job.tape_device, jcr->JobId);
356 }
357
358 /*
359 * See if there were any errors during the backup.
360 */
361 jcr->impl->jr.FileIndex = 1;
362 if (!extract_post_backup_stats_ndmp_native(jcr, item, &ndmp_sess)) {
363 goto cleanup;
364 }
365 UnregisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);
366
367 /*
368 * Reset the NDMP session states.
369 */
370 ndma_session_decommission(&ndmp_sess);
371
372 /*
373 * Cleanup the job after it has run.
374 */
375 ndma_destroy_env_list(&ndmp_sess.control_acb->job.env_tab);
376 ndma_destroy_env_list(&ndmp_sess.control_acb->job.result_env_tab);
377 ndma_destroy_nlist(&ndmp_sess.control_acb->job.nlist_tab);
378
379 /*
380 * Destroy the session.
381 */
382 ndma_session_destroy(&ndmp_sess);
383
384 /*
385 * Free the param block.
386 */
387 free(ndmp_sess.param->log_tag);
388 free(ndmp_sess.param);
389 ndmp_sess.param = NULL;
390
391 /*
392 * Reset the initialized state so we don't try to cleanup again.
393 */
394 session_initialized = false;
395
396
397 status = JS_Terminated;
398 retval = true;
399
400
401 /*
402 * If we do incremental backups it can happen that the backup is empty if
403 * nothing changed but we always write a filestream. So we use the counter
404 * which counts the number of actual NDMP backup sessions we run to
405 * completion.
406 */
407 if (jcr->JobFiles == 0) { jcr->JobFiles = 1; }
408
409 /*
410 * Jump to the generic cleanup done for every Job.
411 */
412 goto ok_out;
413
414 cleanup:
415
416 if (!unreserve_ndmp_tapedevice_for_job(store, jcr)) {
417 Jmsg(jcr, M_ERROR, 0, "could not free ndmp tape device %s from job %d",
418 ndmp_job.tape_device, jcr->JobId);
419 }
420
421 /*
422 * Only need to cleanup when things are initialized.
423 */
424 if (session_initialized) {
425 ndma_destroy_env_list(&ndmp_sess.control_acb->job.env_tab);
426 ndma_destroy_env_list(&ndmp_sess.control_acb->job.result_env_tab);
427 ndma_destroy_nlist(&ndmp_sess.control_acb->job.nlist_tab);
428 /*
429 if (ndmp_sess.control_acb->job.tape_device) {
430 free(ndmp_sess.control_acb->job.tape_device);
431 }
432 */
433 UnregisterCallbackHooks(&ndmp_sess.control_acb->job.index_log);
434
435 /*
436 * Destroy the session.
437 */
438 ndma_session_destroy(&ndmp_sess);
439 }
440
441 /*
442 * Free the param block.
443 */
444 free(ndmp_sess.param->log_tag);
445 free(ndmp_sess.param);
446 ndmp_sess.param = NULL;
447
448 bail_out:
449 /*
450 * Error handling of failed Job.
451 */
452 status = JS_ErrorTerminated;
453 jcr->setJobStatus(JS_ErrorTerminated);
454
455 ok_out:
456 if (nis) {
457 if (nis->virtual_filename) { free(nis->virtual_filename); }
458 free(nis);
459 }
460
461 if (status == JS_Terminated) { NdmpBackupCleanup(jcr, status); }
462
463 return retval;
464 }
465
466 /*
467 * Setup a NDMP backup session.
468 */
DoNdmpBackupInitNdmpNative(JobControlRecord * jcr)469 bool DoNdmpBackupInitNdmpNative(JobControlRecord* jcr)
470 {
471 FreeRstorage(jcr); /* we don't read so release */
472
473 if (!AllowDuplicateJob(jcr)) { return false; }
474
475 jcr->impl->jr.PoolId =
476 GetOrCreatePoolRecord(jcr, jcr->impl->res.pool->resource_name_);
477 if (jcr->impl->jr.PoolId == 0) { return false; }
478
479 jcr->start_time = time(NULL);
480 jcr->impl->jr.StartTime = jcr->start_time;
481 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
482 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
483 return false;
484 }
485
486 /*
487 * If pool storage specified, use it instead of job storage
488 */
489 CopyWstorage(jcr, jcr->impl->res.pool->storage, _("Pool resource"));
490
491 if (!jcr->impl->res.write_storage_list) {
492 Jmsg(jcr, M_FATAL, 0,
493 _("No Storage specification found in Job or Pool.\n"));
494 return false;
495 }
496
497 /*
498 * Validate the Job to have a NDMP client and NDMP storage.
499 */
500 if (!NdmpValidateClient(jcr)) { return false; }
501
502 if (!NdmpValidateStorage(jcr)) { return false; }
503
504 return true;
505 }
506
507 /*
508 * Extract any post backup statistics for native NDMP
509 */
extract_post_backup_stats_ndmp_native(JobControlRecord * jcr,char * filesystem,struct ndm_session * sess)510 static inline bool extract_post_backup_stats_ndmp_native(
511 JobControlRecord* jcr,
512 char* filesystem,
513 struct ndm_session* sess)
514 {
515 bool retval = true;
516 struct ndmmedia* media;
517 ndmp_backup_format_option* nbf_options;
518 struct ndm_env_entry* ndm_ee;
519 char mediabuf[100];
520 /*
521 * See if we know this backup format and get it options.
522 */
523 nbf_options =
524 ndmp_lookup_backup_format_options(sess->control_acb->job.bu_type);
525
526 /*
527 * See if an error was raised during the backup session.
528 */
529 if (sess->error_raised) { return false; }
530
531 /*
532 * extract_post_backup_stats
533 */
534 for (media = sess->control_acb->job.media_tab.head; media;
535 media = media->next) {
536 /*
537 * translate Physical to Logical Slot before storing into database
538 */
539
540 media->slot_addr = GetBareosSlotNumberByElementAddress(
541 &jcr->impl->res.write_storage->runtime_storage_status->storage_mapping,
542 slot_type_t::kSlotTypeStorage, media->slot_addr);
543 #if 0
544 Jmsg(jcr, M_INFO, 0, _("Physical Slot is %d\n"), media->slot_addr);
545 Jmsg(jcr, M_INFO, 0, _("Logical slot is : %d\n"), media->slot_addr);
546 Jmsg(jcr, M_INFO, 0, _("label : %s\n"), media->label);
547 Jmsg(jcr, M_INFO, 0, _("index : %d\n"), media->index);
548 Jmsg(jcr, M_INFO, 0, _("n_bytes : %lld\n"), media->n_bytes);
549 Jmsg(jcr, M_INFO, 0, _("begin_offset : %u\n"), media->begin_offset);
550 Jmsg(jcr, M_INFO, 0, _("end_offset : %u\n"), media->end_offset);
551 #endif
552
553 StoreNdmmediaInfoInDatabase(media, jcr);
554
555 ndmmedia_to_str(media, mediabuf);
556
557 Jmsg(jcr, M_INFO, 0, _("Media: %s\n"), mediabuf);
558
559 /*
560 * See if there is any media error.
561 */
562 if (media->media_open_error || media->media_io_error ||
563 media->label_io_error || media->label_mismatch || media->fmark_error) {
564 retval = false;
565 }
566 }
567
568 /*
569 * Process the FHDB.
570 */
571 ProcessFhdb(&sess->control_acb->job.index_log);
572
573 /*
574 * insert batched files into database
575 */
576
577 jcr->db->WriteBatchFileRecords(jcr);
578 /*
579 * Update the Job statistics from the NDMP statistics.
580 */
581 jcr->JobBytes += sess->control_acb->job.bytes_written;
582
583 /*
584 * After a successful backup we need to store all NDMP ENV variables
585 * for doing a successful restore operation.
586 */
587 ndm_ee = sess->control_acb->job.result_env_tab.head;
588 while (ndm_ee) {
589 if (!jcr->db->CreateNdmpEnvironmentString(
590 jcr, &jcr->impl->jr, ndm_ee->pval.name, ndm_ee->pval.value)) {
591 break;
592 }
593 ndm_ee = ndm_ee->next;
594 }
595
596 /*
597 * If we are doing a backup type that uses dumplevels save the last used dump
598 * level.
599 */
600 if (nbf_options && nbf_options->uses_level) {
601 jcr->db->UpdateNdmpLevelMapping(jcr, &jcr->impl->jr, filesystem,
602 sess->control_acb->job.bu_level);
603 }
604
605 return retval;
606 }
607
608 #else
609
610 bool DoNdmpBackupInitNdmpNative(JobControlRecord* jcr)
611 {
612 Jmsg(jcr, M_FATAL, 0, _("NDMP protocol not supported\n"));
613 return false;
614 }
615
616 bool DoNdmpBackupNdmpNative(JobControlRecord* jcr)
617 {
618 Jmsg(jcr, M_FATAL, 0, _("NDMP protocol not supported\n"));
619 return false;
620 }
621
622 #endif /* HAVE_NDMP */
623 } /* namespace directordaemon */
624