1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2000-2011 Free Software Foundation Europe e.V.
5 Copyright (C) 2011-2016 Planets Communications B.V.
6 Copyright (C) 2013-2019 Bareos GmbH & Co. KG
7
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
11 in the file LICENSE.
12
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 02110-1301, USA.
22 */
23 /*
24 * Kern Sibbald, November MM
25 */
26 /**
27 * @file
28 * responsible for restoring files
29 *
30 * This routine is run as a separate thread.
31 *
32 * Current implementation is Catalog verification only (i.e. no verification
33 * versus tape).
34 *
35 * Basic tasks done here:
36 * Open DB
37 * Open Message Channel with Storage daemon to tell him a job will be
38 * starting. Open connection with File daemon and pass him commands to do the
39 * restore.
40 */
41
42
43 #include "include/bareos.h"
44 #include "dird.h"
45 #include "dird/dird_globals.h"
46 #include "dird/backup.h"
47 #include "dird/fd_cmds.h"
48 #include "dird/getmsg.h"
49 #include "dird/jcr_private.h"
50 #include "dird/job.h"
51 #include "dird/msgchan.h"
52 #include "dird/restore.h"
53 #include "dird/sd_cmds.h"
54 #include "dird/storage.h"
55 #include "include/protocol_types.h"
56 #include "lib/edit.h"
57 #include "lib/util.h"
58
59 namespace directordaemon {
60
61 /* Commands sent to File daemon */
62 static char restorecmd[] = "restore replace=%c prelinks=%d where=%s\n";
63 static char restorecmdR[] = "restore replace=%c prelinks=%d regexwhere=%s\n";
64 static char storaddrcmd[] =
65 "storage address=%s port=%d ssl=%d Authorization=%s\n";
66 static char setauthorizationcmd[] = "setauthorization Authorization=%s\n";
67 static char passiveclientcmd[] = "passive client address=%s port=%d ssl=%d\n";
68
69 /* Responses received from File daemon */
70 static char OKrestore[] = "2000 OK restore\n";
71 static char OKstore[] = "2000 OK storage\n";
72 static char OKstoreend[] = "2000 OK storage end\n";
73 static char OKAuthorization[] = "2000 OK Authorization\n";
74 static char OKpassiveclient[] = "2000 OK passive client\n";
75
76 /* Responses received from the Storage daemon */
77 static char OKbootstrap[] = "3000 OK bootstrap\n";
78
BuildRestoreCommand(JobControlRecord * jcr,PoolMem & ret)79 static void BuildRestoreCommand(JobControlRecord* jcr, PoolMem& ret)
80 {
81 char replace, *where, *cmd;
82 char empty = '\0';
83
84 /*
85 * Build the restore command
86 */
87 if (jcr->impl->replace != 0) {
88 replace = jcr->impl->replace;
89 } else if (jcr->impl->res.job->replace != 0) {
90 replace = jcr->impl->res.job->replace;
91 } else {
92 replace = REPLACE_ALWAYS; /* always replace */
93 }
94
95 if (jcr->RegexWhere) {
96 where = jcr->RegexWhere; /* override */
97 cmd = restorecmdR;
98 } else if (jcr->impl->res.job->RegexWhere) {
99 where = jcr->impl->res.job->RegexWhere; /* no override take from job */
100 cmd = restorecmdR;
101 } else if (jcr->where) {
102 where = jcr->where; /* override */
103 cmd = restorecmd;
104 } else if (jcr->impl->res.job->RestoreWhere) {
105 where = jcr->impl->res.job->RestoreWhere; /* no override take from job */
106 cmd = restorecmd;
107 } else { /* nothing was specified */
108 where = ∅ /* use default */
109 cmd = restorecmd;
110 }
111
112 jcr->prefix_links = jcr->impl->res.job->PrefixLinks;
113
114 BashSpaces(where);
115 Mmsg(ret, cmd, replace, jcr->prefix_links, where);
116 UnbashSpaces(where);
117 }
118
119 /**
120 * The bootstrap is stored in a file, so open the file, and loop
121 * through it processing each storage device in turn. If the
122 * storage is different from the prior one, we open a new connection
123 * to the new storage and do a restore for that part.
124 *
125 * This permits handling multiple storage daemons for a single
126 * restore. E.g. your Full is stored on tape, and Incrementals
127 * on disk.
128 */
DoNativeRestoreBootstrap(JobControlRecord * jcr)129 static inline bool DoNativeRestoreBootstrap(JobControlRecord* jcr)
130 {
131 StorageResource* store;
132 ClientResource* client;
133 bootstrap_info info;
134 BareosSocket* fd = NULL;
135 BareosSocket* sd = NULL;
136 bool first_time = true;
137 PoolMem RestoreCmd(PM_MESSAGE);
138 char* connection_target_address;
139
140 client = jcr->impl->res.client;
141 /*
142 * This command is used for each part
143 */
144 BuildRestoreCommand(jcr, RestoreCmd);
145
146 /*
147 * Open the bootstrap file
148 */
149 if (!OpenBootstrapFile(jcr, info)) { goto bail_out; }
150
151 /*
152 * Read the bootstrap file
153 */
154 jcr->passive_client = client->passive;
155 while (!feof(info.bs)) {
156 if (!SelectNextRstore(jcr, info)) { goto bail_out; }
157 store = jcr->impl->res.read_storage;
158
159 /**
160 * Open a message channel connection with the Storage
161 * daemon. This is to let him know that our client
162 * will be contacting him for a backup session.
163 *
164 */
165 Dmsg0(10, "Open connection with storage daemon\n");
166 jcr->setJobStatus(JS_WaitSD);
167
168 /*
169 * Start conversation with Storage daemon
170 */
171 if (!ConnectToStorageDaemon(jcr, 10, me->SDConnectTimeout, true)) {
172 goto bail_out;
173 }
174 sd = jcr->store_bsock;
175
176 /*
177 * Now start a job with the Storage daemon
178 */
179 if (!StartStorageDaemonJob(jcr, jcr->impl->res.read_storage_list, NULL)) {
180 goto bail_out;
181 }
182
183 if (first_time) {
184 /*
185 * Start conversation with File daemon
186 */
187 jcr->setJobStatus(JS_WaitFD);
188 jcr->impl->keep_sd_auth_key = true; /* don't clear the sd_auth_key now */
189
190 if (!ConnectToFileDaemon(jcr, 10, me->FDConnectTimeout, true)) {
191 goto bail_out;
192 }
193 SendJobInfoToFileDaemon(jcr);
194 fd = jcr->file_bsock;
195
196 if (!SendSecureEraseReqToFd(jcr)) {
197 Dmsg1(500, "Unexpected %s secure erase\n", "client");
198 }
199
200 /*
201 * Check if the file daemon supports passive client mode.
202 */
203 if (jcr->passive_client && jcr->impl->FDVersion < FD_VERSION_51) {
204 Jmsg(jcr, M_FATAL, 0,
205 _("Client \"%s\" doesn't support passive client mode. "
206 "Please upgrade your client or disable compat mode.\n"),
207 jcr->impl->res.client->resource_name_);
208 goto bail_out;
209 }
210 }
211
212 jcr->setJobStatus(JS_Running);
213
214 /*
215 * Send the bootstrap file -- what Volumes/files to restore
216 */
217 bool success = false;
218 if (SendBootstrapFile(jcr, sd, info)) {
219 Bmicrosleep(2, 0);
220 if (response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
221 success = true;
222 }
223 }
224 if (!success) { goto bail_out; }
225
226 if (!jcr->passive_client) {
227 /*
228 * When the client is not in passive mode we can put the SD in
229 * listen mode for the FD connection. And ask the FD to connect
230 * to the SD.
231 */
232 if (!sd->fsend("run")) { goto bail_out; }
233
234 /*
235 * Now start a Storage daemon message thread
236 */
237 if (!StartStorageDaemonMessageThread(jcr)) { goto bail_out; }
238 Dmsg0(50, "Storage daemon connection OK\n");
239
240 /*
241 * Send Storage daemon address to the File daemon,
242 * then wait for File daemon to make connection
243 * with Storage daemon.
244 */
245 if (store->SDDport == 0) { store->SDDport = store->SDport; }
246
247 /*
248 * TLS Requirement
249 */
250
251 TlsPolicy tls_policy;
252 if (jcr->impl->res.client->connection_successful_handshake_ !=
253 ClientConnectionHandshakeMode::kTlsFirst) {
254 tls_policy = store->GetPolicy();
255 } else {
256 tls_policy = store->IsTlsConfigured() ? TlsPolicy::kBnetTlsAuto
257 : TlsPolicy::kBnetTlsNone;
258 }
259
260 Dmsg1(200, "Tls Policy for active client is: %d\n", tls_policy);
261
262 connection_target_address = StorageAddressToContact(client, store);
263
264 fd->fsend(storaddrcmd, connection_target_address, store->SDDport,
265 tls_policy, jcr->sd_auth_key);
266 memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key));
267
268 Dmsg1(6, "dird>filed: %s", fd->msg);
269 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
270 goto bail_out;
271 }
272 } else {
273 /*
274 * In passive mode we tell the FD what authorization key to use
275 * and the ask the SD to initiate the connection.
276 */
277 fd->fsend(setauthorizationcmd, jcr->sd_auth_key);
278 memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key));
279
280 Dmsg1(6, "dird>filed: %s", fd->msg);
281 if (!response(jcr, fd, OKAuthorization, "Setauthorization",
282 DISPLAY_ERROR)) {
283 goto bail_out;
284 }
285
286 TlsPolicy tls_policy;
287
288 if (jcr->impl->res.client->connection_successful_handshake_ !=
289 ClientConnectionHandshakeMode::kTlsFirst) {
290 tls_policy = client->GetPolicy();
291 } else {
292 tls_policy = client->IsTlsConfigured() ? TlsPolicy::kBnetTlsAuto
293 : TlsPolicy::kBnetTlsNone;
294 }
295
296 Dmsg1(200, "Tls Policy for passive client is: %d\n", tls_policy);
297
298 connection_target_address = ClientAddressToContact(client, store);
299 /*
300 * Tell the SD to connect to the FD.
301 */
302 sd->fsend(passiveclientcmd, connection_target_address, client->FDport,
303 tls_policy);
304 Bmicrosleep(2, 0);
305 if (!response(jcr, sd, OKpassiveclient, "Passive client",
306 DISPLAY_ERROR)) {
307 goto bail_out;
308 }
309
310 /*
311 * Start the Job in the SD.
312 */
313 if (!sd->fsend("run")) { goto bail_out; }
314
315 /*
316 * Now start a Storage daemon message thread
317 */
318 if (!StartStorageDaemonMessageThread(jcr)) { goto bail_out; }
319 Dmsg0(50, "Storage daemon connection OK\n");
320 }
321
322 /*
323 * Declare the job started to start the MaxRunTime check
324 */
325 jcr->setJobStarted();
326
327 /*
328 * Only pass "global" commands to the FD once
329 */
330 if (first_time) {
331 first_time = false;
332 if (!SendRunscriptsCommands(jcr)) { goto bail_out; }
333
334 /*
335 * Only FD version 52 and later understand the sending of plugin options.
336 */
337 if (jcr->impl->FDVersion >= FD_VERSION_52) {
338 if (!SendPluginOptions(jcr)) {
339 Dmsg0(000, "FAIL: Send plugin options\n");
340 goto bail_out;
341 }
342 } else {
343 /*
344 * Plugin options specified and not a FD that understands the new
345 * protocol keyword.
346 */
347 if (jcr->impl->plugin_options) {
348 Jmsg(jcr, M_FATAL, 0,
349 _("Client \"%s\" doesn't support plugin option passing. "
350 "Please upgrade your client or disable compat mode.\n"),
351 jcr->impl->res.client->resource_name_);
352 goto bail_out;
353 }
354 }
355
356 if (!SendRestoreObjects(jcr, 0, true)) {
357 Dmsg0(000, "FAIL: Send restore objects\n");
358 goto bail_out;
359 }
360 }
361
362 fd->fsend("%s", RestoreCmd.c_str());
363
364 if (!response(jcr, fd, OKrestore, "Restore", DISPLAY_ERROR)) {
365 goto bail_out;
366 }
367
368 if (jcr->impl->FDVersion < FD_VERSION_2) { /* Old FD */
369 break; /* we do only one loop */
370 } else {
371 if (!response(jcr, fd, OKstoreend, "Store end", DISPLAY_ERROR)) {
372 goto bail_out;
373 }
374 WaitForStorageDaemonTermination(jcr);
375 }
376 } /* the whole boostrap has been send */
377
378 if (fd && jcr->impl->FDVersion >= FD_VERSION_2) { fd->fsend("endrestore"); }
379
380 CloseBootstrapFile(info);
381 return true;
382
383 bail_out:
384 if (jcr->file_bsock) {
385 jcr->file_bsock->signal(BNET_TERMINATE);
386 jcr->file_bsock->close();
387 delete jcr->file_bsock;
388 jcr->file_bsock = NULL;
389 }
390
391 CloseBootstrapFile(info);
392 return false;
393 }
394
395 /**
396 * Do a restore initialization.
397 *
398 * Returns: false on failure
399 * true on success
400 */
DoNativeRestoreInit(JobControlRecord * jcr)401 bool DoNativeRestoreInit(JobControlRecord* jcr)
402 {
403 FreeWstorage(jcr); /* we don't write */
404
405 return true;
406 }
407
408 /**
409 * Do a restore of the specified files
410 *
411 * Returns: false on failure
412 * true on success
413 */
DoNativeRestore(JobControlRecord * jcr)414 bool DoNativeRestore(JobControlRecord* jcr)
415 {
416 int status;
417
418 jcr->impl->jr.JobLevel = L_FULL; /* Full restore */
419 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
420 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
421 goto bail_out;
422 }
423 Dmsg0(20, "Updated job start record\n");
424
425 Dmsg1(20, "RestoreJobId=%d\n", jcr->impl->res.job->RestoreJobId);
426
427 if (!jcr->RestoreBootstrap) {
428 Jmsg(jcr, M_FATAL, 0,
429 _("Cannot restore without a bootstrap file.\n"
430 "You probably ran a restore job directly. All restore jobs must\n"
431 "be run using the restore command.\n"));
432 goto bail_out;
433 }
434
435 /*
436 * Print Job Start message
437 */
438 Jmsg(jcr, M_INFO, 0, _("Start Restore Job %s\n"), jcr->Job);
439
440 /*
441 * Read the bootstrap file and do the restore
442 */
443 if (!DoNativeRestoreBootstrap(jcr)) { goto bail_out; }
444
445 /*
446 * Wait for Job Termination
447 */
448 status = WaitForJobTermination(jcr);
449 NativeRestoreCleanup(jcr, status);
450 return true;
451
452 bail_out:
453 NativeRestoreCleanup(jcr, JS_ErrorTerminated);
454 return false;
455 }
456
457 /**
458 * Release resources allocated during restore.
459 */
NativeRestoreCleanup(JobControlRecord * jcr,int TermCode)460 void NativeRestoreCleanup(JobControlRecord* jcr, int TermCode)
461 {
462 char term_code[100];
463 const char* TermMsg;
464 int msg_type = M_INFO;
465
466 Dmsg0(20, "In NativeRestoreCleanup\n");
467 UpdateJobEnd(jcr, TermCode);
468
469 if (jcr->impl->unlink_bsr && jcr->RestoreBootstrap) {
470 SecureErase(jcr, jcr->RestoreBootstrap);
471 jcr->impl->unlink_bsr = false;
472 }
473
474 if (JobCanceled(jcr)) { CancelStorageDaemonJob(jcr); }
475
476 switch (TermCode) {
477 case JS_Terminated:
478 if (jcr->impl->ExpectedFiles > jcr->impl->jr.JobFiles) {
479 TermMsg = _("Restore OK -- warning file count mismatch");
480 } else {
481 TermMsg = _("Restore OK");
482 }
483 break;
484 case JS_Warnings:
485 TermMsg = _("Restore OK -- with warnings");
486 break;
487 case JS_FatalError:
488 case JS_ErrorTerminated:
489 TermMsg = _("*** Restore Error ***");
490 msg_type = M_ERROR; /* Generate error message */
491 if (jcr->store_bsock) {
492 jcr->store_bsock->signal(BNET_TERMINATE);
493 if (jcr->impl->SD_msg_chan_started) {
494 pthread_cancel(jcr->impl->SD_msg_chan);
495 }
496 }
497 break;
498 case JS_Canceled:
499 TermMsg = _("Restore Canceled");
500 if (jcr->store_bsock) {
501 jcr->store_bsock->signal(BNET_TERMINATE);
502 if (jcr->impl->SD_msg_chan_started) {
503 pthread_cancel(jcr->impl->SD_msg_chan);
504 }
505 }
506 break;
507 default:
508 TermMsg = term_code;
509 sprintf(term_code, _("Inappropriate term code: %c\n"), TermCode);
510 break;
511 }
512
513 GenerateRestoreSummary(jcr, msg_type, TermMsg);
514
515 Dmsg0(20, "Leaving NativeRestoreCleanup\n");
516 }
517
518 /*
519 * Generic function which generates a restore summary message.
520 * Used by:
521 * - NativeRestoreCleanup e.g. normal restores
522 * - NdmpRestoreCleanup e.g. NDMP restores
523 */
GenerateRestoreSummary(JobControlRecord * jcr,int msg_type,const char * TermMsg)524 void GenerateRestoreSummary(JobControlRecord* jcr,
525 int msg_type,
526 const char* TermMsg)
527 {
528 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
529 char ec1[30], ec2[30], ec3[30], elapsed[50];
530 char fd_term_msg[100], sd_term_msg[100];
531 utime_t RunTime;
532 double kbps;
533 PoolMem temp, secure_erase_status;
534
535 bstrftimes(sdt, sizeof(sdt), jcr->impl->jr.StartTime);
536 bstrftimes(edt, sizeof(edt), jcr->impl->jr.EndTime);
537 RunTime = jcr->impl->jr.EndTime - jcr->impl->jr.StartTime;
538 if (RunTime <= 0) {
539 kbps = 0;
540 } else {
541 kbps = ((double)jcr->impl->jr.JobBytes) / (1000.0 * (double)RunTime);
542 }
543 if (kbps < 0.05) { kbps = 0; }
544
545 JobstatusToAscii(jcr->impl->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
546 JobstatusToAscii(jcr->impl->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
547
548 switch (jcr->getJobProtocol()) {
549 case PT_NDMP_BAREOS:
550 case PT_NDMP_NATIVE:
551 Jmsg(jcr, msg_type, 0,
552 _("%s %s %s (%s):\n"
553 " Build OS: %s %s %s\n"
554 " JobId: %d\n"
555 " Job: %s\n"
556 " Restore Client: %s\n"
557 " Start time: %s\n"
558 " End time: %s\n"
559 " Elapsed time: %s\n"
560 " Files Expected: %s\n"
561 " Files Restored: %s\n"
562 " Bytes Restored: %s\n"
563 " Rate: %.1f KB/s\n"
564 " SD termination status: %s\n"
565 " Bareos binary info: %s\n"
566 " Termination: %s\n\n"),
567 BAREOS, my_name, kBareosVersionStrings.Full,
568 kBareosVersionStrings.ShortDate, HOST_OS, DISTNAME, DISTVER,
569 jcr->impl->jr.JobId, jcr->impl->jr.Job,
570 jcr->impl->res.client->resource_name_, sdt, edt,
571 edit_utime(RunTime, elapsed, sizeof(elapsed)),
572 edit_uint64_with_commas((uint64_t)jcr->impl->ExpectedFiles, ec1),
573 edit_uint64_with_commas((uint64_t)jcr->impl->jr.JobFiles, ec2),
574 edit_uint64_with_commas(jcr->impl->jr.JobBytes, ec3), (float)kbps,
575 sd_term_msg, kBareosVersionStrings.JoblogMessage, TermMsg);
576 break;
577 default:
578 if (me->secure_erase_cmdline) {
579 Mmsg(temp, " Dir Secure Erase Cmd: %s\n", me->secure_erase_cmdline);
580 PmStrcat(secure_erase_status, temp.c_str());
581 }
582 if (!bstrcmp(jcr->impl->FDSecureEraseCmd, "*None*")) {
583 Mmsg(temp, " FD Secure Erase Cmd: %s\n",
584 jcr->impl->FDSecureEraseCmd);
585 PmStrcat(secure_erase_status, temp.c_str());
586 }
587 if (!bstrcmp(jcr->impl->SDSecureEraseCmd, "*None*")) {
588 Mmsg(temp, " SD Secure Erase Cmd: %s\n",
589 jcr->impl->SDSecureEraseCmd);
590 PmStrcat(secure_erase_status, temp.c_str());
591 }
592
593 Jmsg(jcr, msg_type, 0,
594 _("%s %s %s (%s):\n"
595 " Build OS: %s %s %s\n"
596 " JobId: %d\n"
597 " Job: %s\n"
598 " Restore Client: %s\n"
599 " Start time: %s\n"
600 " End time: %s\n"
601 " Elapsed time: %s\n"
602 " Files Expected: %s\n"
603 " Files Restored: %s\n"
604 " Bytes Restored: %s\n"
605 " Rate: %.1f KB/s\n"
606 " FD Errors: %d\n"
607 " FD termination status: %s\n"
608 " SD termination status: %s\n"
609 "%s"
610 " Bareos binary info: %s\n"
611 " Termination: %s\n\n"),
612 BAREOS, my_name, kBareosVersionStrings.Full,
613 kBareosVersionStrings.ShortDate, HOST_OS, DISTNAME, DISTVER,
614 jcr->impl->jr.JobId, jcr->impl->jr.Job,
615 jcr->impl->res.client->resource_name_, sdt, edt,
616 edit_utime(RunTime, elapsed, sizeof(elapsed)),
617 edit_uint64_with_commas((uint64_t)jcr->impl->ExpectedFiles, ec1),
618 edit_uint64_with_commas((uint64_t)jcr->impl->jr.JobFiles, ec2),
619 edit_uint64_with_commas(jcr->impl->jr.JobBytes, ec3), (float)kbps,
620 jcr->JobErrors, fd_term_msg, sd_term_msg,
621 secure_erase_status.c_str(), kBareosVersionStrings.JoblogMessage,
622 TermMsg);
623 break;
624 }
625 }
626 } /* namespace directordaemon */
627