1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2008-2012 Free Software Foundation Europe e.V.
5 Copyright (C) 2011-2016 Planets Communications B.V.
6 Copyright (C) 2013-2019 Bareos GmbH & Co. KG
7
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
11 in the file LICENSE.
12
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 02110-1301, USA.
22 */
23 /*
24 * Kern Sibbald, July MMVIII
25 */
26 /* @file
27 * responsible for doing virtual backup jobs or
28 * in other words, consolidation or synthetic backups.
29 *
30 * Basic tasks done here:
31 * * Open DB and create records for this job.
32 * * Figure out what Jobs to copy.
33 * * Open Message Channel with Storage daemon to tell him a job will be
34 * starting.
35 * * Open connection with File daemon and pass him commands to do the backup.
36 * * When the File daemon finishes the job, update the DB.
37 */
38
39 #include "include/bareos.h"
40 #include "dird.h"
41 #include "dird/dird_globals.h"
42 #include "dird/backup.h"
43 #include "dird/bsr.h"
44 #include "dird/jcr_private.h"
45 #include "dird/job.h"
46 #include "dird/migration.h"
47 #include "dird/msgchan.h"
48 #include "dird/sd_cmds.h"
49 #include "dird/storage.h"
50 #include "dird/ua_server.h"
51 #include "dird/ua_purge.h"
52 #include "dird/vbackup.h"
53 #include "lib/edit.h"
54 #include "lib/util.h"
55 #include "include/make_unique.h"
56
57 namespace directordaemon {
58
59 static const int dbglevel = 10;
60
61 static bool CreateBootstrapFile(JobControlRecord* jcr, char* jobids);
62
63 /**
64 * Called here before the job is run to do the job specific setup.
65 */
DoNativeVbackupInit(JobControlRecord * jcr)66 bool DoNativeVbackupInit(JobControlRecord* jcr)
67 {
68 const char* storage_source;
69
70 if (!GetOrCreateFilesetRecord(jcr)) {
71 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
72 return false;
73 }
74
75 ApplyPoolOverrides(jcr);
76
77 if (!AllowDuplicateJob(jcr)) { return false; }
78
79 jcr->impl->jr.PoolId =
80 GetOrCreatePoolRecord(jcr, jcr->impl->res.pool->resource_name_);
81 if (jcr->impl->jr.PoolId == 0) {
82 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
83 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
84 return false;
85 }
86
87 /*
88 * Note, at this point, pool is the pool for this job.
89 * We transfer it to rpool (read pool), and a bit later,
90 * pool will be changed to point to the write pool,
91 * which comes from pool->NextPool.
92 */
93 jcr->impl->res.rpool = jcr->impl->res.pool; /* save read pool */
94 PmStrcpy(jcr->impl->res.rpool_source, jcr->impl->res.pool_source);
95
96 /*
97 * If pool storage specified, use it for restore
98 */
99 CopyRstorage(jcr, jcr->impl->res.pool->storage, _("Pool resource"));
100
101 Dmsg2(dbglevel, "Read pool=%s (From %s)\n",
102 jcr->impl->res.rpool->resource_name_, jcr->impl->res.rpool_source);
103
104 jcr->start_time = time(NULL);
105 jcr->impl->jr.StartTime = jcr->start_time;
106 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
107 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
108 }
109
110 /*
111 * See if there is a next pool override.
112 */
113 if (jcr->impl->res.run_next_pool_override) {
114 PmStrcpy(jcr->impl->res.npool_source, _("Run NextPool override"));
115 PmStrcpy(jcr->impl->res.pool_source, _("Run NextPool override"));
116 storage_source = _("Storage from Run NextPool override");
117 } else {
118 /*
119 * See if there is a next pool override in the Job definition.
120 */
121 if (jcr->impl->res.job->next_pool) {
122 jcr->impl->res.next_pool = jcr->impl->res.job->next_pool;
123 PmStrcpy(jcr->impl->res.npool_source, _("Job's NextPool resource"));
124 PmStrcpy(jcr->impl->res.pool_source, _("Job's NextPool resource"));
125 storage_source = _("Storage from Job's NextPool resource");
126 } else {
127 /*
128 * Fall back to the pool's NextPool definition.
129 */
130 jcr->impl->res.next_pool = jcr->impl->res.pool->NextPool;
131 PmStrcpy(jcr->impl->res.npool_source, _("Job Pool's NextPool resource"));
132 PmStrcpy(jcr->impl->res.pool_source, _("Job Pool's NextPool resource"));
133 storage_source = _("Storage from Pool's NextPool resource");
134 }
135 }
136
137 /*
138 * If the original backup pool has a NextPool, make sure a
139 * record exists in the database. Note, in this case, we
140 * will be migrating from pool to pool->NextPool.
141 */
142 if (jcr->impl->res.next_pool) {
143 jcr->impl->jr.PoolId =
144 GetOrCreatePoolRecord(jcr, jcr->impl->res.next_pool->resource_name_);
145 if (jcr->impl->jr.PoolId == 0) { return false; }
146 }
147
148 if (!SetMigrationWstorage(jcr, jcr->impl->res.pool,
149 jcr->impl->res.next_pool, storage_source)) {
150 return false;
151 }
152
153 jcr->impl->res.pool = jcr->impl->res.next_pool;
154
155 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n",
156 jcr->impl->res.pool->resource_name_,
157 jcr->impl->res.rpool->resource_name_);
158
159 // CreateClones(jcr);
160
161 return true;
162 }
163
164 /**
165 * Do a virtual backup, which consolidates all previous backups into a sort of
166 * synthetic Full.
167 *
168 * Returns: false on failure
169 * true on success
170 */
DoNativeVbackup(JobControlRecord * jcr)171 bool DoNativeVbackup(JobControlRecord* jcr)
172 {
173 char* p;
174 BareosSocket* sd;
175 char* jobids;
176 char ed1[100];
177 int JobLevel_of_first_job;
178
179 if (!jcr->impl->res.read_storage_list) {
180 Jmsg(jcr, M_FATAL, 0, _("No storage for reading given.\n"));
181 return false;
182 }
183
184 if (!jcr->impl->res.write_storage_list) {
185 Jmsg(jcr, M_FATAL, 0, _("No storage for writing given.\n"));
186 return false;
187 }
188
189 Dmsg2(100, "read_storage_list=%p write_storage_list=%p\n",
190 jcr->impl->res.read_storage_list, jcr->impl->res.write_storage_list);
191 Dmsg2(100, "Read store=%s, write store=%s\n",
192 ((StorageResource*)jcr->impl->res.read_storage_list->first())
193 ->resource_name_,
194 ((StorageResource*)jcr->impl->res.write_storage_list->first())
195 ->resource_name_);
196
197 /*
198 * Print Job Start message
199 */
200 Jmsg(jcr, M_INFO, 0, _("Start Virtual Backup JobId %s, Job=%s\n"),
201 edit_uint64(jcr->JobId, ed1), jcr->Job);
202
203 if (!jcr->accurate) {
204 Jmsg(jcr, M_WARNING, 0,
205 _("This Job is not an Accurate backup so is not equivalent to a Full "
206 "backup.\n"));
207 }
208
209 /*
210 * See if we already got a list of jobids to use.
211 */
212 if (jcr->impl->vf_jobids) {
213 Dmsg1(10, "jobids=%s\n", jcr->impl->vf_jobids);
214 jobids = strdup(jcr->impl->vf_jobids);
215
216 } else {
217 db_list_ctx jobids_ctx;
218 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
219 Dmsg1(10, "consolidate candidates: %s.\n", jobids_ctx.list);
220
221 if (jobids_ctx.count == 0) {
222 Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n"));
223 return false;
224 }
225
226 jobids = strdup(jobids_ctx.list);
227 }
228
229 Jmsg(jcr, M_INFO, 0, _("Consolidating JobIds %s\n"), jobids);
230
231 /*
232 * Find oldest Jobid, get the db record and find its level
233 */
234 p = strchr(jobids, ','); /* find oldest jobid */
235 if (p) { *p = '\0'; }
236 jcr->impl->previous_jr = JobDbRecord{};
237 jcr->impl->previous_jr.JobId = str_to_int64(jobids);
238 Dmsg1(10, "Previous JobId=%s\n", jobids);
239
240 /*
241 * See if we need to restore the stripped ','
242 */
243 if (p) { *p = ','; }
244
245 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->previous_jr)) {
246 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for first Job: ERR=%s"),
247 jcr->db->strerror());
248 goto bail_out;
249 }
250
251 JobLevel_of_first_job = jcr->impl->previous_jr.JobLevel;
252 Dmsg2(10, "Level of first consolidated job %d: %s\n",
253 jcr->impl->previous_jr.JobId, job_level_to_str(JobLevel_of_first_job));
254
255 /*
256 * Now we find the newest job that ran and store its info in
257 * the previous_jr record. We will set our times to the
258 * values from that job so that anything changed after that
259 * time will be picked up on the next backup.
260 */
261 p = strrchr(jobids, ','); /* find newest jobid */
262 if (p) {
263 p++;
264 } else {
265 p = jobids;
266 }
267
268 jcr->impl->previous_jr = JobDbRecord{};
269 jcr->impl->previous_jr.JobId = str_to_int64(p);
270 Dmsg1(10, "Previous JobId=%s\n", p);
271
272 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->previous_jr)) {
273 Jmsg(jcr, M_FATAL, 0,
274 _("Error getting Job record for previous Job: ERR=%s"),
275 jcr->db->strerror());
276 goto bail_out;
277 }
278
279 if (!CreateBootstrapFile(jcr, jobids)) {
280 Jmsg(jcr, M_FATAL, 0, _("Could not create bootstrap file\n"));
281 goto bail_out;
282 }
283
284 /*
285 * Open a message channel connection with the Storage
286 * daemon.
287 */
288 Dmsg0(110, "Open connection with storage daemon\n");
289 jcr->setJobStatus(JS_WaitSD);
290
291 /*
292 * Start conversation with Storage daemon
293 */
294 if (!ConnectToStorageDaemon(jcr, 10, me->SDConnectTimeout, true)) {
295 goto bail_out;
296 }
297 sd = jcr->store_bsock;
298
299 /*
300 * Now start a job with the Storage daemon
301 */
302 if (!StartStorageDaemonJob(jcr, jcr->impl->res.read_storage_list,
303 jcr->impl->res.write_storage_list,
304 /* send_bsr */ true)) {
305 goto bail_out;
306 }
307 Dmsg0(100, "Storage daemon connection OK\n");
308
309 /*
310 * We re-update the job start record so that the start
311 * time is set after the run before job. This avoids
312 * that any files created by the run before job will
313 * be saved twice. They will be backed up in the current
314 * job, but not in the next one unless they are changed.
315 * Without this, they will be backed up in this job and
316 * in the next job run because in that case, their date
317 * is after the start of this run.
318 */
319 jcr->start_time = time(NULL);
320 jcr->impl->jr.StartTime = jcr->start_time;
321 jcr->impl->jr.JobTDate = jcr->start_time;
322 jcr->setJobStatus(JS_Running);
323
324 /*
325 * Update job start record
326 */
327 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->impl->jr)) {
328 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
329 goto bail_out;
330 }
331
332 /*
333 * Declare the job started to start the MaxRunTime check
334 */
335 jcr->setJobStarted();
336
337 /*
338 * Start the job prior to starting the message thread below
339 * to avoid two threads from using the BareosSocket structure at
340 * the same time.
341 */
342 if (!sd->fsend("run")) { goto bail_out; }
343
344 /*
345 * Now start a Storage daemon message thread
346 */
347 if (!StartStorageDaemonMessageThread(jcr)) { goto bail_out; }
348
349 jcr->setJobStatus(JS_Running);
350
351 /*
352 * Pickup Job termination data
353 * Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors
354 */
355 WaitForStorageDaemonTermination(jcr);
356 jcr->setJobStatus(jcr->impl->SDJobStatus);
357 jcr->db_batch->WriteBatchFileRecords(
358 jcr); /* used by bulk batch file insert */
359 if (!jcr->is_JobStatus(JS_Terminated)) { goto bail_out; }
360
361 NativeVbackupCleanup(jcr, jcr->JobStatus, JobLevel_of_first_job);
362
363 /*
364 * Remove the successfully consolidated jobids from the database
365 */
366 if (jcr->impl->res.job->AlwaysIncremental &&
367 jcr->impl->res.job->AlwaysIncrementalJobRetention) {
368 UaContext* ua;
369 ua = new_ua_context(jcr);
370 PurgeJobsFromCatalog(ua, jobids);
371 Jmsg(jcr, M_INFO, 0,
372 _("purged JobIds %s as they were consolidated into Job %s\n"), jobids,
373 edit_uint64(jcr->JobId, ed1));
374 }
375
376 free(jobids);
377 return true;
378
379 bail_out:
380 free(jobids);
381 return false;
382 }
383
384 /**
385 * Release resources allocated during backup.
386 */
NativeVbackupCleanup(JobControlRecord * jcr,int TermCode,int JobLevel)387 void NativeVbackupCleanup(JobControlRecord* jcr, int TermCode, int JobLevel)
388 {
389 char ec1[30], ec2[30];
390 char term_code[100];
391 const char* TermMsg;
392 int msg_type = M_INFO;
393 ClientDbRecord cr;
394 PoolMem query(PM_MESSAGE);
395
396 Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode);
397
398 switch (jcr->JobStatus) {
399 case JS_Terminated:
400 case JS_Warnings:
401 jcr->impl->jr.JobLevel = JobLevel; /* We want this to appear as what the
402 first consolidated job was */
403 Jmsg(jcr, M_INFO, 0,
404 _("Joblevel was set to joblevel of first consolidated job: %s\n"),
405 job_level_to_str(JobLevel));
406 break;
407 default:
408 break;
409 }
410
411 jcr->JobFiles = jcr->impl->SDJobFiles;
412 jcr->JobBytes = jcr->impl->SDJobBytes;
413
414 if (jcr->getJobStatus() == JS_Terminated &&
415 (jcr->JobErrors || jcr->impl->SDErrors)) {
416 TermCode = JS_Warnings;
417 }
418
419 UpdateJobEnd(jcr, TermCode);
420
421 /*
422 * Update final items to set them to the previous job's values
423 */
424 Mmsg(query,
425 "UPDATE Job SET StartTime='%s',EndTime='%s',"
426 "JobTDate=%s WHERE JobId=%s",
427 jcr->impl->previous_jr.cStartTime, jcr->impl->previous_jr.cEndTime,
428 edit_uint64(jcr->impl->previous_jr.JobTDate, ec1),
429 edit_uint64(jcr->JobId, ec2));
430 jcr->db->SqlQuery(query.c_str());
431
432 /*
433 * Get the fully updated job record
434 */
435 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->jr)) {
436 Jmsg(jcr, M_WARNING, 0,
437 _("Error getting Job record for Job report: ERR=%s"),
438 jcr->db->strerror());
439 jcr->setJobStatus(JS_ErrorTerminated);
440 }
441
442 bstrncpy(cr.Name, jcr->impl->res.client->resource_name_, sizeof(cr.Name));
443 if (!jcr->db->GetClientRecord(jcr, &cr)) {
444 Jmsg(jcr, M_WARNING, 0,
445 _("Error getting Client record for Job report: ERR=%s"),
446 jcr->db->strerror());
447 }
448
449 UpdateBootstrapFile(jcr);
450
451 switch (jcr->JobStatus) {
452 case JS_Terminated:
453 TermMsg = _("Backup OK");
454 break;
455 case JS_Warnings:
456 TermMsg = _("Backup OK -- with warnings");
457 break;
458 case JS_FatalError:
459 case JS_ErrorTerminated:
460 TermMsg = _("*** Backup Error ***");
461 msg_type = M_ERROR; /* Generate error message */
462 if (jcr->store_bsock) {
463 jcr->store_bsock->signal(BNET_TERMINATE);
464 if (jcr->impl->SD_msg_chan_started) {
465 pthread_cancel(jcr->impl->SD_msg_chan);
466 }
467 }
468 break;
469 case JS_Canceled:
470 TermMsg = _("Backup Canceled");
471 if (jcr->store_bsock) {
472 jcr->store_bsock->signal(BNET_TERMINATE);
473 if (jcr->impl->SD_msg_chan_started) {
474 pthread_cancel(jcr->impl->SD_msg_chan);
475 }
476 }
477 break;
478 default:
479 TermMsg = term_code;
480 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
481 break;
482 }
483
484 GenerateBackupSummary(jcr, &cr, msg_type, TermMsg);
485
486 Dmsg0(100, "Leave vbackup_cleanup()\n");
487 }
488
489 /**
490 * This callback routine is responsible for inserting the
491 * items it gets into the bootstrap structure. For each JobId selected
492 * this routine is called once for each file. We do not allow
493 * duplicate filenames, but instead keep the info from the most
494 * recent file entered (i.e. the JobIds are assumed to be sorted)
495 *
496 * See uar_sel_files in sql_cmds.c for query that calls us.
497 * row[0]=Path, row[1]=Filename, row[2]=FileIndex
498 * row[3]=JobId row[4]=LStat
499 */
InsertBootstrapHandler(void * ctx,int num_fields,char ** row)500 static int InsertBootstrapHandler(void* ctx, int num_fields, char** row)
501 {
502 JobId_t JobId;
503 int FileIndex;
504 RestoreBootstrapRecord* bsr = (RestoreBootstrapRecord*)ctx;
505
506 JobId = str_to_int64(row[3]);
507 FileIndex = str_to_int64(row[2]);
508 AddFindex(bsr, JobId, FileIndex);
509 return 0;
510 }
511
CreateBootstrapFile(JobControlRecord * jcr,char * jobids)512 static bool CreateBootstrapFile(JobControlRecord* jcr, char* jobids)
513 {
514 RestoreContext rx;
515 UaContext* ua;
516
517 rx.bsr = std::make_unique<RestoreBootstrapRecord>();
518 ua = new_ua_context(jcr);
519 rx.JobIds = jobids;
520
521 if (!jcr->db->OpenBatchConnection(jcr)) {
522 Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion");
523 return false;
524 }
525
526 if (!jcr->db_batch->GetFileList(jcr, jobids, false /* don't use md5 */,
527 true /* use delta */, InsertBootstrapHandler,
528 (void*)rx.bsr.get())) {
529 Jmsg(jcr, M_ERROR, 0, "%s", jcr->db_batch->strerror());
530 }
531
532 AddVolumeInformationToBsr(ua, rx.bsr.get());
533 jcr->impl->ExpectedFiles = WriteBsrFile(ua, rx);
534 if (debug_level >= 10) {
535 Dmsg1(000, "Found %d files to consolidate.\n", jcr->impl->ExpectedFiles);
536 }
537 FreeUaContext(ua);
538 rx.bsr.reset(nullptr);
539 return jcr->impl->ExpectedFiles != 0;
540 }
541 } /* namespace directordaemon */
542