1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2008-2012 Free Software Foundation Europe e.V.
5 Copyright (C) 2011-2016 Planets Communications B.V.
6 Copyright (C) 2013-2018 Bareos GmbH & Co. KG
7
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version three of the GNU Affero General Public
10 License as published by the Free Software Foundation and included
11 in the file LICENSE.
12
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Affero General Public License for more details.
17
18 You should have received a copy of the GNU Affero General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 02110-1301, USA.
22 */
23 /*
24 * Kern Sibbald, July MMVIII
25 */
26 /* @file
27 * responsible for doing virtual backup jobs or
28 * in other words, consolidation or synthetic backups.
29 *
30 * Basic tasks done here:
31 * * Open DB and create records for this job.
32 * * Figure out what Jobs to copy.
33 * * Open Message Channel with Storage daemon to tell him a job will be starting.
34 * * Open connection with File daemon and pass him commands to do the backup.
35 * * When the File daemon finishes the job, update the DB.
36 */
37
38 #include "include/bareos.h"
39 #include "dird.h"
40 #include "dird/dird_globals.h"
41 #include "dird/backup.h"
42 #include "dird/bsr.h"
43 #include "dird/job.h"
44 #include "dird/migration.h"
45 #include "dird/msgchan.h"
46 #include "dird/sd_cmds.h"
47 #include "dird/storage.h"
48 #include "dird/ua_server.h"
49 #include "dird/ua_purge.h"
50 #include "dird/vbackup.h"
51 #include "lib/edit.h"
52
53 namespace directordaemon {
54
55 static const int dbglevel = 10;
56
57 static bool CreateBootstrapFile(JobControlRecord *jcr, char *jobids);
58
59 /**
60 * Called here before the job is run to do the job specific setup.
61 */
DoNativeVbackupInit(JobControlRecord * jcr)62 bool DoNativeVbackupInit(JobControlRecord *jcr)
63 {
64 const char *storage_source;
65
66 if (!GetOrCreateFilesetRecord(jcr)) {
67 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
68 return false;
69 }
70
71 ApplyPoolOverrides(jcr);
72
73 if (!AllowDuplicateJob(jcr)) {
74 return false;
75 }
76
77 jcr->jr.PoolId = GetOrCreatePoolRecord(jcr, jcr->res.pool->name());
78 if (jcr->jr.PoolId == 0) {
79 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
80 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
81 return false;
82 }
83
84 /*
85 * Note, at this point, pool is the pool for this job.
86 * We transfer it to rpool (read pool), and a bit later,
87 * pool will be changed to point to the write pool,
88 * which comes from pool->NextPool.
89 */
90 jcr->res.rpool = jcr->res.pool; /* save read pool */
91 PmStrcpy(jcr->res.rpool_source, jcr->res.pool_source);
92
93 /*
94 * If pool storage specified, use it for restore
95 */
96 CopyRstorage(jcr, jcr->res.pool->storage, _("Pool resource"));
97
98 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->res.rpool->name(), jcr->res.rpool_source);
99
100 jcr->start_time = time(NULL);
101 jcr->jr.StartTime = jcr->start_time;
102 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->jr)) {
103 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
104 }
105
106 /*
107 * See if there is a next pool override.
108 */
109 if (jcr->res.run_next_pool_override) {
110 PmStrcpy(jcr->res.npool_source, _("Run NextPool override"));
111 PmStrcpy(jcr->res.pool_source, _("Run NextPool override"));
112 storage_source = _("Storage from Run NextPool override");
113 } else {
114 /*
115 * See if there is a next pool override in the Job definition.
116 */
117 if (jcr->res.job->next_pool) {
118 jcr->res.next_pool = jcr->res.job->next_pool;
119 PmStrcpy(jcr->res.npool_source, _("Job's NextPool resource"));
120 PmStrcpy(jcr->res.pool_source, _("Job's NextPool resource"));
121 storage_source = _("Storage from Job's NextPool resource");
122 } else {
123 /*
124 * Fall back to the pool's NextPool definition.
125 */
126 jcr->res.next_pool = jcr->res.pool->NextPool;
127 PmStrcpy(jcr->res.npool_source, _("Job Pool's NextPool resource"));
128 PmStrcpy(jcr->res.pool_source, _("Job Pool's NextPool resource"));
129 storage_source = _("Storage from Pool's NextPool resource");
130 }
131 }
132
133 /*
134 * If the original backup pool has a NextPool, make sure a
135 * record exists in the database. Note, in this case, we
136 * will be migrating from pool to pool->NextPool.
137 */
138 if (jcr->res.next_pool) {
139 jcr->jr.PoolId = GetOrCreatePoolRecord(jcr, jcr->res.next_pool->name());
140 if (jcr->jr.PoolId == 0) {
141 return false;
142 }
143 }
144
145 if (!SetMigrationWstorage(jcr, jcr->res.pool, jcr->res.next_pool, storage_source)) {
146 return false;
147 }
148
149 jcr->res.pool = jcr->res.next_pool;
150
151 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->res.pool->name(), jcr->res.rpool->name());
152
153 // CreateClones(jcr);
154
155 return true;
156 }
157
158 /**
159 * Do a virtual backup, which consolidates all previous backups into a sort of synthetic Full.
160 *
161 * Returns: false on failure
162 * true on success
163 */
DoNativeVbackup(JobControlRecord * jcr)164 bool DoNativeVbackup(JobControlRecord *jcr)
165 {
166 char *p;
167 BareosSocket *sd;
168 char *jobids;
169 char ed1[100];
170 int JobLevel_of_first_job;
171
172 if (!jcr->res.read_storage_list) {
173 Jmsg(jcr, M_FATAL, 0, _("No storage for reading given.\n"));
174 return false;
175 }
176
177 if (!jcr->res.write_storage_list) {
178 Jmsg(jcr, M_FATAL, 0, _("No storage for writing given.\n"));
179 return false;
180 }
181
182 Dmsg2(100, "read_storage_list=%p write_storage_list=%p\n", jcr->res.read_storage_list, jcr->res.write_storage_list);
183 Dmsg2(100, "Read store=%s, write store=%s\n",
184 ((StorageResource *)jcr->res.read_storage_list->first())->name(),
185 ((StorageResource *)jcr->res.write_storage_list->first())->name());
186
187 /*
188 * Print Job Start message
189 */
190 Jmsg(jcr, M_INFO, 0, _("Start Virtual Backup JobId %s, Job=%s\n"),
191 edit_uint64(jcr->JobId, ed1), jcr->Job);
192
193 if (!jcr->accurate) {
194 Jmsg(jcr, M_WARNING, 0,
195 _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n"));
196 }
197
198 /*
199 * See if we already got a list of jobids to use.
200 */
201 if (jcr->vf_jobids) {
202 Dmsg1(10, "jobids=%s\n", jcr->vf_jobids);
203 jobids = bstrdup(jcr->vf_jobids);
204
205 } else {
206 db_list_ctx jobids_ctx;
207 jcr->db->AccurateGetJobids(jcr, &jcr->jr, &jobids_ctx);
208 Dmsg1(10, "consolidate candidates: %s.\n", jobids_ctx.list);
209
210 if (jobids_ctx.count == 0) {
211 Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n"));
212 return false;
213 }
214
215 jobids = bstrdup(jobids_ctx.list);
216 }
217
218 Jmsg(jcr, M_INFO, 0, _("Consolidating JobIds %s\n"), jobids);
219
220 /*
221 * Find oldest Jobid, get the db record and find its level
222 */
223 p = strchr(jobids, ','); /* find oldest jobid */
224 if (p) {
225 *p = '\0';
226 }
227
228 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
229 jcr->previous_jr.JobId = str_to_int64(jobids);
230 Dmsg1(10, "Previous JobId=%s\n", jobids);
231
232 /*
233 * See if we need to restore the stripped ','
234 */
235 if (p) {
236 *p = ',';
237 }
238
239 if (!jcr->db->GetJobRecord(jcr, &jcr->previous_jr)) {
240 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for first Job: ERR=%s"), jcr->db->strerror());
241 goto bail_out;
242 }
243
244 JobLevel_of_first_job = jcr->previous_jr.JobLevel;
245 Dmsg2(10, "Level of first consolidated job %d: %s\n", jcr->previous_jr.JobId, job_level_to_str(JobLevel_of_first_job));
246
247 /*
248 * Now we find the newest job that ran and store its info in
249 * the previous_jr record. We will set our times to the
250 * values from that job so that anything changed after that
251 * time will be picked up on the next backup.
252 */
253 p = strrchr(jobids, ','); /* find newest jobid */
254 if (p) {
255 p++;
256 } else {
257 p = jobids;
258 }
259
260 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
261 jcr->previous_jr.JobId = str_to_int64(p);
262 Dmsg1(10, "Previous JobId=%s\n", p);
263
264 if (!jcr->db->GetJobRecord(jcr, &jcr->previous_jr)) {
265 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for previous Job: ERR=%s"), jcr->db->strerror());
266 goto bail_out;
267 }
268
269 if (!CreateBootstrapFile(jcr, jobids)) {
270 Jmsg(jcr, M_FATAL, 0, _("Could not create bootstrap file\n"));
271 goto bail_out;
272 }
273
274 /*
275 * Open a message channel connection with the Storage
276 * daemon.
277 */
278 Dmsg0(110, "Open connection with storage daemon\n");
279 jcr->setJobStatus(JS_WaitSD);
280
281 /*
282 * Start conversation with Storage daemon
283 */
284 if (!ConnectToStorageDaemon(jcr, 10, me->SDConnectTimeout, true)) {
285 goto bail_out;
286 }
287 sd = jcr->store_bsock;
288
289 /*
290 * Now start a job with the Storage daemon
291 */
292 if (!StartStorageDaemonJob(jcr, jcr->res.read_storage_list, jcr->res.write_storage_list, /* send_bsr */ true)) {
293 goto bail_out;
294 }
295 Dmsg0(100, "Storage daemon connection OK\n");
296
297 /*
298 * We re-update the job start record so that the start
299 * time is set after the run before job. This avoids
300 * that any files created by the run before job will
301 * be saved twice. They will be backed up in the current
302 * job, but not in the next one unless they are changed.
303 * Without this, they will be backed up in this job and
304 * in the next job run because in that case, their date
305 * is after the start of this run.
306 */
307 jcr->start_time = time(NULL);
308 jcr->jr.StartTime = jcr->start_time;
309 jcr->jr.JobTDate = jcr->start_time;
310 jcr->setJobStatus(JS_Running);
311
312 /*
313 * Update job start record
314 */
315 if (!jcr->db->UpdateJobStartRecord(jcr, &jcr->jr)) {
316 Jmsg(jcr, M_FATAL, 0, "%s", jcr->db->strerror());
317 goto bail_out;
318 }
319
320 /*
321 * Declare the job started to start the MaxRunTime check
322 */
323 jcr->setJobStarted();
324
325 /*
326 * Start the job prior to starting the message thread below
327 * to avoid two threads from using the BareosSocket structure at
328 * the same time.
329 */
330 if (!sd->fsend("run")) {
331 goto bail_out;
332 }
333
334 /*
335 * Now start a Storage daemon message thread
336 */
337 if (!StartStorageDaemonMessageThread(jcr)) {
338 goto bail_out;
339 }
340
341 jcr->setJobStatus(JS_Running);
342
343 /*
344 * Pickup Job termination data
345 * Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors
346 */
347 WaitForStorageDaemonTermination(jcr);
348 jcr->setJobStatus(jcr->SDJobStatus);
349 jcr->db_batch->WriteBatchFileRecords(jcr); /* used by bulk batch file insert */
350 if (!jcr->is_JobStatus(JS_Terminated)) {
351 goto bail_out;
352 }
353
354 NativeVbackupCleanup(jcr, jcr->JobStatus, JobLevel_of_first_job);
355
356 /*
357 * Remove the successfully consolidated jobids from the database
358 */
359 if (jcr->res.job->AlwaysIncremental && jcr->res.job->AlwaysIncrementalJobRetention) {
360 UaContext *ua;
361 ua = new_ua_context(jcr);
362 PurgeJobsFromCatalog(ua, jobids);
363 Jmsg(jcr, M_INFO, 0, _("purged JobIds %s as they were consolidated into Job %s\n"), jobids, edit_uint64(jcr->JobId, ed1) );
364 }
365
366 free(jobids);
367 return true;
368
369 bail_out:
370 free(jobids);
371 return false;
372 }
373
374 /**
375 * Release resources allocated during backup.
376 */
NativeVbackupCleanup(JobControlRecord * jcr,int TermCode,int JobLevel)377 void NativeVbackupCleanup(JobControlRecord *jcr, int TermCode, int JobLevel)
378 {
379 char ec1[30], ec2[30];
380 char term_code[100];
381 const char *TermMsg;
382 int msg_type = M_INFO;
383 ClientDbRecord cr;
384 PoolMem query(PM_MESSAGE);
385
386 Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode);
387 memset(&cr, 0, sizeof(cr));
388
389 switch (jcr->JobStatus) {
390 case JS_Terminated:
391 case JS_Warnings:
392 jcr->jr.JobLevel = JobLevel; /* We want this to appear as what the first consolidated job was */
393 Jmsg(jcr, M_INFO, 0, _("Joblevel was set to joblevel of first consolidated job: %s\n"), job_level_to_str(JobLevel));
394 break;
395 default:
396 break;
397 }
398
399 jcr->JobFiles = jcr->SDJobFiles;
400 jcr->JobBytes = jcr->SDJobBytes;
401
402 if (jcr->getJobStatus() == JS_Terminated &&
403 (jcr->JobErrors || jcr->SDErrors)) {
404 TermCode = JS_Warnings;
405 }
406
407 UpdateJobEnd(jcr, TermCode);
408
409 /*
410 * Update final items to set them to the previous job's values
411 */
412 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
413 "JobTDate=%s WHERE JobId=%s",
414 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
415 edit_uint64(jcr->previous_jr.JobTDate, ec1),
416 edit_uint64(jcr->JobId, ec2));
417 jcr->db->SqlQuery(query.c_str());
418
419 /*
420 * Get the fully updated job record
421 */
422 if (!jcr->db->GetJobRecord(jcr, &jcr->jr)) {
423 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
424 jcr->db->strerror());
425 jcr->setJobStatus(JS_ErrorTerminated);
426 }
427
428 bstrncpy(cr.Name, jcr->res.client->name(), sizeof(cr.Name));
429 if (!jcr->db->GetClientRecord(jcr, &cr)) {
430 Jmsg(jcr, M_WARNING, 0, _("Error getting Client record for Job report: ERR=%s"),
431 jcr->db->strerror());
432 }
433
434 UpdateBootstrapFile(jcr);
435
436 switch (jcr->JobStatus) {
437 case JS_Terminated:
438 TermMsg = _("Backup OK");
439 break;
440 case JS_Warnings:
441 TermMsg = _("Backup OK -- with warnings");
442 break;
443 case JS_FatalError:
444 case JS_ErrorTerminated:
445 TermMsg = _("*** Backup Error ***");
446 msg_type = M_ERROR; /* Generate error message */
447 if (jcr->store_bsock) {
448 jcr->store_bsock->signal(BNET_TERMINATE);
449 if (jcr->SD_msg_chan_started) {
450 pthread_cancel(jcr->SD_msg_chan);
451 }
452 }
453 break;
454 case JS_Canceled:
455 TermMsg = _("Backup Canceled");
456 if (jcr->store_bsock) {
457 jcr->store_bsock->signal(BNET_TERMINATE);
458 if (jcr->SD_msg_chan_started) {
459 pthread_cancel(jcr->SD_msg_chan);
460 }
461 }
462 break;
463 default:
464 TermMsg = term_code;
465 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
466 break;
467 }
468
469 GenerateBackupSummary(jcr, &cr, msg_type, TermMsg);
470
471 Dmsg0(100, "Leave vbackup_cleanup()\n");
472 }
473
474 /**
475 * This callback routine is responsible for inserting the
476 * items it gets into the bootstrap structure. For each JobId selected
477 * this routine is called once for each file. We do not allow
478 * duplicate filenames, but instead keep the info from the most
479 * recent file entered (i.e. the JobIds are assumed to be sorted)
480 *
481 * See uar_sel_files in sql_cmds.c for query that calls us.
482 * row[0]=Path, row[1]=Filename, row[2]=FileIndex
483 * row[3]=JobId row[4]=LStat
484 */
InsertBootstrapHandler(void * ctx,int num_fields,char ** row)485 static int InsertBootstrapHandler(void *ctx, int num_fields, char **row)
486 {
487 JobId_t JobId;
488 int FileIndex;
489 RestoreBootstrapRecord *bsr = (RestoreBootstrapRecord *)ctx;
490
491 JobId = str_to_int64(row[3]);
492 FileIndex = str_to_int64(row[2]);
493 AddFindex(bsr, JobId, FileIndex);
494 return 0;
495 }
496
CreateBootstrapFile(JobControlRecord * jcr,char * jobids)497 static bool CreateBootstrapFile(JobControlRecord *jcr, char *jobids)
498 {
499 RestoreContext rx;
500 UaContext *ua;
501
502 memset(&rx, 0, sizeof(rx));
503 rx.bsr = new_bsr();
504 ua = new_ua_context(jcr);
505 rx.JobIds = jobids;
506
507 if (!jcr->db->OpenBatchConnection(jcr)) {
508 Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion");
509 return false;
510 }
511
512 if (!jcr->db_batch->GetFileList(jcr, jobids, false /* don't use md5 */,
513 true /* use delta */,
514 InsertBootstrapHandler, (void *)rx.bsr)) {
515 Jmsg(jcr, M_ERROR, 0, "%s", jcr->db_batch->strerror());
516 }
517
518 CompleteBsr(ua, rx.bsr);
519 jcr->ExpectedFiles = WriteBsrFile(ua, rx);
520 if (debug_level >= 10) {
521 Dmsg1(000, "Found %d files to consolidate.\n", jcr->ExpectedFiles);
522 }
523 if (jcr->ExpectedFiles == 0) {
524 FreeUaContext(ua);
525 directordaemon::FreeBsr(rx.bsr);
526 return false;
527 }
528 FreeUaContext(ua);
529 directordaemon::FreeBsr(rx.bsr);
530 return true;
531 }
532 } /* namespace directordaemon */
533