1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2016-2020 Bareos GmbH & Co. KG
5
6 This program is Free Software; you can redistribute it and/or
7 modify it under the terms of version three of the GNU Affero General Public
8 License as published by the Free Software Foundation and included
9 in the file LICENSE.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Affero General Public License for more details.
15
16 You should have received a copy of the GNU Affero General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 02110-1301, USA.
20 */
21 /*
22 * Philipp Storz, May 2016
23 */
24 /** @file
25 * responsible for doing consolidation jobs
26 *
27 * Basic tasks done here:
28 * run a virtual full job for all jobs that are configured to be always
29 * incremental based on admin.c
30 *
31 */
32
33 #include "include/bareos.h"
34 #include "dird.h"
35 #include "dird/consolidate.h"
36 #include "dird/jcr_private.h"
37 #include "dird/job.h"
38 #include "dird/storage.h"
39 #include "dird/ua_input.h"
40 #include "dird/ua_server.h"
41 #include "dird/ua_run.h"
42 #include "dird/dird_globals.h"
43 #include "lib/edit.h"
44 #include "lib/parse_conf.h"
45
46 namespace directordaemon {
47
48 static const int debuglevel = 100;
49
DoConsolidateInit(JobControlRecord * jcr)50 bool DoConsolidateInit(JobControlRecord* jcr)
51 {
52 FreeRstorage(jcr);
53 if (!AllowDuplicateJob(jcr)) { return false; }
54 return true;
55 }
56
57 /**
58 * Start a Virtual(Full) Job that creates a new virtual backup
59 * containing the jobids given in jcr->impl_->vf_jobids
60 */
StartNewConsolidationJob(JobControlRecord * jcr,char * jobname)61 static inline void StartNewConsolidationJob(JobControlRecord* jcr,
62 char* jobname)
63 {
64 JobId_t jobid;
65 UaContext* ua;
66 PoolMem cmd(PM_MESSAGE);
67
68 ua = new_ua_context(jcr);
69 ua->batch = true;
70 Mmsg(ua->cmd, "run job=\"%s\" jobid=%s level=VirtualFull %s", jobname,
71 jcr->impl->vf_jobids, jcr->accurate ? "accurate=yes" : "accurate=no");
72
73 Dmsg1(debuglevel, "=============== consolidate cmd=%s\n", ua->cmd);
74 ParseUaArgs(ua); /* parse command */
75
76 jobid = DoRunCmd(ua, ua->cmd);
77 if (jobid == 0) {
78 Jmsg(jcr, M_ERROR, 0, _("Could not start %s job.\n"),
79 jcr->get_OperationName());
80 } else {
81 Jmsg(jcr, M_INFO, 0, _("%s JobId %d started.\n"), jcr->get_OperationName(),
82 (int)jobid);
83 }
84
85 FreeUaContext(ua);
86 }
87
88 /**
89 * The actual consolidation worker
90 *
91 * Returns: false on failure
92 * true on success
93 */
DoConsolidate(JobControlRecord * jcr)94 bool DoConsolidate(JobControlRecord* jcr)
95 {
96 char* p;
97 JobResource* job;
98 JobResource* tmpjob;
99 bool retval = true;
100 char* jobids = NULL;
101 time_t now = time(NULL);
102 int32_t fullconsolidations_started = 0;
103 int32_t max_full_consolidations = 0;
104
105 tmpjob = jcr->impl->res.job; /* Memorize job */
106
107 /*
108 * Get Value for MaxFullConsolidations from Consolidation job
109 */
110 max_full_consolidations = jcr->impl->res.job->MaxFullConsolidations;
111
112 jcr->impl->jr.JobId = jcr->JobId;
113 jcr->impl->fname = (char*)GetPoolMemory(PM_FNAME);
114
115 /*
116 * Print Job Start message
117 */
118 Jmsg(jcr, M_INFO, 0, _("Start Consolidate JobId %d, Job=%s\n"), jcr->JobId,
119 jcr->Job);
120
121 jcr->setJobStatus(JS_Running);
122
123 foreach_res (job, R_JOB) {
124 if (job->AlwaysIncremental) {
125 db_list_ctx jobids_ctx;
126 int32_t incrementals_total;
127 int32_t incrementals_to_consolidate;
128 int32_t max_incrementals_to_consolidate;
129
130 Jmsg(jcr, M_INFO, 0, _("Looking at always incremental job %s\n"),
131 job->resource_name_);
132
133 /*
134 * Fake always incremental job as job of current jcr.
135 */
136 jcr->impl->res.job = job;
137 jcr->impl->res.fileset = job->fileset;
138 jcr->impl->res.client = job->client;
139 jcr->impl->jr.JobLevel = L_INCREMENTAL;
140 jcr->impl->jr.limit = 0;
141 jcr->impl->jr.StartTime = 0;
142
143 if (!GetOrCreateFilesetRecord(jcr)) {
144 Jmsg(jcr, M_FATAL, 0, _("JobId=%d no FileSet\n"), (int)jcr->JobId);
145 retval = false;
146 goto bail_out;
147 }
148
149 if (!GetOrCreateClientRecord(jcr)) {
150 Jmsg(jcr, M_FATAL, 0, _("JobId=%d no ClientId\n"), (int)jcr->JobId);
151 retval = false;
152 goto bail_out;
153 }
154
155 /*
156 * First determine the number of total incrementals
157 */
158 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
159 incrementals_total = jobids_ctx.size() - 1;
160 Dmsg1(10, "unlimited jobids list: %s.\n",
161 jobids_ctx.GetAsString().c_str());
162
163 /*
164 * If we are doing always incremental, we need to limit the search to
165 * only include incrementals that are older than (now -
166 * AlwaysIncrementalJobRetention)
167 */
168 if (job->AlwaysIncrementalJobRetention) {
169 char sdt[50];
170
171 jcr->impl->jr.StartTime = now - job->AlwaysIncrementalJobRetention;
172 bstrftimes(sdt, sizeof(sdt), jcr->impl->jr.StartTime);
173 Jmsg(jcr, M_INFO, 0,
174 _("%s: considering jobs older than %s for consolidation.\n"),
175 job->resource_name_, sdt);
176 Dmsg4(10,
177 _("%s: considering jobs with ClientId %d and FilesetId %d older "
178 "than %s for consolidation.\n"),
179 job->resource_name_, jcr->impl->jr.ClientId,
180 jcr->impl->jr.FileSetId, sdt);
181 }
182
183 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
184 Dmsg1(10, "consolidate candidates: %s.\n",
185 jobids_ctx.GetAsString().c_str());
186
187 /**
188 * Consolidation of zero or one job does not make sense, we leave it like
189 * it is
190 */
191 if (incrementals_total < 1) {
192 Jmsg(jcr, M_INFO, 0,
193 _("%s: less than two jobs to consolidate found, doing nothing.\n"),
194 job->resource_name_);
195 continue;
196 }
197
198 /**
199 * Calculate limit for query. We specify how many incrementals should be
200 * left. the limit is total number of incrementals - number required - 1
201 */
202 max_incrementals_to_consolidate
203 = incrementals_total - job->AlwaysIncrementalKeepNumber;
204
205 Dmsg2(10, "Incrementals found/required. (%d/%d).\n", incrementals_total,
206 job->AlwaysIncrementalKeepNumber);
207 if ((max_incrementals_to_consolidate + 1) > 1) {
208 jcr->impl->jr.limit = max_incrementals_to_consolidate + 1;
209 Dmsg3(10, "total: %d, to_consolidate: %d, limit: %d.\n",
210 incrementals_total, max_incrementals_to_consolidate,
211 jcr->impl->jr.limit);
212 jobids_ctx.clear();
213 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
214 incrementals_to_consolidate = jobids_ctx.size() - 1;
215 Dmsg2(10, "%d consolidate ids after limit: %s.\n", jobids_ctx.size(),
216 jobids_ctx.GetAsString().c_str());
217 if (incrementals_to_consolidate < 1) {
218 Jmsg(jcr, M_INFO, 0,
219 _("%s: After limited query: less incrementals than required, "
220 "not consolidating\n"),
221 job->resource_name_);
222 continue;
223 }
224 } else {
225 Jmsg(jcr, M_INFO, 0,
226 _("%s: less incrementals than required, not consolidating\n"),
227 job->resource_name_);
228 continue;
229 }
230
231 if (jobids) {
232 free(jobids);
233 jobids = NULL;
234 }
235
236 jobids = strdup(jobids_ctx.GetAsString().c_str());
237 p = jobids;
238
239 /**
240 * Check if we need to skip the first (full) job from consolidation
241 */
242 if (job->AlwaysIncrementalMaxFullAge) {
243 char sdt_allowed[50];
244 char sdt_starttime[50];
245 time_t starttime, oldest_allowed_starttime;
246
247 if (incrementals_to_consolidate < 2) {
248 Jmsg(jcr, M_INFO, 0,
249 _("%s: less incrementals than required to consolidate without "
250 "full, not consolidating\n"),
251 job->resource_name_);
252 continue;
253 }
254 Jmsg(jcr, M_INFO, 0, _("before ConsolidateFull: jobids: %s\n"), jobids);
255
256 p = strchr(jobids, ','); /* find oldest jobid and optionally skip it */
257 if (p) { *p = '\0'; }
258
259 /**
260 * Get db record of oldest jobid and check its age
261 */
262 jcr->impl->previous_jr = JobDbRecord{};
263 jcr->impl->previous_jr.JobId = str_to_int64(jobids);
264 Dmsg1(10, "Previous JobId=%s\n", jobids);
265
266 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->previous_jr)) {
267 Jmsg(jcr, M_FATAL, 0,
268 _("Error getting Job record for first Job: ERR=%s"),
269 jcr->db->strerror());
270 goto bail_out;
271 }
272
273 starttime = jcr->impl->previous_jr.JobTDate;
274 oldest_allowed_starttime = now - job->AlwaysIncrementalMaxFullAge;
275 bstrftimes(sdt_allowed, sizeof(sdt_allowed), oldest_allowed_starttime);
276 bstrftimes(sdt_starttime, sizeof(sdt_starttime), starttime);
277
278 /**
279 * Check if job is older than AlwaysIncrementalMaxFullAge
280 */
281 Jmsg(jcr, M_INFO, 0, _("check full age: full is %s, allowed is %s\n"),
282 sdt_starttime, sdt_allowed);
283 if (starttime > oldest_allowed_starttime) {
284 Jmsg(jcr, M_INFO, 0,
285 _("Full is newer than AlwaysIncrementalMaxFullAge -> skipping "
286 "first jobid %s because of age\n"),
287 jobids);
288 if (p) { *p++ = ','; /* Restore , and point to rest of list */ }
289
290 } else if (max_full_consolidations
291 && fullconsolidations_started >= max_full_consolidations) {
292 Jmsg(jcr, M_INFO, 0,
293 _("%d AlwaysIncrementalFullConsolidations reached -> skipping "
294 "first jobid %s independent of age\n"),
295 max_full_consolidations, jobids);
296 if (p) { *p++ = ','; /* Restore , and point to rest of list */ }
297
298 } else {
299 Jmsg(jcr, M_INFO, 0,
300 _("Full is older than AlwaysIncrementalMaxFullAge -> also "
301 "consolidating Full jobid %s\n"),
302 jobids);
303 if (p) {
304 *p = ','; /* Restore ,*/
305 p = jobids; /* Point to full list */
306 }
307 fullconsolidations_started++;
308 }
309 Jmsg(jcr, M_INFO, 0, _("after ConsolidateFull: jobids: %s\n"), p);
310 }
311
312 /**
313 * Set the virtualfull jobids to be consolidated
314 */
315 if (!jcr->impl->vf_jobids) {
316 jcr->impl->vf_jobids = GetPoolMemory(PM_MESSAGE);
317 }
318 PmStrcpy(jcr->impl->vf_jobids, p);
319
320 Jmsg(jcr, M_INFO, 0, _("%s: Start new consolidation\n"),
321 job->resource_name_);
322 StartNewConsolidationJob(jcr, job->resource_name_);
323 }
324 }
325
326 bail_out:
327 /**
328 * Restore original job back to jcr.
329 */
330 jcr->impl->res.job = tmpjob;
331 jcr->setJobStatus(JS_Terminated);
332 ConsolidateCleanup(jcr, JS_Terminated);
333
334 if (jobids) { free(jobids); }
335
336 return retval;
337 }
338
339 /**
340 * Release resources allocated during backup.
341 */
ConsolidateCleanup(JobControlRecord * jcr,int TermCode)342 void ConsolidateCleanup(JobControlRecord* jcr, int TermCode)
343 {
344 int msg_type;
345 char term_code[100];
346 const char* TermMsg;
347 char sdt[50], edt[50], schedt[50];
348
349 Dmsg0(debuglevel, "Enter backup_cleanup()\n");
350
351 UpdateJobEnd(jcr, TermCode);
352
353 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->jr)) {
354 Jmsg(jcr, M_WARNING, 0,
355 _("Error getting Job record for Job report: ERR=%s"),
356 jcr->db->strerror());
357 jcr->setJobStatus(JS_ErrorTerminated);
358 }
359
360 msg_type = M_INFO; /* by default INFO message */
361 switch (jcr->JobStatus) {
362 case JS_Terminated:
363 TermMsg = _("Consolidate OK");
364 break;
365 case JS_FatalError:
366 case JS_ErrorTerminated:
367 TermMsg = _("*** Consolidate Error ***");
368 msg_type = M_ERROR; /* Generate error message */
369 break;
370 case JS_Canceled:
371 TermMsg = _("Consolidate Canceled");
372 break;
373 default:
374 TermMsg = term_code;
375 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
376 break;
377 }
378 bstrftimes(schedt, sizeof(schedt), jcr->impl->jr.SchedTime);
379 bstrftimes(sdt, sizeof(sdt), jcr->impl->jr.StartTime);
380 bstrftimes(edt, sizeof(edt), jcr->impl->jr.EndTime);
381
382 Jmsg(jcr, msg_type, 0,
383 _("BAREOS %s (%s): %s\n"
384 " JobId: %d\n"
385 " Job: %s\n"
386 " Scheduled time: %s\n"
387 " Start time: %s\n"
388 " End time: %s\n"
389 " Bareos binary info: %s\n"
390 " Job triggered by: %s\n"
391 " Termination: %s\n\n"),
392 kBareosVersionStrings.Full, kBareosVersionStrings.ShortDate, edt,
393 jcr->impl->jr.JobId, jcr->impl->jr.Job, schedt, sdt, edt,
394 kBareosVersionStrings.JoblogMessage,
395 JobTriggerToString(jcr->impl->job_trigger).c_str(), TermMsg);
396
397 Dmsg0(debuglevel, "Leave ConsolidateCleanup()\n");
398 }
399 } /* namespace directordaemon */
400