1 /*
2 BAREOS® - Backup Archiving REcovery Open Sourced
3
4 Copyright (C) 2016-2019 Bareos GmbH & Co. KG
5
6 This program is Free Software; you can redistribute it and/or
7 modify it under the terms of version three of the GNU Affero General Public
8 License as published by the Free Software Foundation and included
9 in the file LICENSE.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Affero General Public License for more details.
15
16 You should have received a copy of the GNU Affero General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 02110-1301, USA.
20 */
21 /*
22 * Philipp Storz, May 2016
23 */
24 /** @file
25 * responsible for doing consolidation jobs
26 *
27 * Basic tasks done here:
28 * run a virtual full job for all jobs that are configured to be always
29 * incremental based on admin.c
30 *
31 */
32
33 #include "include/bareos.h"
34 #include "dird.h"
35 #include "dird/consolidate.h"
36 #include "dird/jcr_private.h"
37 #include "dird/job.h"
38 #include "dird/storage.h"
39 #include "dird/ua_input.h"
40 #include "dird/ua_server.h"
41 #include "dird/ua_run.h"
42 #include "dird/dird_globals.h"
43 #include "lib/edit.h"
44 #include "lib/parse_conf.h"
45
46 namespace directordaemon {
47
48 static const int debuglevel = 100;
49
DoConsolidateInit(JobControlRecord * jcr)50 bool DoConsolidateInit(JobControlRecord* jcr)
51 {
52 FreeRstorage(jcr);
53 if (!AllowDuplicateJob(jcr)) { return false; }
54 return true;
55 }
56
57 /**
58 * Start a Virtual(Full) Job that creates a new virtual backup
59 * containing the jobids given in jcr->impl_->vf_jobids
60 */
StartNewConsolidationJob(JobControlRecord * jcr,char * jobname)61 static inline void StartNewConsolidationJob(JobControlRecord* jcr,
62 char* jobname)
63 {
64 JobId_t jobid;
65 UaContext* ua;
66 PoolMem cmd(PM_MESSAGE);
67
68 ua = new_ua_context(jcr);
69 ua->batch = true;
70 Mmsg(ua->cmd, "run job=\"%s\" jobid=%s level=VirtualFull %s", jobname,
71 jcr->impl->vf_jobids, jcr->accurate ? "accurate=yes" : "accurate=no");
72
73 Dmsg1(debuglevel, "=============== consolidate cmd=%s\n", ua->cmd);
74 ParseUaArgs(ua); /* parse command */
75
76 jobid = DoRunCmd(ua, ua->cmd);
77 if (jobid == 0) {
78 Jmsg(jcr, M_ERROR, 0, _("Could not start %s job.\n"),
79 jcr->get_OperationName());
80 } else {
81 Jmsg(jcr, M_INFO, 0, _("%s JobId %d started.\n"), jcr->get_OperationName(),
82 (int)jobid);
83 }
84
85 FreeUaContext(ua);
86 }
87
88 /**
89 * The actual consolidation worker
90 *
91 * Returns: false on failure
92 * true on success
93 */
DoConsolidate(JobControlRecord * jcr)94 bool DoConsolidate(JobControlRecord* jcr)
95 {
96 char* p;
97 JobResource* job;
98 JobResource* tmpjob;
99 bool retval = true;
100 char* jobids = NULL;
101 time_t now = time(NULL);
102 int32_t fullconsolidations_started = 0;
103 int32_t max_full_consolidations = 0;
104
105 tmpjob = jcr->impl->res.job; /* Memorize job */
106
107 /*
108 * Get Value for MaxFullConsolidations from Consolidation job
109 */
110 max_full_consolidations = jcr->impl->res.job->MaxFullConsolidations;
111
112 jcr->impl->jr.JobId = jcr->JobId;
113 jcr->impl->fname = (char*)GetPoolMemory(PM_FNAME);
114
115 /*
116 * Print Job Start message
117 */
118 Jmsg(jcr, M_INFO, 0, _("Start Consolidate JobId %d, Job=%s\n"), jcr->JobId,
119 jcr->Job);
120
121 jcr->setJobStatus(JS_Running);
122
123 foreach_res (job, R_JOB) {
124 if (job->AlwaysIncremental) {
125 db_list_ctx jobids_ctx;
126 int32_t incrementals_total;
127 int32_t incrementals_to_consolidate;
128 int32_t max_incrementals_to_consolidate;
129
130 Jmsg(jcr, M_INFO, 0, _("Looking at always incremental job %s\n"),
131 job->resource_name_);
132
133 /*
134 * Fake always incremental job as job of current jcr.
135 */
136 jcr->impl->res.job = job;
137 jcr->impl->res.fileset = job->fileset;
138 jcr->impl->res.client = job->client;
139 jcr->impl->jr.JobLevel = L_INCREMENTAL;
140 jcr->impl->jr.limit = 0;
141 jcr->impl->jr.StartTime = 0;
142
143 if (!GetOrCreateFilesetRecord(jcr)) {
144 Jmsg(jcr, M_FATAL, 0, _("JobId=%d no FileSet\n"), (int)jcr->JobId);
145 retval = false;
146 goto bail_out;
147 }
148
149 if (!GetOrCreateClientRecord(jcr)) {
150 Jmsg(jcr, M_FATAL, 0, _("JobId=%d no ClientId\n"), (int)jcr->JobId);
151 retval = false;
152 goto bail_out;
153 }
154
155 /*
156 * First determine the number of total incrementals
157 */
158 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
159 incrementals_total = jobids_ctx.count - 1;
160 Dmsg1(10, "unlimited jobids list: %s.\n", jobids_ctx.list);
161
162 /*
163 * If we are doing always incremental, we need to limit the search to
164 * only include incrementals that are older than (now -
165 * AlwaysIncrementalJobRetention)
166 */
167 if (job->AlwaysIncrementalJobRetention) {
168 char sdt[50];
169
170 jcr->impl->jr.StartTime = now - job->AlwaysIncrementalJobRetention;
171 bstrftimes(sdt, sizeof(sdt), jcr->impl->jr.StartTime);
172 Jmsg(jcr, M_INFO, 0,
173 _("%s: considering jobs older than %s for consolidation.\n"),
174 job->resource_name_, sdt);
175 Dmsg4(10,
176 _("%s: considering jobs with ClientId %d and FilesetId %d older "
177 "than %s for consolidation.\n"),
178 job->resource_name_, jcr->impl->jr.ClientId,
179 jcr->impl->jr.FileSetId, sdt);
180 }
181
182 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
183 Dmsg1(10, "consolidate candidates: %s.\n", jobids_ctx.list);
184
185 /**
186 * Consolidation of zero or one job does not make sense, we leave it like
187 * it is
188 */
189 if (incrementals_total < 1) {
190 Jmsg(jcr, M_INFO, 0,
191 _("%s: less than two jobs to consolidate found, doing nothing.\n"),
192 job->resource_name_);
193 continue;
194 }
195
196 /**
197 * Calculate limit for query. We specify how many incrementals should be
198 * left. the limit is total number of incrementals - number required - 1
199 */
200 max_incrementals_to_consolidate =
201 incrementals_total - job->AlwaysIncrementalKeepNumber;
202
203 Dmsg2(10, "Incrementals found/required. (%d/%d).\n", incrementals_total,
204 job->AlwaysIncrementalKeepNumber);
205 if ((max_incrementals_to_consolidate + 1) > 1) {
206 jcr->impl->jr.limit = max_incrementals_to_consolidate + 1;
207 Dmsg3(10, "total: %d, to_consolidate: %d, limit: %d.\n",
208 incrementals_total, max_incrementals_to_consolidate,
209 jcr->impl->jr.limit);
210 jobids_ctx.reset();
211 jcr->db->AccurateGetJobids(jcr, &jcr->impl->jr, &jobids_ctx);
212 incrementals_to_consolidate = jobids_ctx.count - 1;
213 Dmsg2(10, "%d consolidate ids after limit: %s.\n", jobids_ctx.count,
214 jobids_ctx.list);
215 if (incrementals_to_consolidate < 1) {
216 Jmsg(jcr, M_INFO, 0,
217 _("%s: After limited query: less incrementals than required, "
218 "not consolidating\n"),
219 job->resource_name_);
220 continue;
221 }
222 } else {
223 Jmsg(jcr, M_INFO, 0,
224 _("%s: less incrementals than required, not consolidating\n"),
225 job->resource_name_);
226 continue;
227 }
228
229 if (jobids) {
230 free(jobids);
231 jobids = NULL;
232 }
233
234 jobids = strdup(jobids_ctx.list);
235 p = jobids;
236
237 /**
238 * Check if we need to skip the first (full) job from consolidation
239 */
240 if (job->AlwaysIncrementalMaxFullAge) {
241 char sdt_allowed[50];
242 char sdt_starttime[50];
243 time_t starttime, oldest_allowed_starttime;
244
245 if (incrementals_to_consolidate < 2) {
246 Jmsg(jcr, M_INFO, 0,
247 _("%s: less incrementals than required to consolidate without "
248 "full, not consolidating\n"),
249 job->resource_name_);
250 continue;
251 }
252 Jmsg(jcr, M_INFO, 0, _("before ConsolidateFull: jobids: %s\n"), jobids);
253
254 p = strchr(jobids, ','); /* find oldest jobid and optionally skip it */
255 if (p) { *p = '\0'; }
256
257 /**
258 * Get db record of oldest jobid and check its age
259 */
260 jcr->impl->previous_jr = JobDbRecord{};
261 jcr->impl->previous_jr.JobId = str_to_int64(jobids);
262 Dmsg1(10, "Previous JobId=%s\n", jobids);
263
264 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->previous_jr)) {
265 Jmsg(jcr, M_FATAL, 0,
266 _("Error getting Job record for first Job: ERR=%s"),
267 jcr->db->strerror());
268 goto bail_out;
269 }
270
271 starttime = jcr->impl->previous_jr.JobTDate;
272 oldest_allowed_starttime = now - job->AlwaysIncrementalMaxFullAge;
273 bstrftimes(sdt_allowed, sizeof(sdt_allowed), oldest_allowed_starttime);
274 bstrftimes(sdt_starttime, sizeof(sdt_starttime), starttime);
275
276 /**
277 * Check if job is older than AlwaysIncrementalMaxFullAge
278 */
279 Jmsg(jcr, M_INFO, 0, _("check full age: full is %s, allowed is %s\n"),
280 sdt_starttime, sdt_allowed);
281 if (starttime > oldest_allowed_starttime) {
282 Jmsg(jcr, M_INFO, 0,
283 _("Full is newer than AlwaysIncrementalMaxFullAge -> skipping "
284 "first jobid %s because of age\n"),
285 jobids);
286 if (p) { *p++ = ','; /* Restore , and point to rest of list */ }
287
288 } else if (max_full_consolidations &&
289 fullconsolidations_started >= max_full_consolidations) {
290 Jmsg(jcr, M_INFO, 0,
291 _("%d AlwaysIncrementalFullConsolidations reached -> skipping "
292 "first jobid %s independent of age\n"),
293 max_full_consolidations, jobids);
294 if (p) { *p++ = ','; /* Restore , and point to rest of list */ }
295
296 } else {
297 Jmsg(jcr, M_INFO, 0,
298 _("Full is older than AlwaysIncrementalMaxFullAge -> also "
299 "consolidating Full jobid %s\n"),
300 jobids);
301 if (p) {
302 *p = ','; /* Restore ,*/
303 p = jobids; /* Point to full list */
304 }
305 fullconsolidations_started++;
306 }
307 Jmsg(jcr, M_INFO, 0, _("after ConsolidateFull: jobids: %s\n"), p);
308 }
309
310 /**
311 * Set the virtualfull jobids to be consolidated
312 */
313 if (!jcr->impl->vf_jobids) {
314 jcr->impl->vf_jobids = GetPoolMemory(PM_MESSAGE);
315 }
316 PmStrcpy(jcr->impl->vf_jobids, p);
317
318 Jmsg(jcr, M_INFO, 0, _("%s: Start new consolidation\n"),
319 job->resource_name_);
320 StartNewConsolidationJob(jcr, job->resource_name_);
321 }
322 }
323
324 bail_out:
325 /**
326 * Restore original job back to jcr.
327 */
328 jcr->impl->res.job = tmpjob;
329 jcr->setJobStatus(JS_Terminated);
330 ConsolidateCleanup(jcr, JS_Terminated);
331
332 if (jobids) { free(jobids); }
333
334 return retval;
335 }
336
337 /**
338 * Release resources allocated during backup.
339 */
ConsolidateCleanup(JobControlRecord * jcr,int TermCode)340 void ConsolidateCleanup(JobControlRecord* jcr, int TermCode)
341 {
342 int msg_type;
343 char term_code[100];
344 const char* TermMsg;
345 char sdt[50], edt[50], schedt[50];
346
347 Dmsg0(debuglevel, "Enter backup_cleanup()\n");
348
349 UpdateJobEnd(jcr, TermCode);
350
351 if (!jcr->db->GetJobRecord(jcr, &jcr->impl->jr)) {
352 Jmsg(jcr, M_WARNING, 0,
353 _("Error getting Job record for Job report: ERR=%s"),
354 jcr->db->strerror());
355 jcr->setJobStatus(JS_ErrorTerminated);
356 }
357
358 msg_type = M_INFO; /* by default INFO message */
359 switch (jcr->JobStatus) {
360 case JS_Terminated:
361 TermMsg = _("Consolidate OK");
362 break;
363 case JS_FatalError:
364 case JS_ErrorTerminated:
365 TermMsg = _("*** Consolidate Error ***");
366 msg_type = M_ERROR; /* Generate error message */
367 break;
368 case JS_Canceled:
369 TermMsg = _("Consolidate Canceled");
370 break;
371 default:
372 TermMsg = term_code;
373 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
374 break;
375 }
376 bstrftimes(schedt, sizeof(schedt), jcr->impl->jr.SchedTime);
377 bstrftimes(sdt, sizeof(sdt), jcr->impl->jr.StartTime);
378 bstrftimes(edt, sizeof(edt), jcr->impl->jr.EndTime);
379
380 Jmsg(jcr, msg_type, 0,
381 _("BAREOS %s (%s): %s\n"
382 " JobId: %d\n"
383 " Job: %s\n"
384 " Scheduled time: %s\n"
385 " Start time: %s\n"
386 " End time: %s\n"
387 " Bareos binary info: %s\n"
388 " Termination: %s\n\n"),
389 kBareosVersionStrings.Full, kBareosVersionStrings.ShortDate, edt,
390 jcr->impl->jr.JobId, jcr->impl->jr.Job, schedt, sdt, edt,
391 kBareosVersionStrings.JoblogMessage, TermMsg);
392
393 Dmsg0(debuglevel, "Leave ConsolidateCleanup()\n");
394 }
395 } /* namespace directordaemon */
396