1 /*
2 Bacula(R) - The Network Backup Solution
3
4 Copyright (C) 2000-2020 Kern Sibbald
5
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
8
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
13
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
16
17 Bacula(R) is a registered trademark of Kern Sibbald.
18 */
19 /*
20 * Bacula File Daemon verify-vol.c Verify files on a Volume
21 * versus attributes in Catalog
22 *
23 * Kern Sibbald, July MMII
24 *
25 * Data verification added by Eric Bollengier
26 *
27 */
28
29 #include "bacula.h"
30 #include "filed.h"
31 #include "findlib/win32filter.h"
32
33 #if defined(HAVE_LIBZ)
34 const bool have_libz = true;
35 #else
36 const bool have_libz = false;
37 #endif
38
39 #ifdef HAVE_LZO
40 const bool have_lzo = true;
41 #else
42 const bool have_lzo = false;
43 #endif
44
45 /* Context used during Verify Data job. We use it in the
46 * verify loop to compute checksums and check attributes.
47 */
48 class v_ctx {
49 public:
50 JCR *jcr;
51 int32_t stream; /* stream less new bits */
52 int32_t prev_stream; /* previous stream */
53 int32_t full_stream; /* full stream including new bits */
54 int32_t type; /* file type FT_ */
55 int64_t size; /* current file size */
56 ATTR *attr; /* Pointer to attributes */
57
58 bool check_size; /* Check or not the size attribute */
59 bool check_chksum; /* Check the checksum */
60
61 crypto_digest_t digesttype;
62 Win32Filter win32filter;
63 char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)]; /* current digest */
64
v_ctx(JCR * ajcr)65 v_ctx(JCR *ajcr) :
66 jcr(ajcr), stream(0), prev_stream(0), full_stream(0), type(0), size(-1),
67 attr(new_attr(jcr)), check_size(false), check_chksum(false),
68 digesttype(CRYPTO_DIGEST_NONE), win32filter()
69 {
70 *digest = 0;
71 scan_fileset();
72 };
~v_ctx()73 ~v_ctx() {
74 free_attr(attr);
75 };
76 /* Call this function when we change the file
77 * We check the st_size and we compute the digest
78 */
79 bool close_previous_stream();
80
81 /* Call when we have a sparse record */
82 void skip_sparse_header(char **data, uint32_t *length);
83
84 /* Scan the fileset to know if we want to check checksums or st_size */
85 void scan_fileset();
86
87 /* Check the catalog to locate the file */
88 void check_accurate();
89
90 /* In cleanup, we reset the current file size to -1 */
reset_size()91 void reset_size() {
92 size = -1;
93 };
94
95 /* Used for sparse files */
set_size(int64_t val)96 void set_size(int64_t val) {
97 size = MAX(size, val);
98 };
99
update_size(int64_t val)100 void update_size(int64_t val) {
101 if (size == -1) {
102 size = 0;
103 }
104 size += val;
105 };
106
update_checksum(char * wbuf,int32_t wsize)107 void update_checksum(char *wbuf, int32_t wsize) {
108 if (wsize > 0 && check_chksum) {
109 if (!jcr->crypto.digest) {
110 jcr->crypto.digest = crypto_digest_new(jcr, digesttype);
111 }
112 crypto_digest_update(jcr->crypto.digest, (uint8_t *)wbuf, wsize);
113 }
114 };
115 };
116
117 /* Data received from Storage Daemon */
118 static char rec_header[] = "rechdr %ld %ld %ld %ld %ld";
119
120 /* Forward referenced functions */
121
122 /* We don't know in advance which digest mode is needed, we do not
123 * want to store files on disk either to check afterward. So, we read
124 * the fileset definition and we try to guess the digest that will be
125 * used. If the FileSet uses multiple digests, it will not work.
126 */
scan_fileset()127 void v_ctx::scan_fileset()
128 {
129 findFILESET *fileset;
130
131 check_size = check_chksum = false;
132 digesttype = CRYPTO_DIGEST_NONE;
133
134 if (!jcr->ff || !jcr->ff->fileset) {
135 return;
136 }
137
138 fileset = jcr->ff->fileset;
139
140 for (int i=0; i<fileset->include_list.size(); i++) {
141 findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i);
142
143 for (int j=0; j<incexe->opts_list.size(); j++) {
144 findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j);
145 check_size = (strchr(fo->VerifyOpts, 's') != NULL);
146 if ((strchr(fo->VerifyOpts, '1') != NULL) ||
147 (strchr(fo->VerifyOpts, '5') != NULL))
148 {
149 check_chksum = true;
150 }
151
152 if (fo->flags & FO_MD5) {
153 digesttype = CRYPTO_DIGEST_MD5;
154 return;
155 }
156 if (fo->flags & FO_SHA1) {
157 digesttype = CRYPTO_DIGEST_SHA1;
158 return;
159 }
160 if (fo->flags & FO_SHA256) {
161 digesttype = CRYPTO_DIGEST_SHA256;
162 return;
163 }
164 if (fo->flags & FO_SHA512) {
165 digesttype = CRYPTO_DIGEST_SHA512;
166 return;
167 }
168 }
169 }
170 digesttype = CRYPTO_DIGEST_NONE;
171 if (check_chksum) {
172 Jmsg(jcr, M_WARNING, 0, _("Checksum verification required in Verify FileSet option, but no Signature found in the FileSet\n"));
173 check_chksum = false;
174 }
175 }
176
177 /* Compute the file size for sparse records and adjust the data */
skip_sparse_header(char ** data,uint32_t * length)178 void v_ctx::skip_sparse_header(char **data, uint32_t *length)
179 {
180 unser_declare;
181 uint64_t faddr;
182 unser_begin(*data, OFFSET_FADDR_SIZE);
183 unser_uint64(faddr);
184
185 /* For sparse, we assume that the file is at least big as faddr */
186 set_size(faddr);
187
188 *data += OFFSET_FADDR_SIZE;
189 *length -= OFFSET_FADDR_SIZE;
190 }
191
check_accurate()192 void v_ctx::check_accurate()
193 {
194 attr->fname = jcr->last_fname; /* struct stat is still valid, but not the fname */
195 if (accurate_check_file(jcr, attr, digest)) {
196 jcr->setJobStatus(JS_Differences);
197 }
198 }
199
200 /*
201 * If extracting, close any previous stream
202 */
close_previous_stream()203 bool v_ctx::close_previous_stream()
204 {
205 bool rtn = true;
206 uint8_t buf[CRYPTO_DIGEST_MAX_SIZE];
207 uint32_t len = CRYPTO_DIGEST_MAX_SIZE;
208 char ed1[50], ed2[50];
209
210 /* Reset the win32 filter that strips header stream out of the file */
211 win32filter.init();
212
213 /* Check the size if possible */
214 if (check_size && size >= 0) {
215 if (attr->type == FT_REG && size != (int64_t)attr->statp.st_size) {
216 Dmsg1(50, "Size comparison failed for %s\n", jcr->last_fname);
217 Jmsg(jcr, M_INFO, 0,
218 _(" st_size differs on \"%s\". Vol: %s File: %s\n"),
219 jcr->last_fname,
220 edit_int64(size, ed1),
221 edit_int64((int64_t)attr->statp.st_size, ed2));
222 jcr->setJobStatus(JS_Differences);
223 }
224 reset_size();
225 }
226
227 /* Compute the digest and store it */
228 *digest = 0;
229 if (jcr->crypto.digest) {
230 if (!crypto_digest_finalize(jcr->crypto.digest, buf, &len)) {
231 Dmsg1(50, "Unable to finalize digest for %s\n", jcr->last_fname);
232 rtn = false;
233
234 } else {
235 bin_to_base64(digest, sizeof(digest), (char *)buf, len, true);
236 }
237 crypto_digest_free(jcr->crypto.digest);
238 jcr->crypto.digest = NULL;
239 }
240 return rtn;
241 }
242
243 /*
244 * Verify attributes or data of the requested files on the Volume
245 *
246 */
do_verify_volume(JCR * jcr)247 void do_verify_volume(JCR *jcr)
248 {
249 BSOCK *sd, *dir;
250 uint32_t size;
251 uint32_t VolSessionId, VolSessionTime, file_index;
252 char digest[BASE64_SIZE(CRYPTO_DIGEST_MAX_SIZE)];
253 int stat;
254 int bget_ret = 0;
255 char *wbuf; /* write buffer */
256 uint32_t wsize; /* write size */
257 uint32_t rsize; /* read size */
258 bool msg_encrypt = false, do_check_accurate=false;
259 v_ctx vctx(jcr);
260 ATTR *attr = vctx.attr;
261
262 sd = jcr->store_bsock;
263 if (!sd) {
264 Jmsg(jcr, M_FATAL, 0, _("Storage command not issued before Verify.\n"));
265 jcr->setJobStatus(JS_FatalError);
266 return;
267 }
268 dir = jcr->dir_bsock;
269 jcr->setJobStatus(JS_Running);
270
271 LockRes();
272 CLIENT *client = (CLIENT *)GetNextRes(R_CLIENT, NULL);
273 UnlockRes();
274 uint32_t buf_size;
275 if (client) {
276 buf_size = client->max_network_buffer_size;
277 } else {
278 buf_size = 0; /* use default */
279 }
280 if (!sd->set_buffer_size(buf_size, BNET_SETBUF_WRITE)) {
281 jcr->setJobStatus(JS_FatalError);
282 return;
283 }
284 jcr->buf_size = sd->msglen;
285
286 /* use the same buffer size to decompress both gzip and lzo */
287 if (have_libz || have_lzo) {
288 uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100;
289 jcr->compress_buf = get_memory(compress_buf_size);
290 jcr->compress_buf_size = compress_buf_size;
291 }
292
293 GetMsg *fdmsg = get_msg_buffer(jcr, sd, rec_header);
294 fdmsg->start_read_sock();
295 bmessage *bmsg = fdmsg->new_msg(); /* get a message, to exchange with fdmsg */
296
297 /*
298 * Get a record from the Storage daemon
299 */
300 while ((bget_ret = fdmsg->bget_msg(&bmsg)) >= 0 && !job_canceled(jcr)) {
301 /* Remember previous stream type */
302 vctx.prev_stream = vctx.stream;
303
304 /*
305 * First we expect a Stream Record Header
306 */
307 if (sscanf(bmsg->rbuf, rec_header, &VolSessionId, &VolSessionTime, &file_index,
308 &vctx.full_stream, &size) != 5) {
309 Jmsg1(jcr, M_FATAL, 0, _("Record header scan error: %s\n"), bmsg->rbuf);
310 goto bail_out;
311 }
312 vctx.stream = vctx.full_stream & STREAMMASK_TYPE;
313 Dmsg4(30, "Got hdr: FilInx=%d FullStream=%d Stream=%d size=%d.\n",
314 file_index, vctx.full_stream, vctx.stream, size);
315
316 /*
317 * Now we expect the Stream Data
318 */
319 if ((bget_ret = fdmsg->bget_msg(&bmsg)) < 0) {
320 if (bget_ret != BNET_EXT_TERMINATE) {
321 Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror());
322 } else {
323 /* The error has been handled somewhere else, just quit */
324 }
325 goto bail_out;
326 }
327 if (size != ((uint32_t)bmsg->origlen)) {
328 Jmsg2(jcr, M_FATAL, 0, _("Actual data size %d not same as header %d\n"), bmsg->origlen, size);
329 goto bail_out;
330 }
331 Dmsg2(30, "Got stream data %s, len=%d\n", stream_to_ascii(vctx.stream), bmsg->rbuflen);
332
333 /* File Attributes stream */
334 switch (vctx.stream) {
335 case STREAM_UNIX_ATTRIBUTES:
336 case STREAM_UNIX_ATTRIBUTES_EX:
337 Dmsg0(400, "Stream=Unix Attributes.\n");
338 if (!vctx.close_previous_stream()) {
339 goto bail_out;
340 }
341 if (do_check_accurate) {
342 vctx.check_accurate();
343 }
344 /* Next loop, we want to check the file (or we do it with the md5) */
345 do_check_accurate = true;
346
347 /*
348 * Unpack attributes and do sanity check them
349 */
350 if (!unpack_attributes_record(jcr, vctx.stream,
351 bmsg->rbuf, bmsg->rbuflen, attr)) {
352 goto bail_out;
353 }
354
355 attr->data_stream = decode_stat(attr->attr, &attr->statp,
356 sizeof(attr->statp), &attr->LinkFI);
357
358 jcr->lock();
359 jcr->JobFiles++;
360 jcr->num_files_examined++;
361 pm_strcpy(jcr->last_fname, attr->fname); /* last file examined */
362 jcr->unlock();
363
364 if (jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) {
365 /*
366 * Send file attributes to Director
367 * File_index
368 * Stream
369 * Verify Options
370 * Filename (full path)
371 * Encoded attributes
372 * Link name (if type==FT_LNK)
373 * For a directory, link is the same as fname, but with trailing
374 * slash. For a linked file, link is the link.
375 */
376 /* Send file attributes to Director */
377 Dmsg2(200, "send ATTR inx=%d fname=%s\n", jcr->JobFiles, attr->fname);
378 if (attr->type == FT_LNK || attr->type == FT_LNKSAVED) {
379 stat = dir->fsend("%d %d %s %s%c%s%c%s%c", jcr->JobFiles,
380 STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname,
381 0, attr->attr, 0, attr->lname, 0);
382 /* for a deleted record, we set fileindex=0 */
383 } else if (attr->type == FT_DELETED) {
384 stat = dir->fsend("%d %d %s %s%c%s%c%c", 0,
385 STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname,
386 0, attr->attr, 0, 0);
387 } else {
388 stat = dir->fsend("%d %d %s %s%c%s%c%c", jcr->JobFiles,
389 STREAM_UNIX_ATTRIBUTES, "pinsug5", attr->fname,
390 0, attr->attr, 0, 0);
391 }
392 Dmsg2(200, "bfiled>bdird: attribs len=%d: msg=%s\n", dir->msglen, dir->msg);
393 if (!stat) {
394 Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), dir->bstrerror());
395 goto bail_out;
396 }
397 }
398 break;
399
400 /*
401 * Restore stream object is counted, but not restored here
402 */
403 case STREAM_RESTORE_OBJECT:
404 jcr->lock();
405 jcr->JobFiles++;
406 jcr->num_files_examined++;
407 jcr->unlock();
408 break;
409
410 default:
411 break;
412 }
413
414 const char *digest_code = NULL;
415
416 switch(vctx.stream) {
417 case STREAM_MD5_DIGEST:
418 bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_MD5_SIZE, true);
419 digest_code = "MD5";
420 break;
421
422 case STREAM_SHA1_DIGEST:
423 bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA1_SIZE, true);
424 digest_code = "SHA1";
425 break;
426
427 case STREAM_SHA256_DIGEST:
428 bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA256_SIZE, true);
429 digest_code = "SHA256";
430 break;
431
432 case STREAM_SHA512_DIGEST:
433 bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_SHA512_SIZE, true);
434 digest_code = "SHA512";
435 break;
436
437 default:
438 *digest = 0;
439 break;
440 }
441
442 if (digest_code && jcr->getJobLevel() == L_VERIFY_VOLUME_TO_CATALOG) {
443 dir->fsend("%d %d %s *%s-%d*", jcr->JobFiles, vctx.stream,
444 digest, digest_code, jcr->JobFiles);
445
446 } else if (jcr->getJobLevel() == L_VERIFY_DATA) {
447 /* Compare digest */
448 if (vctx.check_chksum && *digest) {
449 /* probably an empty file, we can create an empty crypto session */
450 if (!jcr->crypto.digest) {
451 jcr->crypto.digest = crypto_digest_new(jcr, vctx.digesttype);
452 }
453 vctx.close_previous_stream();
454 if (strncmp(digest, vctx.digest,
455 MIN(sizeof(digest), sizeof(vctx.digest))) != 0)
456 {
457 Jmsg(jcr, M_INFO, 0,
458 _(" %s differs on \"%s\". File=%s Vol=%s\n"),
459 stream_to_ascii(vctx.stream), jcr->last_fname,
460 vctx.digest, digest);
461 jcr->setJobStatus(JS_Differences);
462 Dmsg3(50, "Signature verification failed for %s %s != %s\n",
463 jcr->last_fname, digest, vctx.digest);
464 }
465 if (do_check_accurate) {
466 vctx.check_accurate();
467 do_check_accurate = false; /* Don't do it in the next loop */
468 }
469 }
470
471 /* Compute size and checksum for level=Data */
472 switch (vctx.stream) {
473 case STREAM_ENCRYPTED_FILE_DATA:
474 case STREAM_ENCRYPTED_WIN32_DATA:
475 case STREAM_ENCRYPTED_FILE_GZIP_DATA:
476 case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
477 case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
478 case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
479 if (!msg_encrypt) {
480 Jmsg(jcr, M_WARNING, 0,
481 _("Verification of encrypted file data is not supported.\n"));
482 msg_encrypt = true;
483 }
484 break;
485
486 case STREAM_PLUGIN_DATA:
487 case STREAM_FILE_DATA:
488 case STREAM_SPARSE_DATA:
489 case STREAM_WIN32_DATA:
490 case STREAM_GZIP_DATA:
491 case STREAM_SPARSE_GZIP_DATA:
492 case STREAM_WIN32_GZIP_DATA:
493 case STREAM_COMPRESSED_DATA:
494 case STREAM_SPARSE_COMPRESSED_DATA:
495 case STREAM_WIN32_COMPRESSED_DATA:
496 if (!(attr->type == FT_RAW || attr->type == FT_FIFO || attr->type == FT_REG || attr->type == FT_REGE)) {
497 break;
498 }
499
500 wbuf = bmsg->rbuf;
501 rsize = bmsg->rbuflen;
502 jcr->ReadBytes += rsize;
503 wsize = rsize;
504
505 if (vctx.stream == STREAM_SPARSE_DATA
506 || vctx.stream == STREAM_SPARSE_COMPRESSED_DATA
507 || vctx.stream == STREAM_SPARSE_GZIP_DATA
508 || vctx.full_stream & STREAM_BIT_OFFSETS) {
509 vctx.skip_sparse_header(&wbuf, &wsize);
510 }
511
512 if (vctx.stream == STREAM_GZIP_DATA
513 || vctx.stream == STREAM_SPARSE_GZIP_DATA
514 || vctx.stream == STREAM_WIN32_GZIP_DATA
515 || vctx.stream == STREAM_ENCRYPTED_FILE_GZIP_DATA
516 || vctx.stream == STREAM_COMPRESSED_DATA
517 || vctx.stream == STREAM_SPARSE_COMPRESSED_DATA
518 || vctx.stream == STREAM_WIN32_COMPRESSED_DATA
519 || vctx.stream == STREAM_ENCRYPTED_FILE_COMPRESSED_DATA
520 || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA
521 || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) {
522
523 if (!decompress_data(jcr, vctx.stream, &wbuf, &wsize)) {
524 dequeue_messages(jcr);
525 goto bail_out;
526 }
527 }
528
529 vctx.update_checksum(wbuf, wsize);
530
531 if (vctx.stream == STREAM_WIN32_GZIP_DATA
532 || vctx.stream == STREAM_WIN32_DATA
533 || vctx.stream == STREAM_WIN32_COMPRESSED_DATA
534 || vctx.stream == STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA
535 || vctx.stream == STREAM_ENCRYPTED_WIN32_GZIP_DATA) {
536
537 int64_t wbuf_len = wsize;
538 int64_t wsize64 = 0;
539 if (vctx.win32filter.have_data(&wbuf, &wbuf_len, &wsize64)) {
540 wsize = wsize64;
541 }
542 }
543 jcr->JobBytes += wsize;
544 vctx.update_size(wsize);
545 break;
546
547 /* TODO: Handle data to compute checksums */
548 /* Ignore everything else */
549 default:
550 break;
551 }
552 } /* end switch */
553 } /* end while bnet_get */
554 if (bget_ret == BNET_EXT_TERMINATE) {
555 goto bail_out;
556 }
557 if (!vctx.close_previous_stream()) {
558 goto bail_out;
559 }
560 /* Check the last file */
561 if (do_check_accurate) {
562 vctx.check_accurate();
563 }
564 if (!accurate_finish(jcr)) {
565 goto bail_out;
566 }
567 jcr->setJobStatus(JS_Terminated);
568 goto ok_out;
569
570 bail_out:
571 jcr->setJobStatus(JS_ErrorTerminated);
572
573 ok_out:
574 Dmsg0(DT_DEDUP|215, "wait BufferedMsg\n");
575 fdmsg->wait_read_sock(jcr->is_job_canceled());
576 delete bmsg;
577 free_GetMsg(fdmsg);
578 if (jcr->compress_buf) {
579 free_pool_memory(jcr->compress_buf);
580 jcr->compress_buf = NULL;
581 }
582 /* TODO: We probably want to mark the job as failed if we have errors */
583 Dmsg2(50, "End Verify-Vol. Files=%d Bytes=%" lld "\n", jcr->JobFiles,
584 jcr->JobBytes);
585 }
586