1 /* verify.c --- verification of FSFS filesystems
2 *
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
19 * under the License.
20 * ====================================================================
21 */
22
23 #include "svn_sorts.h"
24 #include "svn_checksum.h"
25 #include "svn_time.h"
26 #include "private/svn_subr_private.h"
27
28 #include "verify.h"
29 #include "fs_fs.h"
30
31 #include "cached_data.h"
32 #include "rep-cache.h"
33 #include "revprops.h"
34 #include "util.h"
35 #include "index.h"
36
37 #include "../libsvn_fs/fs-loader.h"
38
39 #include "svn_private_config.h"
40
41
42 /** Verifying. **/
43
44 /* Baton type expected by verify_walker(). The purpose is to reuse open
45 * rev / pack file handles between calls. Its contents need to be cleaned
46 * periodically to limit resource usage.
47 */
48 typedef struct verify_walker_baton_t
49 {
50 /* number of calls to verify_walker() since the last clean */
51 int iteration_count;
52
53 /* number of files opened since the last clean */
54 int file_count;
55
56 /* progress notification callback to invoke periodically (may be NULL) */
57 svn_fs_progress_notify_func_t notify_func;
58
59 /* baton to use with NOTIFY_FUNC */
60 void *notify_baton;
61
62 /* remember the last revision for which we called notify_func */
63 svn_revnum_t last_notified_revision;
64
65 /* cached hint for successive calls to svn_fs_fs__check_rep() */
66 void *hint;
67
68 /* pool to use for the file handles etc. */
69 apr_pool_t *pool;
70 } verify_walker_baton_t;
71
72 /* Used by svn_fs_fs__verify().
73 Implements svn_fs_fs__walk_rep_reference().walker. */
74 static svn_error_t *
verify_walker(representation_t * rep,void * baton,svn_fs_t * fs,apr_pool_t * scratch_pool)75 verify_walker(representation_t *rep,
76 void *baton,
77 svn_fs_t *fs,
78 apr_pool_t *scratch_pool)
79 {
80 verify_walker_baton_t *walker_baton = baton;
81 void *previous_hint;
82
83 /* notify and free resources periodically */
84 if ( walker_baton->iteration_count > 1000
85 || walker_baton->file_count > 16)
86 {
87 if ( walker_baton->notify_func
88 && rep->revision != walker_baton->last_notified_revision)
89 {
90 walker_baton->notify_func(rep->revision,
91 walker_baton->notify_baton,
92 scratch_pool);
93 walker_baton->last_notified_revision = rep->revision;
94 }
95
96 svn_pool_clear(walker_baton->pool);
97
98 walker_baton->iteration_count = 0;
99 walker_baton->file_count = 0;
100 walker_baton->hint = NULL;
101 }
102
103 /* access the repo data */
104 previous_hint = walker_baton->hint;
105 SVN_ERR(svn_fs_fs__check_rep(rep, fs, &walker_baton->hint,
106 walker_baton->pool));
107
108 /* update resource usage counters */
109 walker_baton->iteration_count++;
110 if (previous_hint != walker_baton->hint)
111 walker_baton->file_count++;
112
113 return SVN_NO_ERROR;
114 }
115
116 /* Verify the rep cache DB's consistency with our rev / pack data.
117 * The function signature is similar to svn_fs_fs__verify.
118 * The values of START and END have already been auto-selected and
119 * verified.
120 */
121 static svn_error_t *
verify_rep_cache(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t end,svn_fs_progress_notify_func_t notify_func,void * notify_baton,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)122 verify_rep_cache(svn_fs_t *fs,
123 svn_revnum_t start,
124 svn_revnum_t end,
125 svn_fs_progress_notify_func_t notify_func,
126 void *notify_baton,
127 svn_cancel_func_t cancel_func,
128 void *cancel_baton,
129 apr_pool_t *pool)
130 {
131 svn_boolean_t exists;
132
133 /* rep-cache verification. */
134 SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool));
135 if (exists)
136 {
137 /* provide a baton to allow the reuse of open file handles between
138 iterations (saves 2/3 of OS level file operations). */
139 verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton));
140 baton->pool = svn_pool_create(pool);
141 baton->last_notified_revision = SVN_INVALID_REVNUM;
142 baton->notify_func = notify_func;
143 baton->notify_baton = notify_baton;
144
145 /* tell the user that we are now ready to do *something* */
146 if (notify_func)
147 notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool);
148
149 /* Do not attempt to walk the rep-cache database if its file does
150 not exist, since doing so would create it --- which may confuse
151 the administrator. Don't take any lock. */
152 SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end,
153 verify_walker, baton,
154 cancel_func, cancel_baton,
155 pool));
156
157 /* walker resource cleanup */
158 svn_pool_destroy(baton->pool);
159 }
160
161 return SVN_NO_ERROR;
162 }
163
164 /* Verify that the MD5 checksum of the data between offsets START and END
165 * in FILE matches the EXPECTED checksum. If there is a mismatch use the
166 * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC
167 * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */
168 static svn_error_t *
verify_index_checksum(apr_file_t * file,const char * name,apr_off_t start,apr_off_t end,svn_checksum_t * expected,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)169 verify_index_checksum(apr_file_t *file,
170 const char *name,
171 apr_off_t start,
172 apr_off_t end,
173 svn_checksum_t *expected,
174 svn_cancel_func_t cancel_func,
175 void *cancel_baton,
176 apr_pool_t *scratch_pool)
177 {
178 unsigned char buffer[SVN__STREAM_CHUNK_SIZE];
179 apr_off_t size = end - start;
180 svn_checksum_t *actual;
181 svn_checksum_ctx_t *context
182 = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool);
183
184 /* Calculate the index checksum. */
185 SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool));
186 while (size > 0)
187 {
188 apr_size_t to_read = size > sizeof(buffer)
189 ? sizeof(buffer)
190 : (apr_size_t)size;
191 SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL,
192 scratch_pool));
193 SVN_ERR(svn_checksum_update(context, buffer, to_read));
194 size -= to_read;
195
196 if (cancel_func)
197 SVN_ERR(cancel_func(cancel_baton));
198 }
199
200 SVN_ERR(svn_checksum_final(&actual, context, scratch_pool));
201
202 /* Verify that it matches the expected checksum. */
203 if (!svn_checksum_match(expected, actual))
204 {
205 const char *file_name;
206
207 SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool));
208 SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool,
209 _("%s checksum mismatch in file %s"),
210 name, file_name));
211 }
212
213 return SVN_NO_ERROR;
214 }
215
216 /* Verify the MD5 checksums of the index data in the rev / pack file
217 * containing revision START in FS. If given, invoke CANCEL_FUNC with
218 * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary
219 * allocations.
220 */
221 static svn_error_t *
verify_index_checksums(svn_fs_t * fs,svn_revnum_t start,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)222 verify_index_checksums(svn_fs_t *fs,
223 svn_revnum_t start,
224 svn_cancel_func_t cancel_func,
225 void *cancel_baton,
226 apr_pool_t *scratch_pool)
227 {
228 svn_fs_fs__revision_file_t *rev_file;
229
230 /* Open the rev / pack file and read the footer */
231 SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start,
232 scratch_pool, scratch_pool));
233 SVN_ERR(svn_fs_fs__auto_read_footer(rev_file));
234
235 /* Verify the index contents against the checksum from the footer. */
236 SVN_ERR(verify_index_checksum(rev_file->file, "L2P index",
237 rev_file->l2p_offset, rev_file->p2l_offset,
238 rev_file->l2p_checksum,
239 cancel_func, cancel_baton, scratch_pool));
240 SVN_ERR(verify_index_checksum(rev_file->file, "P2L index",
241 rev_file->p2l_offset, rev_file->footer_offset,
242 rev_file->p2l_checksum,
243 cancel_func, cancel_baton, scratch_pool));
244
245 /* Done. */
246 SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
247
248 return SVN_NO_ERROR;
249 }
250
251 /* Verify that for all log-to-phys index entries for revisions START to
252 * START + COUNT-1 in FS there is a consistent entry in the phys-to-log
253 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular
254 * intervals. Use POOL for allocations.
255 */
256 static svn_error_t *
compare_l2p_to_p2l_index(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t count,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)257 compare_l2p_to_p2l_index(svn_fs_t *fs,
258 svn_revnum_t start,
259 svn_revnum_t count,
260 svn_cancel_func_t cancel_func,
261 void *cancel_baton,
262 apr_pool_t *pool)
263 {
264 svn_revnum_t i;
265 apr_pool_t *iterpool = svn_pool_create(pool);
266 apr_array_header_t *max_ids;
267
268 /* common file access structure */
269 svn_fs_fs__revision_file_t *rev_file;
270 SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool,
271 iterpool));
272
273 /* determine the range of items to check for each revision */
274 SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool,
275 iterpool));
276
277 /* check all items in all revisions if the given range */
278 for (i = 0; i < max_ids->nelts; ++i)
279 {
280 apr_uint64_t k;
281 apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t);
282 svn_revnum_t revision = start + i;
283
284 for (k = 0; k < max_id; ++k)
285 {
286 apr_off_t offset;
287 svn_fs_fs__p2l_entry_t *p2l_entry;
288 svn_pool_clear(iterpool);
289
290 /* get L2P entry. Ignore unused entries. */
291 SVN_ERR(svn_fs_fs__item_offset(&offset, fs, rev_file, revision,
292 NULL, k, iterpool));
293 if (offset == -1)
294 continue;
295
296 /* find the corresponding P2L entry */
297 SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, rev_file,
298 revision, offset, iterpool,
299 iterpool));
300
301 if (p2l_entry == NULL)
302 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
303 NULL,
304 _("p2l index entry not found for "
305 "PHYS %s returned by "
306 "l2p index for LOG r%ld:i%ld"),
307 apr_off_t_toa(pool, offset),
308 revision, (long)k);
309
310 if ( p2l_entry->item.number != k
311 || p2l_entry->item.revision != revision)
312 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
313 NULL,
314 _("p2l index info LOG r%ld:i%ld"
315 " does not match "
316 "l2p index for LOG r%ld:i%ld"),
317 p2l_entry->item.revision,
318 (long)p2l_entry->item.number,
319 revision, (long)k);
320 }
321
322 if (cancel_func)
323 SVN_ERR(cancel_func(cancel_baton));
324 }
325
326 svn_pool_destroy(iterpool);
327
328 SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
329
330 return SVN_NO_ERROR;
331 }
332
333 /* Verify that for all phys-to-log index entries for revisions START to
334 * START + COUNT-1 in FS there is a consistent entry in the log-to-phys
335 * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular
336 * intervals. Use POOL for allocations.
337 *
338 * Please note that we can only check on pack / rev file granularity and
339 * must only be called for a single rev / pack file.
340 */
341 static svn_error_t *
compare_p2l_to_l2p_index(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t count,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)342 compare_p2l_to_l2p_index(svn_fs_t *fs,
343 svn_revnum_t start,
344 svn_revnum_t count,
345 svn_cancel_func_t cancel_func,
346 void *cancel_baton,
347 apr_pool_t *pool)
348 {
349 fs_fs_data_t *ffd = fs->fsap_data;
350 apr_pool_t *iterpool = svn_pool_create(pool);
351 apr_off_t max_offset;
352 apr_off_t offset = 0;
353
354 /* common file access structure */
355 svn_fs_fs__revision_file_t *rev_file;
356 SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool,
357 iterpool));
358
359 /* get the size of the rev / pack file as covered by the P2L index */
360 SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start,
361 pool));
362
363 /* for all offsets in the file, get the P2L index entries and check
364 them against the L2P index */
365 for (offset = 0; offset < max_offset; )
366 {
367 apr_array_header_t *entries;
368 svn_fs_fs__p2l_entry_t *last_entry;
369 int i;
370
371 svn_pool_clear(iterpool);
372
373 /* get all entries for the current block */
374 SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start,
375 offset, ffd->p2l_page_size,
376 iterpool, iterpool));
377 if (entries->nelts == 0)
378 return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION,
379 NULL,
380 _("p2l does not cover offset %s"
381 " for revision %ld"),
382 apr_off_t_toa(pool, offset), start);
383
384 /* process all entries (and later continue with the next block) */
385 last_entry
386 = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_fs__p2l_entry_t);
387 offset = last_entry->offset + last_entry->size;
388
389 for (i = 0; i < entries->nelts; ++i)
390 {
391 svn_fs_fs__p2l_entry_t *entry
392 = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t);
393
394 /* check all sub-items for consist entries in the L2P index */
395 if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED)
396 {
397 /* There is no L2P entry for unused rev file sections.
398 * And its P2L index data is hardly ever used. But we
399 * should still check whether someone tempered with it. */
400 if ( entry->item.revision != SVN_INVALID_REVNUM
401 && ( entry->item.revision < start
402 || entry->item.revision >= start + count))
403 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
404 NULL,
405 _("Empty P2L entry for PHYS %s "
406 "refers to revision %ld outside "
407 "the rev / pack file (%ld-%ld)"),
408 apr_off_t_toa(pool, entry->offset),
409 entry->item.revision,
410 start, start + count - 1);
411 }
412 else
413 {
414 apr_off_t l2p_offset;
415 SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, rev_file,
416 entry->item.revision, NULL,
417 entry->item.number, iterpool));
418
419 if (l2p_offset != entry->offset)
420 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
421 NULL,
422 _("l2p index entry PHYS %s"
423 "does not match p2l index value "
424 "LOG r%ld:i%ld for PHYS %s"),
425 apr_off_t_toa(pool, l2p_offset),
426 entry->item.revision,
427 (long)entry->item.number,
428 apr_off_t_toa(pool, entry->offset));
429 }
430 }
431
432 if (cancel_func)
433 SVN_ERR(cancel_func(cancel_baton));
434 }
435
436 svn_pool_destroy(iterpool);
437
438 SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
439
440 return SVN_NO_ERROR;
441 }
442
443 /* Items smaller than this can be read at once into a buffer and directly
444 * be checksummed. Larger items require stream processing.
445 * Must be a multiple of 8. */
446 #define STREAM_THRESHOLD 4096
447
448 /* Verify that the next SIZE bytes read from FILE are NUL.
449 * SIZE must not exceed STREAM_THRESHOLD. Use POOL for allocations.
450 */
451 static svn_error_t *
expect_buffer_nul(apr_file_t * file,apr_off_t size,apr_pool_t * pool)452 expect_buffer_nul(apr_file_t *file,
453 apr_off_t size,
454 apr_pool_t *pool)
455 {
456 union
457 {
458 unsigned char buffer[STREAM_THRESHOLD];
459 apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)];
460 } data;
461
462 apr_size_t i;
463 SVN_ERR_ASSERT(size <= STREAM_THRESHOLD);
464
465 /* read the whole data block; error out on failure */
466 data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0;
467 SVN_ERR(svn_io_file_read_full2(file, data.buffer, (apr_size_t)size, NULL,
468 NULL, pool));
469
470 /* chunky check */
471 for (i = 0; i < size / sizeof(apr_uint64_t); ++i)
472 if (data.chunks[i] != 0)
473 break;
474
475 /* byte-wise check upon mismatch or at the end of the block */
476 for (i *= sizeof(apr_uint64_t); i < size; ++i)
477 if (data.buffer[i] != 0)
478 {
479 const char *file_name;
480 apr_off_t offset;
481
482 SVN_ERR(svn_io_file_name_get(&file_name, file, pool));
483 SVN_ERR(svn_io_file_get_offset(&offset, file, pool));
484 offset -= size - i;
485
486 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
487 _("Empty section in file %s contains "
488 "non-NUL data at offset %s"),
489 file_name, apr_off_t_toa(pool, offset));
490 }
491
492 return SVN_NO_ERROR;
493 }
494
495 /* Verify that the next SIZE bytes read from FILE are NUL.
496 * Use POOL for allocations.
497 */
498 static svn_error_t *
read_all_nul(apr_file_t * file,apr_off_t size,apr_pool_t * pool)499 read_all_nul(apr_file_t *file,
500 apr_off_t size,
501 apr_pool_t *pool)
502 {
503 for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD)
504 SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, pool));
505
506 if (size)
507 SVN_ERR(expect_buffer_nul(file, size, pool));
508
509 return SVN_NO_ERROR;
510 }
511
512 /* Compare the ACTUAL checksum with the one expected by ENTRY.
513 * Return an error in case of mismatch. Use the name of FILE
514 * in error message. Allocate data in POOL.
515 */
516 static svn_error_t *
expected_checksum(apr_file_t * file,svn_fs_fs__p2l_entry_t * entry,apr_uint32_t actual,apr_pool_t * pool)517 expected_checksum(apr_file_t *file,
518 svn_fs_fs__p2l_entry_t *entry,
519 apr_uint32_t actual,
520 apr_pool_t *pool)
521 {
522 if (actual != entry->fnv1_checksum)
523 {
524 const char *file_name;
525
526 SVN_ERR(svn_io_file_name_get(&file_name, file, pool));
527 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
528 _("Checksum mismatch in item at offset %s of "
529 "length %s bytes in file %s"),
530 apr_off_t_toa(pool, entry->offset),
531 apr_off_t_toa(pool, entry->size), file_name);
532 }
533
534 return SVN_NO_ERROR;
535 }
536
537 /* Verify that the FNV checksum over the next ENTRY->SIZE bytes read
538 * from FILE will match ENTRY's expected checksum. SIZE must not
539 * exceed STREAM_THRESHOLD. Use POOL for allocations.
540 */
541 static svn_error_t *
expected_buffered_checksum(apr_file_t * file,svn_fs_fs__p2l_entry_t * entry,apr_pool_t * pool)542 expected_buffered_checksum(apr_file_t *file,
543 svn_fs_fs__p2l_entry_t *entry,
544 apr_pool_t *pool)
545 {
546 unsigned char buffer[STREAM_THRESHOLD];
547 SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD);
548
549 SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size,
550 NULL, NULL, pool));
551 SVN_ERR(expected_checksum(file, entry,
552 svn__fnv1a_32x4(buffer, (apr_size_t)entry->size),
553 pool));
554
555 return SVN_NO_ERROR;
556 }
557
558 /* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from
559 * FILE will match ENTRY's expected checksum. Use POOL for allocations.
560 */
561 static svn_error_t *
expected_streamed_checksum(apr_file_t * file,svn_fs_fs__p2l_entry_t * entry,apr_pool_t * pool)562 expected_streamed_checksum(apr_file_t *file,
563 svn_fs_fs__p2l_entry_t *entry,
564 apr_pool_t *pool)
565 {
566 unsigned char buffer[STREAM_THRESHOLD];
567 svn_checksum_t *checksum;
568 svn_checksum_ctx_t *context
569 = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool);
570 apr_off_t size = entry->size;
571
572 while (size > 0)
573 {
574 apr_size_t to_read = size > sizeof(buffer)
575 ? sizeof(buffer)
576 : (apr_size_t)size;
577 SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL,
578 pool));
579 SVN_ERR(svn_checksum_update(context, buffer, to_read));
580 size -= to_read;
581 }
582
583 SVN_ERR(svn_checksum_final(&checksum, context, pool));
584 SVN_ERR(expected_checksum(file, entry,
585 ntohl(*(const apr_uint32_t *)checksum->digest),
586 pool));
587
588 return SVN_NO_ERROR;
589 }
590
591 /* Verify that for all phys-to-log index entries for revisions START to
592 * START + COUNT-1 in FS match the actual pack / rev file contents.
593 * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals.
594 * Use POOL for allocations.
595 *
596 * Please note that we can only check on pack / rev file granularity and
597 * must only be called for a single rev / pack file.
598 */
599 static svn_error_t *
compare_p2l_to_rev(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t count,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)600 compare_p2l_to_rev(svn_fs_t *fs,
601 svn_revnum_t start,
602 svn_revnum_t count,
603 svn_cancel_func_t cancel_func,
604 void *cancel_baton,
605 apr_pool_t *pool)
606 {
607 fs_fs_data_t *ffd = fs->fsap_data;
608 apr_pool_t *iterpool = svn_pool_create(pool);
609 apr_off_t max_offset;
610 apr_off_t offset = 0;
611 svn_fs_fs__revision_file_t *rev_file;
612
613 /* open the pack / rev file that is covered by the p2l index */
614 SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool,
615 iterpool));
616
617 /* check file size vs. range covered by index */
618 SVN_ERR(svn_fs_fs__auto_read_footer(rev_file));
619 SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start,
620 pool));
621
622 if (rev_file->l2p_offset != max_offset)
623 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL,
624 _("File size of %s for revision r%ld does "
625 "not match p2l index size of %s"),
626 apr_off_t_toa(pool, rev_file->l2p_offset), start,
627 apr_off_t_toa(pool, max_offset));
628
629 SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0,
630 pool));
631
632 /* for all offsets in the file, get the P2L index entries and check
633 them against the L2P index */
634 for (offset = 0; offset < max_offset; )
635 {
636 apr_array_header_t *entries;
637 int i;
638
639 svn_pool_clear(iterpool);
640
641 /* get all entries for the current block */
642 SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start,
643 offset, ffd->p2l_page_size,
644 iterpool, iterpool));
645
646 /* The above might have moved the file pointer.
647 * Ensure we actually start reading at OFFSET. */
648 SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size,
649 NULL, offset, iterpool));
650
651 /* process all entries (and later continue with the next block) */
652 for (i = 0; i < entries->nelts; ++i)
653 {
654 svn_fs_fs__p2l_entry_t *entry
655 = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t);
656
657 /* skip bits we previously checked */
658 if (i == 0 && entry->offset < offset)
659 continue;
660
661 /* skip zero-sized entries */
662 if (entry->size == 0)
663 continue;
664
665 /* p2l index must cover all rev / pack file offsets exactly once */
666 if (entry->offset != offset)
667 return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT,
668 NULL,
669 _("p2l index entry for revision r%ld"
670 " is non-contiguous between offsets "
671 " %s and %s"),
672 start,
673 apr_off_t_toa(pool, offset),
674 apr_off_t_toa(pool, entry->offset));
675
676 /* Check type <-> item dependencies. */
677
678 /* Entry types must be within the valid range. */
679 if (entry->type >= SVN_FS_FS__ITEM_TYPE_ANY_REP)
680 return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION,
681 NULL,
682 _("p2l index entry for revision r%ld"
683 " at offset %s contains invalid item"
684 " type %u"),
685 start,
686 apr_off_t_toa(pool, offset),
687 (unsigned int)entry->type);
688
689 /* There can be only one changes entry and that has a fixed type
690 * and item number. Its presence and parse-ability will be checked
691 * during later stages of the verification process. */
692 if ( (entry->type == SVN_FS_FS__ITEM_TYPE_CHANGES)
693 != (entry->item.number == SVN_FS_FS__ITEM_INDEX_CHANGES))
694 return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION,
695 NULL,
696 _("p2l index entry for changes in"
697 " revision r%ld is item"
698 " %"APR_UINT64_T_FMT
699 " of type %u at offset %s"),
700 entry->item.revision,
701 entry->item.number,
702 (unsigned int)entry->type,
703 apr_off_t_toa(pool, offset));
704
705 /* Check contents. */
706 if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED)
707 {
708 /* Empty sections must contain NUL bytes only.
709 * Beware of the filler at the end of the p2l index. */
710 if (entry->offset != max_offset)
711 SVN_ERR(read_all_nul(rev_file->file, entry->size, pool));
712 }
713 else
714 {
715 /* Generic contents check against checksum. */
716 if (entry->size < STREAM_THRESHOLD)
717 SVN_ERR(expected_buffered_checksum(rev_file->file, entry,
718 pool));
719 else
720 SVN_ERR(expected_streamed_checksum(rev_file->file, entry,
721 pool));
722 }
723
724 /* advance offset */
725 offset += entry->size;
726 }
727
728 if (cancel_func)
729 SVN_ERR(cancel_func(cancel_baton));
730 }
731
732 svn_pool_destroy(iterpool);
733
734 SVN_ERR(svn_fs_fs__close_revision_file(rev_file));
735
736 return SVN_NO_ERROR;
737 }
738
739 /* Verify that the revprops of the revisions START to END in FS can be
740 * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals.
741 *
742 * The values of START and END have already been auto-selected and
743 * verified.
744 */
745 static svn_error_t *
verify_revprops(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t end,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)746 verify_revprops(svn_fs_t *fs,
747 svn_revnum_t start,
748 svn_revnum_t end,
749 svn_cancel_func_t cancel_func,
750 void *cancel_baton,
751 apr_pool_t *pool)
752 {
753 svn_revnum_t revision;
754 apr_pool_t *iterpool = svn_pool_create(pool);
755
756 /* Invalidate the revprop cache once.
757 * Use the cache inside the loop to speed up packed revprop access. */
758 svn_fs_fs__reset_revprop_cache(fs);
759
760 for (revision = start; revision < end; ++revision)
761 {
762 svn_string_t *date;
763 apr_time_t timetemp;
764
765 svn_pool_clear(iterpool);
766
767 /* Access the svn:date revprop.
768 * This implies parsing all revprops for that revision. */
769 SVN_ERR(svn_fs_fs__revision_prop(&date, fs, revision,
770 SVN_PROP_REVISION_DATE, FALSE,
771 iterpool, iterpool));
772
773 /* The time stamp is the only revprop that, if given, needs to
774 * have a valid content. */
775 if (date)
776 SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool));
777
778 if (cancel_func)
779 SVN_ERR(cancel_func(cancel_baton));
780 }
781
782 svn_pool_destroy(iterpool);
783
784 return SVN_NO_ERROR;
785 }
786
787 static svn_revnum_t
pack_size(svn_fs_t * fs,svn_revnum_t rev)788 pack_size(svn_fs_t *fs, svn_revnum_t rev)
789 {
790 fs_fs_data_t *ffd = fs->fsap_data;
791
792 return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1;
793 }
794
795 /* Verify that on-disk representation has not been tempered with (in a way
796 * that leaves the repository in a corrupted state). This compares log-to-
797 * phys with phys-to-log indexes, verifies the low-level checksums and
798 * checks that all revprops are available. The function signature is
799 * similar to svn_fs_fs__verify.
800 *
801 * The values of START and END have already been auto-selected and
802 * verified. You may call this for format7 or higher repos.
803 */
804 static svn_error_t *
verify_f7_metadata_consistency(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t end,svn_fs_progress_notify_func_t notify_func,void * notify_baton,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)805 verify_f7_metadata_consistency(svn_fs_t *fs,
806 svn_revnum_t start,
807 svn_revnum_t end,
808 svn_fs_progress_notify_func_t notify_func,
809 void *notify_baton,
810 svn_cancel_func_t cancel_func,
811 void *cancel_baton,
812 apr_pool_t *pool)
813 {
814 fs_fs_data_t *ffd = fs->fsap_data;
815 svn_revnum_t revision, next_revision;
816 apr_pool_t *iterpool = svn_pool_create(pool);
817
818 for (revision = start; revision <= end; revision = next_revision)
819 {
820 svn_error_t *err = SVN_NO_ERROR;
821
822 svn_revnum_t count = pack_size(fs, revision);
823 svn_revnum_t pack_start = svn_fs_fs__packed_base_rev(fs, revision);
824 svn_revnum_t pack_end = pack_start + count;
825
826 svn_pool_clear(iterpool);
827
828 if (notify_func && (pack_start % ffd->max_files_per_dir == 0))
829 notify_func(pack_start, notify_baton, iterpool);
830
831 /* Check for external corruption to the indexes. */
832 err = verify_index_checksums(fs, pack_start, cancel_func,
833 cancel_baton, iterpool);
834
835 /* two-way index check */
836 if (!err)
837 err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start,
838 cancel_func, cancel_baton, iterpool);
839 if (!err)
840 err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start,
841 cancel_func, cancel_baton, iterpool);
842
843 /* verify in-index checksums and types vs. actual rev / pack files */
844 if (!err)
845 err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start,
846 cancel_func, cancel_baton, iterpool);
847
848 /* ensure that revprops are available and accessible */
849 if (!err)
850 err = verify_revprops(fs, pack_start, pack_end,
851 cancel_func, cancel_baton, iterpool);
852
853 /* concurrent packing is one of the reasons why verification may fail.
854 Make sure, we operate on up-to-date information. */
855 if (err)
856 {
857 svn_error_t *err2
858 = svn_fs_fs__read_min_unpacked_rev(&ffd->min_unpacked_rev,
859 fs, pool);
860
861 /* Be careful to not leak ERR. */
862 if (err2)
863 return svn_error_trace(svn_error_compose_create(err, err2));
864 }
865
866 /* retry the whole shard if it got packed in the meantime */
867 if (err && count != pack_size(fs, revision))
868 {
869 svn_error_clear(err);
870
871 /* We could simply assign revision here but the code below is
872 more intuitive to maintainers. */
873 next_revision = svn_fs_fs__packed_base_rev(fs, revision);
874 }
875 else
876 {
877 SVN_ERR(err);
878 next_revision = pack_end;
879 }
880 }
881
882 svn_pool_destroy(iterpool);
883
884 return SVN_NO_ERROR;
885 }
886
887 svn_error_t *
svn_fs_fs__verify(svn_fs_t * fs,svn_revnum_t start,svn_revnum_t end,svn_fs_progress_notify_func_t notify_func,void * notify_baton,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * pool)888 svn_fs_fs__verify(svn_fs_t *fs,
889 svn_revnum_t start,
890 svn_revnum_t end,
891 svn_fs_progress_notify_func_t notify_func,
892 void *notify_baton,
893 svn_cancel_func_t cancel_func,
894 void *cancel_baton,
895 apr_pool_t *pool)
896 {
897 fs_fs_data_t *ffd = fs->fsap_data;
898
899 /* Input validation. */
900 if (! SVN_IS_VALID_REVNUM(start))
901 start = 0;
902 if (! SVN_IS_VALID_REVNUM(end))
903 {
904 SVN_ERR(svn_fs_fs__youngest_rev(&end, fs, pool));
905 }
906
907 SVN_ERR(svn_fs_fs__ensure_revision_exists(start, fs, pool));
908 SVN_ERR(svn_fs_fs__ensure_revision_exists(end, fs, pool));
909
910 /* log/phys index consistency. We need to check them first to make
911 sure we can access the rev / pack files in format7. */
912 if (svn_fs_fs__use_log_addressing(fs))
913 SVN_ERR(verify_f7_metadata_consistency(fs, start, end,
914 notify_func, notify_baton,
915 cancel_func, cancel_baton, pool));
916
917 /* rep cache consistency */
918 if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT)
919 SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton,
920 cancel_func, cancel_baton, pool));
921
922 return SVN_NO_ERROR;
923 }
924