1 /* fs-fs-private-test.c --- tests FSFS's private API
2 *
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
19 * under the License.
20 * ====================================================================
21 */
22
23 #include <stdlib.h>
24 #include <string.h>
25
26 #include "../svn_test.h"
27
28 #include "svn_hash.h"
29 #include "svn_pools.h"
30 #include "svn_props.h"
31 #include "svn_fs.h"
32
33 #include "private/svn_string_private.h"
34 #include "private/svn_fs_fs_private.h"
35 #include "private/svn_subr_private.h"
36
37 #include "../../libsvn_fs_fs/index.h"
38 #include "../../libsvn_fs_fs/rep-cache.h"
39 #include "../../libsvn_fs/fs-loader.h"
40
41 #include "../svn_test_fs.h"
42
43
44
45 /* Utility functions */
46
47 /* Create a repo under REPO_NAME using OPTS. Allocate the repository in
48 * RESULT_POOL and return it in *REPOS. Set *REV to the revision containing
49 * the Greek tree addition. Use SCRATCH_POOL for temporary allocations.
50 */
51 static svn_error_t *
create_greek_repo(svn_repos_t ** repos,svn_revnum_t * rev,const svn_test_opts_t * opts,const char * repo_name,apr_pool_t * result_pool,apr_pool_t * scratch_pool)52 create_greek_repo(svn_repos_t **repos,
53 svn_revnum_t *rev,
54 const svn_test_opts_t *opts,
55 const char *repo_name,
56 apr_pool_t *result_pool,
57 apr_pool_t *scratch_pool)
58 {
59 svn_fs_t *fs;
60 svn_fs_txn_t *txn;
61 svn_fs_root_t *txn_root;
62
63 /* Create a filesystem */
64 SVN_ERR(svn_test__create_repos(repos, repo_name, opts, result_pool));
65 fs = svn_repos_fs(*repos);
66
67 /* Add the Greek tree */
68 SVN_ERR(svn_fs_begin_txn(&txn, fs, 0, scratch_pool));
69 SVN_ERR(svn_fs_txn_root(&txn_root, txn, scratch_pool));
70 SVN_ERR(svn_test__create_greek_tree(txn_root, scratch_pool));
71 SVN_ERR(svn_fs_commit_txn(NULL, rev, txn, scratch_pool));
72 SVN_TEST_ASSERT(SVN_IS_VALID_REVNUM(*rev));
73
74 return SVN_NO_ERROR;
75 }
76
77
78 /* ------------------------------------------------------------------------ */
79
80 #define REPO_NAME "test-repo-get-repo-stats-test"
81
82 static svn_error_t *
verify_representation_stats(const svn_fs_fs__representation_stats_t * stats,apr_uint64_t expected_count)83 verify_representation_stats(const svn_fs_fs__representation_stats_t *stats,
84 apr_uint64_t expected_count)
85 {
86 /* Small items, no packing (but inefficiency due to packing attempt). */
87 SVN_TEST_ASSERT(stats->total.count == expected_count);
88 SVN_TEST_ASSERT( stats->total.packed_size >= 10 * expected_count
89 && stats->total.packed_size <= 1000 * expected_count);
90 /* Expect the packed size to be sane, keeping in mind that it might
91 * be less or more than the expanded size due differences in the
92 * compression algorithms or options such as directory deltification. */
93 SVN_TEST_ASSERT(stats->total.packed_size <= 2 * stats->total.expanded_size);
94 SVN_TEST_ASSERT( stats->total.overhead_size >= 5 * expected_count
95 && stats->total.overhead_size <= 100 * expected_count);
96
97 /* Rep sharing has no effect on the Greek tree. */
98 SVN_TEST_ASSERT(stats->total.count == stats->uniques.count);
99 SVN_TEST_ASSERT(stats->total.packed_size == stats->uniques.packed_size);
100 SVN_TEST_ASSERT(stats->total.expanded_size == stats->uniques.expanded_size);
101 SVN_TEST_ASSERT(stats->total.overhead_size == stats->uniques.overhead_size);
102
103 SVN_TEST_ASSERT(stats->shared.count == 0);
104 SVN_TEST_ASSERT(stats->shared.packed_size == 0);
105 SVN_TEST_ASSERT(stats->shared.expanded_size == 0);
106 SVN_TEST_ASSERT(stats->shared.overhead_size == 0);
107
108 /* No rep sharing. */
109 SVN_TEST_ASSERT(stats->references == stats->total.count);
110 SVN_TEST_ASSERT(stats->expanded_size == stats->total.expanded_size);
111
112 /* Reasonable delta chain lengths */
113 SVN_TEST_ASSERT( stats->chain_len >= stats->total.count
114 && stats->chain_len <= 5 * stats->total.count);
115
116 return SVN_NO_ERROR;
117 }
118
119 static svn_error_t *
verify_node_stats(const svn_fs_fs__node_stats_t * node_stats,apr_uint64_t expected_count)120 verify_node_stats(const svn_fs_fs__node_stats_t *node_stats,
121 apr_uint64_t expected_count)
122 {
123 SVN_TEST_ASSERT(node_stats->count == expected_count);
124 SVN_TEST_ASSERT( node_stats->size > 100 * node_stats->count
125 && node_stats->size < 1000 * node_stats->count);
126
127 return SVN_NO_ERROR;
128 }
129
130 static svn_error_t *
verify_large_change(const svn_fs_fs__large_change_info_t * change,svn_revnum_t revision)131 verify_large_change(const svn_fs_fs__large_change_info_t *change,
132 svn_revnum_t revision)
133 {
134 if (change->revision == SVN_INVALID_REVNUM)
135 {
136 /* Unused entry due to the Greek tree being small. */
137 SVN_TEST_ASSERT(change->path->len == 0);
138 SVN_TEST_ASSERT(change->size == 0);
139 }
140 else if (strcmp(change->path->data, "/") == 0)
141 {
142 /* The root folder nodes are always there, i.e. aren't in the
143 * Greek tree "do add" list. */
144 SVN_TEST_ASSERT( SVN_IS_VALID_REVNUM(change->revision)
145 && change->revision <= revision);
146 }
147 else
148 {
149 const struct svn_test__tree_entry_t *node;
150 for (node = svn_test__greek_tree_nodes; node->path; node++)
151 if (strcmp(node->path, change->path->data + 1) == 0)
152 {
153 SVN_TEST_ASSERT(change->revision == revision);
154
155 /* When checking content sizes, keep in mind the optional
156 * SVNDIFF overhead.*/
157 if (node->contents)
158 SVN_TEST_ASSERT( change->size >= strlen(node->contents)
159 && change->size <= 12 + strlen(node->contents));
160
161 return SVN_NO_ERROR;
162 }
163
164 SVN_TEST_ASSERT(!"Change is part of Greek tree");
165 }
166
167 return SVN_NO_ERROR;
168 }
169
170 static svn_error_t *
verify_histogram(const svn_fs_fs__histogram_t * histogram)171 verify_histogram(const svn_fs_fs__histogram_t *histogram)
172 {
173 apr_uint64_t sum_count = 0;
174 apr_uint64_t sum_size = 0;
175
176 int i;
177 for (i = 0; i < 64; ++i)
178 {
179 svn_fs_fs__histogram_line_t line = histogram->lines[i];
180
181 if (i > 10 || i < 1)
182 SVN_TEST_ASSERT(line.sum == 0 && line.count == 0);
183 else
184 SVN_TEST_ASSERT( line.sum >= (line.count << (i-1))
185 && line.sum <= (line.count << i));
186
187 sum_count += line.count;
188 sum_size += line.sum;
189 }
190
191 SVN_TEST_ASSERT(histogram->total.count == sum_count);
192 SVN_TEST_ASSERT(histogram->total.sum == sum_size);
193
194 return SVN_NO_ERROR;
195 }
196
197 static svn_error_t *
get_repo_stats(const svn_test_opts_t * opts,apr_pool_t * pool)198 get_repo_stats(const svn_test_opts_t *opts,
199 apr_pool_t *pool)
200 {
201 svn_repos_t *repos;
202 svn_revnum_t rev;
203 apr_size_t i;
204 svn_fs_fs__extension_info_t *extension_info;
205 svn_fs_fs__ioctl_get_stats_input_t input = {0};
206 svn_fs_fs__ioctl_get_stats_output_t *output;
207 const svn_fs_fs__stats_t *stats;
208
209 /* Bail (with success) on known-untestable scenarios */
210 if (strcmp(opts->fs_type, "fsfs") != 0)
211 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
212 "this will test FSFS repositories only");
213
214 /* Create a filesystem */
215 SVN_ERR(create_greek_repo(&repos, &rev, opts, REPO_NAME, pool, pool));
216
217 /* Gather statistics info on that repo. */
218 SVN_ERR(svn_fs_ioctl(svn_repos_fs(repos), SVN_FS_FS__IOCTL_GET_STATS,
219 &input, (void**)&output, NULL, NULL, pool, pool));
220 stats = output->stats;
221
222 /* Check that the stats make sense. */
223 SVN_TEST_ASSERT(stats->total_size > 1000 && stats->total_size < 10000);
224 SVN_TEST_ASSERT(stats->revision_count == 2);
225 SVN_TEST_ASSERT(stats->change_count == 20);
226 SVN_TEST_ASSERT(stats->change_len > 500 && stats->change_len < 2000);
227
228 /* Check representation stats. */
229 SVN_ERR(verify_representation_stats(&stats->total_rep_stats, 20));
230 SVN_ERR(verify_representation_stats(&stats->file_rep_stats, 12));
231 SVN_ERR(verify_representation_stats(&stats->dir_rep_stats, 8));
232 SVN_ERR(verify_representation_stats(&stats->file_prop_rep_stats, 0));
233 SVN_ERR(verify_representation_stats(&stats->dir_prop_rep_stats, 0));
234
235 /* Check node stats against rep stats. */
236 SVN_ERR(verify_node_stats(&stats->total_node_stats, 22));
237 SVN_ERR(verify_node_stats(&stats->file_node_stats, 12));
238 SVN_ERR(verify_node_stats(&stats->dir_node_stats, 10));
239
240 /* Check largest changes. */
241 SVN_TEST_ASSERT(stats->largest_changes->count == 64);
242 SVN_TEST_ASSERT(stats->largest_changes->min_size == 0);
243
244 for (i = 0; i < stats->largest_changes->count; ++i)
245 SVN_ERR(verify_large_change(stats->largest_changes->changes[i], rev));
246
247 /* Check histograms. */
248 SVN_ERR(verify_histogram(&stats->rep_size_histogram));
249 SVN_ERR(verify_histogram(&stats->node_size_histogram));
250 SVN_ERR(verify_histogram(&stats->added_rep_size_histogram));
251 SVN_ERR(verify_histogram(&stats->added_node_size_histogram));
252 SVN_ERR(verify_histogram(&stats->unused_rep_histogram));
253 SVN_ERR(verify_histogram(&stats->file_histogram));
254 SVN_ERR(verify_histogram(&stats->file_rep_histogram));
255 SVN_ERR(verify_histogram(&stats->file_prop_histogram));
256 SVN_ERR(verify_histogram(&stats->file_prop_rep_histogram));
257 SVN_ERR(verify_histogram(&stats->dir_histogram));
258 SVN_ERR(verify_histogram(&stats->dir_rep_histogram));
259 SVN_ERR(verify_histogram(&stats->dir_prop_histogram));
260 SVN_ERR(verify_histogram(&stats->dir_prop_rep_histogram));
261
262 /* No file in the Greek tree has an externsion */
263 SVN_TEST_ASSERT(apr_hash_count(stats->by_extension) == 1);
264 extension_info = svn_hash_gets(stats->by_extension, "(none)");
265 SVN_TEST_ASSERT(extension_info);
266
267 SVN_ERR(verify_histogram(&extension_info->rep_histogram));
268 SVN_ERR(verify_histogram(&extension_info->node_histogram));
269
270 return SVN_NO_ERROR;
271 }
272
273 #undef REPO_NAME
274
275 /* ------------------------------------------------------------------------ */
276
277 #define REPO_NAME "test-repo-dump-index-test"
278
279 typedef struct dump_baton_t
280 {
281 /* Number of callback invocations so far */
282 int invocations;
283
284 /* Rev file location we expect to be reported next */
285 apr_off_t offset;
286
287 /* All items must be from this revision. */
288 svn_revnum_t revision;
289
290 /* Track the item numbers we have already seen. */
291 svn_bit_array__t *numbers_seen;
292 } dump_baton_t;
293
294 static svn_error_t *
dump_index_entry(const svn_fs_fs__p2l_entry_t * entry,void * baton_p,apr_pool_t * scratch_pool)295 dump_index_entry(const svn_fs_fs__p2l_entry_t *entry,
296 void *baton_p,
297 apr_pool_t *scratch_pool)
298 {
299 dump_baton_t *baton = baton_p;
300
301 /* Count invocations. */
302 baton->invocations++;
303
304 /* We expect a report of contiguous non-empty items. */
305 SVN_TEST_ASSERT(entry->offset == baton->offset);
306 SVN_TEST_ASSERT(entry->size > 0 && entry->size < 1000);
307 baton->offset += entry->size;
308
309 /* Type must be valid. */
310 SVN_TEST_ASSERT( entry->type > SVN_FS_FS__ITEM_TYPE_UNUSED
311 && entry->type <= SVN_FS_FS__ITEM_TYPE_CHANGES);
312
313 /* We expect all items to be from the specified revision. */
314 SVN_TEST_ASSERT(entry->item.revision == baton->revision);
315
316 /* Item numnber must be plausibly small and unique. */
317 SVN_TEST_ASSERT(entry->item.number < 100);
318 SVN_TEST_ASSERT(!svn_bit_array__get(baton->numbers_seen,
319 (apr_size_t)entry->item.number));
320 svn_bit_array__set(baton->numbers_seen, (apr_size_t)entry->item.number, 1);
321
322 return SVN_NO_ERROR;
323 }
324
325 static svn_error_t *
dump_index(const svn_test_opts_t * opts,apr_pool_t * pool)326 dump_index(const svn_test_opts_t *opts,
327 apr_pool_t *pool)
328 {
329 svn_repos_t *repos;
330 svn_revnum_t rev;
331 dump_baton_t baton;
332 svn_fs_fs__ioctl_dump_index_input_t input = {0};
333
334 /* Bail (with success) on known-untestable scenarios */
335 if (strcmp(opts->fs_type, "fsfs") != 0)
336 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
337 "this will test FSFS repositories only");
338
339 if (opts->server_minor_version && (opts->server_minor_version < 9))
340 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
341 "pre-1.9 SVN doesn't have FSFS indexes");
342
343 /* Create a filesystem */
344 SVN_ERR(create_greek_repo(&repos, &rev, opts, REPO_NAME, pool, pool));
345
346 /* Read the index data for REV from that repo. */
347 baton.invocations = 0;
348 baton.offset = 0;
349 baton.revision = rev;
350 baton.numbers_seen = svn_bit_array__create(100, pool);
351
352 input.revision = rev;
353 input.callback_func = dump_index_entry;
354 input.callback_baton = &baton;
355 SVN_ERR(svn_fs_ioctl(svn_repos_fs(repos), SVN_FS_FS__IOCTL_DUMP_INDEX,
356 &input, NULL, NULL, NULL, pool, pool));
357
358 /* Check that we've got all data (20 noderevs + 20 reps + 1 changes list). */
359 SVN_TEST_ASSERT(baton.invocations == 41);
360
361 return SVN_NO_ERROR;
362 }
363
364 #undef REPO_NAME
365
366 /* ------------------------------------------------------------------------ */
367
368 static svn_error_t *
receive_index(const svn_fs_fs__p2l_entry_t * entry,void * baton,apr_pool_t * scratch_pool)369 receive_index(const svn_fs_fs__p2l_entry_t *entry,
370 void *baton,
371 apr_pool_t *scratch_pool)
372 {
373 apr_array_header_t *entries = baton;
374 APR_ARRAY_PUSH(entries, svn_fs_fs__p2l_entry_t *)
375 = apr_pmemdup(entries->pool, entry, sizeof(*entry));
376
377 return SVN_NO_ERROR;
378 }
379
380 #define REPO_NAME "test-repo-load-index-test"
381
382 static svn_error_t *
load_index(const svn_test_opts_t * opts,apr_pool_t * pool)383 load_index(const svn_test_opts_t *opts, apr_pool_t *pool)
384 {
385 svn_repos_t *repos;
386 svn_revnum_t rev;
387 apr_array_header_t *entries = apr_array_make(pool, 41, sizeof(void *));
388 apr_array_header_t *alt_entries = apr_array_make(pool, 1, sizeof(void *));
389 svn_fs_fs__p2l_entry_t entry;
390 svn_fs_fs__ioctl_dump_index_input_t dump_input = {0};
391 svn_fs_fs__ioctl_load_index_input_t load_input = {0};
392
393 /* Bail (with success) on known-untestable scenarios */
394 if (strcmp(opts->fs_type, "fsfs") != 0)
395 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
396 "this will test FSFS repositories only");
397
398 if (opts->server_minor_version && (opts->server_minor_version < 9))
399 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
400 "pre-1.9 SVN doesn't have FSFS indexes");
401
402 /* Create a filesystem */
403 SVN_ERR(create_greek_repo(&repos, &rev, opts, REPO_NAME, pool, pool));
404
405 /* Read the original index contents for REV in ENTRIES. */
406 dump_input.revision = rev;
407 dump_input.callback_func = receive_index;
408 dump_input.callback_baton = entries;
409 SVN_ERR(svn_fs_ioctl(svn_repos_fs(repos), SVN_FS_FS__IOCTL_DUMP_INDEX,
410 &dump_input, NULL, NULL, NULL, pool, pool));
411
412 /* Replace it with an index that declares the whole revision contents as
413 * "unused". */
414 entry = *APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_fs__p2l_entry_t *);
415 entry.size += entry.offset;
416 entry.offset = 0;
417 entry.type = SVN_FS_FS__ITEM_TYPE_UNUSED;
418 entry.item.number = SVN_FS_FS__ITEM_INDEX_UNUSED;
419 entry.item.revision = SVN_INVALID_REVNUM;
420 APR_ARRAY_PUSH(alt_entries, svn_fs_fs__p2l_entry_t *) = &entry;
421
422 load_input.revision = rev;
423 load_input.entries = alt_entries;
424 SVN_ERR(svn_fs_ioctl(svn_repos_fs(repos), SVN_FS_FS__IOCTL_LOAD_INDEX,
425 &load_input, NULL, NULL, NULL, pool, pool));
426
427 SVN_TEST_ASSERT_ERROR(svn_repos_verify_fs3(repos, rev, rev, FALSE, FALSE,
428 NULL, NULL, NULL, NULL, NULL,
429 NULL, pool),
430 SVN_ERR_FS_INDEX_CORRUPTION);
431
432 /* Restore the original index. */
433 load_input.revision = rev;
434 load_input.entries = entries;
435 SVN_ERR(svn_fs_ioctl(svn_repos_fs(repos), SVN_FS_FS__IOCTL_LOAD_INDEX,
436 &load_input, NULL, NULL, NULL, pool, pool));
437 SVN_ERR(svn_repos_verify_fs3(repos, rev, rev, FALSE, FALSE, NULL, NULL,
438 NULL, NULL, NULL, NULL, pool));
439
440 return SVN_NO_ERROR;
441 }
442
443 #undef REPO_NAME
444
445 /* ------------------------------------------------------------------------ */
446
447 static svn_error_t *
build_rep_cache(const svn_test_opts_t * opts,apr_pool_t * pool)448 build_rep_cache(const svn_test_opts_t *opts, apr_pool_t *pool)
449 {
450 svn_fs_t *fs;
451 fs_fs_data_t *ffd;
452 svn_fs_txn_t *txn;
453 svn_fs_root_t *txn_root;
454 svn_revnum_t rev;
455 svn_boolean_t exists;
456 const char *fs_path;
457 svn_fs_fs__ioctl_build_rep_cache_input_t input = {0};
458
459 /* Bail (with success) on known-untestable scenarios */
460 if (strcmp(opts->fs_type, "fsfs") != 0)
461 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
462 "this will test FSFS repositories only");
463
464 if (opts->server_minor_version && (opts->server_minor_version < 6))
465 return svn_error_create(SVN_ERR_TEST_SKIPPED, NULL,
466 "pre-1.6 SVN doesn't support FSFS rep-sharing");
467
468 /* Create a filesystem and explicitly disable rep-sharing. */
469 fs_path = "test-repo-build-rep-cache-test";
470 SVN_ERR(svn_test__create_fs2(&fs, fs_path, opts, NULL, pool));
471 ffd = fs->fsap_data;
472 ffd->rep_sharing_allowed = FALSE;
473
474 /* Add the Greek tree. */
475 SVN_ERR(svn_fs_begin_txn(&txn, fs, 0, pool));
476 SVN_ERR(svn_fs_txn_root(&txn_root, txn, pool));
477 SVN_ERR(svn_test__create_greek_tree(txn_root, pool));
478 SVN_ERR(svn_fs_commit_txn(NULL, &rev, txn, pool));
479 SVN_TEST_ASSERT(SVN_IS_VALID_REVNUM(rev));
480
481 /* Make sure the rep-cache does not exist. */
482 SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool));
483 SVN_TEST_ASSERT(!exists);
484
485 /* Build and verify the rep-cache. */
486 ffd->rep_sharing_allowed = TRUE;
487
488 input.start_rev = rev;
489 input.end_rev = rev;
490 SVN_ERR(svn_fs_ioctl(fs, SVN_FS_FS__IOCTL_BUILD_REP_CACHE,
491 &input, NULL, NULL, NULL, pool, pool));
492
493 SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool));
494 SVN_TEST_ASSERT(exists);
495
496 SVN_ERR(svn_fs_verify(fs_path, NULL, 0, SVN_INVALID_REVNUM,
497 NULL, NULL, NULL, NULL, pool));
498
499 return SVN_NO_ERROR;
500 }
501
502
503
504 /* The test table. */
505
506 static int max_threads = 0;
507
508 static struct svn_test_descriptor_t test_funcs[] =
509 {
510 SVN_TEST_NULL,
511 SVN_TEST_OPTS_PASS(get_repo_stats,
512 "get statistics on a FSFS filesystem"),
513 SVN_TEST_OPTS_PASS(dump_index,
514 "dump the P2L index"),
515 SVN_TEST_OPTS_PASS(load_index,
516 "load the P2L index"),
517 SVN_TEST_OPTS_PASS(build_rep_cache,
518 "build the representation cache"),
519 SVN_TEST_NULL
520 };
521
522 SVN_TEST_MAIN
523