1 /* revprops.c --- everything needed to handle revprops in FSFS
2 *
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
19 * under the License.
20 * ====================================================================
21 */
22
23 #include <assert.h>
24
25 #include "svn_pools.h"
26 #include "svn_hash.h"
27 #include "svn_dirent_uri.h"
28 #include "svn_sorts.h"
29
30 #include "fs_fs.h"
31 #include "revprops.h"
32 #include "temp_serializer.h"
33 #include "util.h"
34
35 #include "private/svn_subr_private.h"
36 #include "private/svn_string_private.h"
37 #include "../libsvn_fs/fs-loader.h"
38
39 #include "svn_private_config.h"
40
41 svn_error_t *
svn_fs_fs__upgrade_pack_revprops(svn_fs_t * fs,svn_fs_upgrade_notify_t notify_func,void * notify_baton,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)42 svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs,
43 svn_fs_upgrade_notify_t notify_func,
44 void *notify_baton,
45 svn_cancel_func_t cancel_func,
46 void *cancel_baton,
47 apr_pool_t *scratch_pool)
48 {
49 fs_fs_data_t *ffd = fs->fsap_data;
50 const char *revprops_shard_path;
51 const char *revprops_pack_file_dir;
52 apr_int64_t shard;
53 apr_int64_t first_unpacked_shard
54 = ffd->min_unpacked_rev / ffd->max_files_per_dir;
55
56 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
57 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
58 scratch_pool);
59 int compression_level = ffd->compress_packed_revprops
60 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
61 : SVN_DELTA_COMPRESSION_LEVEL_NONE;
62
63 /* first, pack all revprops shards to match the packed revision shards */
64 for (shard = 0; shard < first_unpacked_shard; ++shard)
65 {
66 svn_pool_clear(iterpool);
67
68 revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
69 apr_psprintf(iterpool,
70 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
71 shard),
72 iterpool);
73 revprops_shard_path = svn_dirent_join(revsprops_dir,
74 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
75 iterpool);
76
77 SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir,
78 revprops_shard_path,
79 shard, ffd->max_files_per_dir,
80 (int)(0.9 * ffd->revprop_pack_size),
81 compression_level,
82 ffd->flush_to_disk,
83 cancel_func, cancel_baton,
84 iterpool));
85 if (notify_func)
86 SVN_ERR(notify_func(notify_baton, shard,
87 svn_fs_upgrade_pack_revprops, iterpool));
88 }
89
90 svn_pool_destroy(iterpool);
91
92 return SVN_NO_ERROR;
93 }
94
95 svn_error_t *
svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t * fs,svn_fs_upgrade_notify_t notify_func,void * notify_baton,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)96 svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs,
97 svn_fs_upgrade_notify_t notify_func,
98 void *notify_baton,
99 svn_cancel_func_t cancel_func,
100 void *cancel_baton,
101 apr_pool_t *scratch_pool)
102 {
103 fs_fs_data_t *ffd = fs->fsap_data;
104 const char *revprops_shard_path;
105 apr_int64_t shard;
106 apr_int64_t first_unpacked_shard
107 = ffd->min_unpacked_rev / ffd->max_files_per_dir;
108
109 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
110 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
111 scratch_pool);
112
113 /* delete the non-packed revprops shards afterwards */
114 for (shard = 0; shard < first_unpacked_shard; ++shard)
115 {
116 svn_pool_clear(iterpool);
117
118 revprops_shard_path = svn_dirent_join(revsprops_dir,
119 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
120 iterpool);
121 SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path,
122 shard,
123 ffd->max_files_per_dir,
124 cancel_func, cancel_baton,
125 iterpool));
126 if (notify_func)
127 SVN_ERR(notify_func(notify_baton, shard,
128 svn_fs_upgrade_cleanup_revprops, iterpool));
129 }
130
131 svn_pool_destroy(iterpool);
132
133 return SVN_NO_ERROR;
134 }
135
136 /* Container for all data required to access the packed revprop file
137 * for a given REVISION. This structure will be filled incrementally
138 * by read_pack_revprops() its sub-routines.
139 */
140 typedef struct packed_revprops_t
141 {
142 /* revision number to read (not necessarily the first in the pack) */
143 svn_revnum_t revision;
144
145 /* the actual revision properties */
146 apr_hash_t *properties;
147
148 /* their size when serialized to a single string
149 * (as found in PACKED_REVPROPS) */
150 apr_size_t serialized_size;
151
152
153 /* name of the pack file (without folder path) */
154 const char *filename;
155
156 /* packed shard folder path */
157 const char *folder;
158
159 /* sum of values in SIZES */
160 apr_size_t total_size;
161
162 /* first revision in the pack (>= MANIFEST_START) */
163 svn_revnum_t start_revision;
164
165 /* size of the revprops in PACKED_REVPROPS */
166 apr_array_header_t *sizes;
167
168 /* offset of the revprops in PACKED_REVPROPS */
169 apr_array_header_t *offsets;
170
171
172 /* concatenation of the serialized representation of all revprops
173 * in the pack, i.e. the pack content without header and compression */
174 svn_stringbuf_t *packed_revprops;
175
176 /* First revision covered by MANIFEST.
177 * Will equal the shard start revision or 1, for the 1st shard. */
178 svn_revnum_t manifest_start;
179
180 /* content of the manifest.
181 * Maps long(rev - MANIFEST_START) to const char* pack file name */
182 apr_array_header_t *manifest;
183 } packed_revprops_t;
184
185 /* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
186 * Also, put them into the revprop cache, if activated, for future use.
187 *
188 * The returned hash will be allocated in RESULT_POOL, SCRATCH_POOL is being
189 * used for temporary allocations.
190 */
191 static svn_error_t *
parse_revprop(apr_hash_t ** properties,svn_fs_t * fs,svn_revnum_t revision,svn_string_t * content,apr_pool_t * result_pool,apr_pool_t * scratch_pool)192 parse_revprop(apr_hash_t **properties,
193 svn_fs_t *fs,
194 svn_revnum_t revision,
195 svn_string_t *content,
196 apr_pool_t *result_pool,
197 apr_pool_t *scratch_pool)
198 {
199 svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
200 *properties = apr_hash_make(result_pool);
201
202 SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR,
203 result_pool),
204 apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.",
205 revision));
206
207 return SVN_NO_ERROR;
208 }
209
210 void
svn_fs_fs__reset_revprop_cache(svn_fs_t * fs)211 svn_fs_fs__reset_revprop_cache(svn_fs_t *fs)
212 {
213 fs_fs_data_t *ffd = fs->fsap_data;
214 ffd->revprop_prefix = 0;
215 }
216
217 /* If FS has not a revprop cache prefix set, generate one.
218 * Always call this before accessing the revprop cache.
219 */
220 static svn_error_t *
prepare_revprop_cache(svn_fs_t * fs,apr_pool_t * scratch_pool)221 prepare_revprop_cache(svn_fs_t *fs,
222 apr_pool_t *scratch_pool)
223 {
224 fs_fs_data_t *ffd = fs->fsap_data;
225 if (!ffd->revprop_prefix)
226 SVN_ERR(svn_atomic__unique_counter(&ffd->revprop_prefix));
227
228 return SVN_NO_ERROR;
229 }
230
231 /* Store the unparsed revprop hash CONTENT for REVISION in FS's revprop
232 * cache. If CACHED is not NULL, set *CACHED if there already is such
233 * an entry and skip the cache write in that case. Use SCRATCH_POOL for
234 * temporary allocations. */
235 static svn_error_t *
cache_revprops(svn_boolean_t * is_cached,svn_fs_t * fs,svn_revnum_t revision,svn_string_t * content,apr_pool_t * scratch_pool)236 cache_revprops(svn_boolean_t *is_cached,
237 svn_fs_t *fs,
238 svn_revnum_t revision,
239 svn_string_t *content,
240 apr_pool_t *scratch_pool)
241 {
242 fs_fs_data_t *ffd = fs->fsap_data;
243 pair_cache_key_t key;
244
245 /* Make sure prepare_revprop_cache() has been called. */
246 SVN_ERR_ASSERT(ffd->revprop_prefix);
247 key.revision = revision;
248 key.second = ffd->revprop_prefix;
249
250 if (is_cached)
251 {
252 SVN_ERR(svn_cache__has_key(is_cached, ffd->revprop_cache, &key,
253 scratch_pool));
254 if (*is_cached)
255 return SVN_NO_ERROR;
256 }
257
258 SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, content, scratch_pool));
259
260 return SVN_NO_ERROR;
261 }
262
263 /* Read the non-packed revprops for revision REV in FS, put them into the
264 * revprop cache if PROPULATE_CACHE is set and return them in *PROPERTIES.
265 *
266 * If the data could not be read due to an otherwise recoverable error,
267 * leave *PROPERTIES unchanged. No error will be returned in that case.
268 *
269 * Allocations will be done in POOL.
270 */
271 static svn_error_t *
read_non_packed_revprop(apr_hash_t ** properties,svn_fs_t * fs,svn_revnum_t rev,svn_boolean_t populate_cache,apr_pool_t * pool)272 read_non_packed_revprop(apr_hash_t **properties,
273 svn_fs_t *fs,
274 svn_revnum_t rev,
275 svn_boolean_t populate_cache,
276 apr_pool_t *pool)
277 {
278 svn_stringbuf_t *content = NULL;
279 apr_pool_t *iterpool = svn_pool_create(pool);
280 svn_boolean_t missing = FALSE;
281 int i;
282
283 for (i = 0;
284 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content;
285 ++i)
286 {
287 svn_pool_clear(iterpool);
288 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content,
289 &missing,
290 svn_fs_fs__path_revprops(fs, rev, iterpool),
291 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT ,
292 iterpool));
293 }
294
295 if (content)
296 {
297 svn_string_t *as_string = svn_stringbuf__morph_into_string(content);
298 SVN_ERR(parse_revprop(properties, fs, rev, as_string, pool, iterpool));
299
300 if (populate_cache)
301 SVN_ERR(cache_revprops(NULL, fs, rev, as_string, iterpool));
302 }
303
304 svn_pool_clear(iterpool);
305
306 return SVN_NO_ERROR;
307 }
308
309 /* Return the minimum length of any packed revprop file name in REVPROPS. */
310 static apr_size_t
get_min_filename_len(packed_revprops_t * revprops)311 get_min_filename_len(packed_revprops_t *revprops)
312 {
313 char number_buffer[SVN_INT64_BUFFER_SIZE];
314
315 /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being
316 * at least the first rev in the shard and <COUNT> having at least one
317 * digit. Thus, the minimum is 2 + #decimal places in the start rev.
318 */
319 return svn__i64toa(number_buffer, revprops->manifest_start) + 2;
320 }
321
322 /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
323 * members. Use RESULT_POOL for allocating results and SCRATCH_POOL for
324 * temporaries.
325 */
326 static svn_error_t *
get_revprop_packname(svn_fs_t * fs,packed_revprops_t * revprops,apr_pool_t * result_pool,apr_pool_t * scratch_pool)327 get_revprop_packname(svn_fs_t *fs,
328 packed_revprops_t *revprops,
329 apr_pool_t *result_pool,
330 apr_pool_t *scratch_pool)
331 {
332 fs_fs_data_t *ffd = fs->fsap_data;
333 svn_stringbuf_t *content = NULL;
334 const char *manifest_file_path;
335 int idx, rev_count;
336 char *buffer, *buffer_end;
337 const char **filenames, **filenames_end;
338 apr_size_t min_filename_len;
339
340 /* Determine the dimensions. Rev 0 is excluded from the first shard. */
341 rev_count = ffd->max_files_per_dir;
342 revprops->manifest_start
343 = revprops->revision - (revprops->revision % rev_count);
344 if (revprops->manifest_start == 0)
345 {
346 ++revprops->manifest_start;
347 --rev_count;
348 }
349
350 revprops->manifest = apr_array_make(result_pool, rev_count,
351 sizeof(const char*));
352
353 /* No line in the file can be less than this number of chars long. */
354 min_filename_len = get_min_filename_len(revprops);
355
356 /* Read the content of the manifest file */
357 revprops->folder
358 = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision,
359 result_pool);
360 manifest_file_path
361 = svn_dirent_join(revprops->folder, PATH_MANIFEST, result_pool);
362
363 SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, result_pool));
364
365 /* There CONTENT must have a certain minimal size and there no
366 * unterminated lines at the end of the file. Both guarantees also
367 * simplify the parser loop below.
368 */
369 if ( content->len < rev_count * (min_filename_len + 1)
370 || content->data[content->len - 1] != '\n')
371 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
372 _("Packed revprop manifest for r%ld not "
373 "properly terminated"), revprops->revision);
374
375 /* Chop (parse) the manifest CONTENT into filenames, one per line.
376 * We only have to replace all newlines with NUL and add all line
377 * starts to REVPROPS->MANIFEST.
378 *
379 * There must be exactly REV_COUNT lines and that is the number of
380 * lines we parse from BUFFER to FILENAMES. Set the end pointer for
381 * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid
382 * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN.
383 *
384 * Please note that this loop is performance critical for e.g. 'svn log'.
385 * It is run 1000x per revprop access, i.e. per revision and about
386 * 50 million times per sec (and CPU core).
387 */
388 for (filenames = (const char **)revprops->manifest->elts,
389 filenames_end = filenames + rev_count,
390 buffer = content->data,
391 buffer_end = buffer + content->len - min_filename_len;
392 (filenames < filenames_end) && (buffer < buffer_end);
393 ++filenames)
394 {
395 /* BUFFER always points to the start of the next line / filename. */
396 *filenames = buffer;
397
398 /* Find the next EOL. This is guaranteed to stay within the CONTENT
399 * buffer because we left enough room after BUFFER_END and we know
400 * we will always see a newline as the last non-NUL char. */
401 buffer += min_filename_len;
402 while (*buffer != '\n')
403 ++buffer;
404
405 /* Found EOL. Turn it into the filename terminator and move BUFFER
406 * to the start of the next line or CONTENT buffer end. */
407 *buffer = '\0';
408 ++buffer;
409 }
410
411 /* We must have reached the end of both buffers. */
412 if (buffer < content->data + content->len)
413 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
414 _("Packed revprop manifest for r%ld "
415 "has too many entries"), revprops->revision);
416
417 if (filenames < filenames_end)
418 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
419 _("Packed revprop manifest for r%ld "
420 "has too few entries"), revprops->revision);
421
422 /* The target array has now exactly one entry per revision. */
423 revprops->manifest->nelts = rev_count;
424
425 /* Now get the file name */
426 idx = (int)(revprops->revision - revprops->manifest_start);
427 revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
428
429 return SVN_NO_ERROR;
430 }
431
432 /* Return TRUE, if revision R1 and R2 refer to the same shard in FS.
433 */
434 static svn_boolean_t
same_shard(svn_fs_t * fs,svn_revnum_t r1,svn_revnum_t r2)435 same_shard(svn_fs_t *fs,
436 svn_revnum_t r1,
437 svn_revnum_t r2)
438 {
439 fs_fs_data_t *ffd = fs->fsap_data;
440 return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir);
441 }
442
443 /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
444 * fill the START_REVISION member, and make PACKED_REVPROPS point to the
445 * first serialized revprop. If READ_ALL is set, initialize the SIZES
446 * and OFFSETS members as well. If POPULATE_CACHE is set, cache all
447 * revprops found in this pack.
448 *
449 * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
450 * well as the SERIALIZED_SIZE member. If revprop caching has been
451 * enabled, parse all revprops in the pack and cache them.
452 */
453 static svn_error_t *
parse_packed_revprops(svn_fs_t * fs,packed_revprops_t * revprops,svn_boolean_t read_all,svn_boolean_t populate_cache,apr_pool_t * result_pool,apr_pool_t * scratch_pool)454 parse_packed_revprops(svn_fs_t *fs,
455 packed_revprops_t *revprops,
456 svn_boolean_t read_all,
457 svn_boolean_t populate_cache,
458 apr_pool_t *result_pool,
459 apr_pool_t *scratch_pool)
460 {
461 svn_stream_t *stream;
462 apr_int64_t first_rev, count, i;
463 apr_size_t offset;
464 const char *header_end;
465 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
466
467 /* Initial value for the "Leaking bucket" pattern. */
468 int bucket = 4;
469
470 /* decompress (even if the data is only "stored", there is still a
471 * length header to remove) */
472 svn_stringbuf_t *compressed = revprops->packed_revprops;
473 svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(result_pool);
474 SVN_ERR(svn__decompress_zlib(compressed->data, compressed->len,
475 uncompressed, APR_SIZE_MAX));
476
477 /* read first revision number and number of revisions in the pack */
478 stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
479 SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream,
480 iterpool));
481 SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream,
482 iterpool));
483
484 /* Check revision range for validity. */
485 if ( !same_shard(fs, revprops->revision, first_rev)
486 || !same_shard(fs, revprops->revision, first_rev + count - 1)
487 || count < 1)
488 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
489 _("Revprop pack for revision r%ld"
490 " contains revprops for r%ld .. r%ld"),
491 revprops->revision,
492 (svn_revnum_t)first_rev,
493 (svn_revnum_t)(first_rev + count -1));
494
495 /* Since start & end are in the same shard, it is enough to just test
496 * the FIRST_REV for being actually packed. That will also cover the
497 * special case of rev 0 never being packed. */
498 if (!svn_fs_fs__is_packed_revprop(fs, first_rev))
499 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
500 _("Revprop pack for revision r%ld"
501 " starts at non-packed revisions r%ld"),
502 revprops->revision, (svn_revnum_t)first_rev);
503
504 /* make PACKED_REVPROPS point to the first char after the header.
505 * This is where the serialized revprops are. */
506 header_end = strstr(uncompressed->data, "\n\n");
507 if (header_end == NULL)
508 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
509 _("Header end not found"));
510
511 offset = header_end - uncompressed->data + 2;
512
513 revprops->packed_revprops = svn_stringbuf_create_empty(result_pool);
514 revprops->packed_revprops->data = uncompressed->data + offset;
515 revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
516 revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize
517 - offset);
518
519 /* STREAM still points to the first entry in the sizes list. */
520 revprops->start_revision = (svn_revnum_t)first_rev;
521 if (read_all)
522 {
523 /* Init / construct REVPROPS members. */
524 revprops->sizes = apr_array_make(result_pool, (int)count,
525 sizeof(offset));
526 revprops->offsets = apr_array_make(result_pool, (int)count,
527 sizeof(offset));
528 }
529
530 /* Now parse, revision by revision, the size and content of each
531 * revisions' revprops. */
532 for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
533 {
534 apr_int64_t size;
535 svn_string_t serialized;
536 svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
537 svn_pool_clear(iterpool);
538
539 /* read & check the serialized size */
540 SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream,
541 iterpool));
542 if (size > (apr_int64_t)revprops->packed_revprops->len - offset)
543 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
544 _("Packed revprop size exceeds pack file size"));
545
546 /* Parse this revprops list, if necessary */
547 serialized.data = revprops->packed_revprops->data + offset;
548 serialized.len = (apr_size_t)size;
549
550 if (revision == revprops->revision)
551 {
552 /* Parse (and possibly cache) the one revprop list we care about. */
553 SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
554 &serialized, result_pool, iterpool));
555 revprops->serialized_size = serialized.len;
556
557 /* If we only wanted the revprops for REVISION then we are done. */
558 if (!read_all && !populate_cache)
559 break;
560 }
561
562 if (populate_cache)
563 {
564 /* Adding all those revprops is expensive, in particular in a
565 * multi-threaded environment. There are situations where hit
566 * rates are low and revprops get evicted before re-using them.
567 *
568 * We try to detect thosse cases here.
569 * Only keep going while most (at least 2/3) aren't cached, yet. */
570 svn_boolean_t already_cached;
571 SVN_ERR(cache_revprops(&already_cached, fs, revision, &serialized,
572 iterpool));
573
574 /* Stop populating the cache once we encountered too many entries
575 * already present relative to the numbers being added. */
576 if (!already_cached)
577 {
578 ++bucket;
579 }
580 else
581 {
582 bucket -= 2;
583 if (bucket < 0)
584 populate_cache = FALSE;
585 }
586 }
587
588 if (read_all)
589 {
590 /* fill REVPROPS data structures */
591 APR_ARRAY_PUSH(revprops->sizes, apr_size_t) = serialized.len;
592 APR_ARRAY_PUSH(revprops->offsets, apr_size_t) = offset;
593 }
594 revprops->total_size += serialized.len;
595
596 offset += serialized.len;
597 }
598
599 return SVN_NO_ERROR;
600 }
601
602 /* In filesystem FS, read the packed revprops for revision REV into
603 * *REVPROPS. Populate the revprop cache, if POPULATE_CACHE is set.
604 * If you want to modify revprop contents / update REVPROPS, READ_ALL
605 * must be set. Otherwise, only the properties of REV are being provided.
606 * Allocate data in POOL.
607 */
608 static svn_error_t *
read_pack_revprop(packed_revprops_t ** revprops,svn_fs_t * fs,svn_revnum_t rev,svn_boolean_t read_all,svn_boolean_t populate_cache,apr_pool_t * pool)609 read_pack_revprop(packed_revprops_t **revprops,
610 svn_fs_t *fs,
611 svn_revnum_t rev,
612 svn_boolean_t read_all,
613 svn_boolean_t populate_cache,
614 apr_pool_t *pool)
615 {
616 apr_pool_t *iterpool = svn_pool_create(pool);
617 svn_boolean_t missing = FALSE;
618 svn_error_t *err;
619 packed_revprops_t *result;
620 int i;
621
622 /* someone insisted that REV is packed. Double-check if necessary */
623 if (!svn_fs_fs__is_packed_revprop(fs, rev))
624 SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool));
625
626 if (!svn_fs_fs__is_packed_revprop(fs, rev))
627 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
628 _("No such packed revision %ld"), rev);
629
630 /* initialize the result data structure */
631 result = apr_pcalloc(pool, sizeof(*result));
632 result->revision = rev;
633
634 /* try to read the packed revprops. This may require retries if we have
635 * concurrent writers. */
636 for (i = 0;
637 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops;
638 ++i)
639 {
640 const char *file_path;
641 svn_pool_clear(iterpool);
642
643 /* there might have been concurrent writes.
644 * Re-read the manifest and the pack file.
645 */
646 SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
647 file_path = svn_dirent_join(result->folder,
648 result->filename,
649 iterpool);
650 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops,
651 &missing,
652 file_path,
653 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT,
654 pool));
655 }
656
657 /* the file content should be available now */
658 if (!result->packed_revprops)
659 return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL,
660 _("Failed to read revprop pack file for r%ld"), rev);
661
662 /* parse it. RESULT will be complete afterwards. */
663 err = parse_packed_revprops(fs, result, read_all, populate_cache, pool,
664 iterpool);
665 svn_pool_destroy(iterpool);
666 if (err)
667 return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
668 _("Revprop pack file for r%ld is corrupt"), rev);
669
670 *revprops = result;
671
672 return SVN_NO_ERROR;
673 }
674
675 svn_error_t *
svn_fs_fs__get_revision_props_size(apr_off_t * props_size_p,svn_fs_t * fs,svn_revnum_t rev,apr_pool_t * scratch_pool)676 svn_fs_fs__get_revision_props_size(apr_off_t *props_size_p,
677 svn_fs_t *fs,
678 svn_revnum_t rev,
679 apr_pool_t *scratch_pool)
680 {
681 fs_fs_data_t *ffd = fs->fsap_data;
682
683 /* should they be available at all? */
684 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool));
685
686 /* if REV had not been packed when we began, try reading it from the
687 * non-packed shard. If that fails, we will fall through to packed
688 * shard reads. */
689 if (!svn_fs_fs__is_packed_revprop(fs, rev))
690 {
691 const char *path = svn_fs_fs__path_revprops(fs, rev, scratch_pool);
692 svn_error_t *err;
693 apr_file_t *file;
694 svn_filesize_t file_size;
695
696 err = svn_io_file_open(&file, path, APR_FOPEN_READ, APR_OS_DEFAULT,
697 scratch_pool);
698 if (!err)
699 err = svn_io_file_size_get(&file_size, file, scratch_pool);
700 if (!err)
701 {
702 *props_size_p = (apr_off_t)file_size;
703 return SVN_NO_ERROR;
704 }
705 else if (!APR_STATUS_IS_ENOENT(err->apr_err)
706 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
707 {
708 return svn_error_trace(err);
709 }
710
711 /* fall through: maybe the revision got packed while we were looking */
712 svn_error_clear(err);
713 }
714
715 /* Try reading packed revprops. If that fails, REV is most
716 * likely invalid (or its revprops highly contested). */
717 {
718 packed_revprops_t *revprops;
719
720 /* ### This is inefficient -- reading all the revprops in a pack. We
721 should just read the index. */
722 SVN_ERR(read_pack_revprop(&revprops, fs, rev,
723 TRUE /*read_all*/, FALSE /*populate_cache*/,
724 scratch_pool));
725 *props_size_p = (apr_off_t)APR_ARRAY_IDX(revprops->sizes,
726 rev - revprops->start_revision,
727 apr_size_t);
728 }
729
730 return SVN_NO_ERROR;
731 }
732
733 /* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
734 *
735 * Allocations will be done in POOL.
736 */
737 svn_error_t *
svn_fs_fs__get_revision_proplist(apr_hash_t ** proplist_p,svn_fs_t * fs,svn_revnum_t rev,svn_boolean_t refresh,apr_pool_t * result_pool,apr_pool_t * scratch_pool)738 svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p,
739 svn_fs_t *fs,
740 svn_revnum_t rev,
741 svn_boolean_t refresh,
742 apr_pool_t *result_pool,
743 apr_pool_t *scratch_pool)
744 {
745 fs_fs_data_t *ffd = fs->fsap_data;
746
747 /* Only populate the cache if we did not just cross a sync barrier.
748 * This is to eliminate overhead from code that always sets REFRESH.
749 * For callers that want caching, the caching kicks in on read "later". */
750 svn_boolean_t populate_cache = !refresh;
751
752 /* not found, yet */
753 *proplist_p = NULL;
754
755 /* should they be available at all? */
756 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, scratch_pool));
757
758 if (refresh)
759 {
760 /* Previous cache contents is invalid now. */
761 svn_fs_fs__reset_revprop_cache(fs);
762 }
763 else
764 {
765 /* Try cache lookup first. */
766 svn_boolean_t is_cached;
767 pair_cache_key_t key;
768
769 /* Auto-alloc prefix and construct the key. */
770 SVN_ERR(prepare_revprop_cache(fs, scratch_pool));
771 key.revision = rev;
772 key.second = ffd->revprop_prefix;
773
774 /* The only way that this might error out is due to parser error. */
775 SVN_ERR_W(svn_cache__get((void **) proplist_p, &is_cached,
776 ffd->revprop_cache, &key, result_pool),
777 apr_psprintf(scratch_pool,
778 "Failed to parse revprops for r%ld.",
779 rev));
780 if (is_cached)
781 return SVN_NO_ERROR;
782 }
783
784 /* if REV had not been packed when we began, try reading it from the
785 * non-packed shard. If that fails, we will fall through to packed
786 * shard reads. */
787 if (!svn_fs_fs__is_packed_revprop(fs, rev))
788 {
789 svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev,
790 populate_cache, result_pool);
791 if (err)
792 {
793 if (!APR_STATUS_IS_ENOENT(err->apr_err)
794 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
795 return svn_error_trace(err);
796
797 svn_error_clear(err);
798 *proplist_p = NULL; /* in case read_non_packed_revprop changed it */
799 }
800 }
801
802 /* if revprop packing is available and we have not read the revprops, yet,
803 * try reading them from a packed shard. If that fails, REV is most
804 * likely invalid (or its revprops highly contested). */
805 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
806 {
807 packed_revprops_t *revprops;
808 SVN_ERR(read_pack_revprop(&revprops, fs, rev, FALSE, populate_cache,
809 result_pool));
810 *proplist_p = revprops->properties;
811 }
812
813 /* The revprops should have been there. Did we get them? */
814 if (!*proplist_p)
815 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
816 _("Could not read revprops for revision %ld"),
817 rev);
818
819 return SVN_NO_ERROR;
820 }
821
822 /* Serialize the revision property list PROPLIST of revision REV in
823 * filesystem FS to a non-packed file. Return the name of that temporary
824 * file in *TMP_PATH and the file path that it must be moved to in
825 * *FINAL_PATH.
826 *
827 * Use POOL for allocations.
828 */
829 static svn_error_t *
write_non_packed_revprop(const char ** final_path,const char ** tmp_path,svn_fs_t * fs,svn_revnum_t rev,apr_hash_t * proplist,apr_pool_t * pool)830 write_non_packed_revprop(const char **final_path,
831 const char **tmp_path,
832 svn_fs_t *fs,
833 svn_revnum_t rev,
834 apr_hash_t *proplist,
835 apr_pool_t *pool)
836 {
837 fs_fs_data_t *ffd = fs->fsap_data;
838 apr_file_t *file;
839 svn_stream_t *stream;
840 *final_path = svn_fs_fs__path_revprops(fs, rev, pool);
841
842 /* ### do we have a directory sitting around already? we really shouldn't
843 ### have to get the dirname here. */
844 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path,
845 svn_dirent_dirname(*final_path, pool),
846 svn_io_file_del_none, pool, pool));
847 stream = svn_stream_from_aprfile2(file, TRUE, pool);
848 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
849 SVN_ERR(svn_stream_close(stream));
850
851 /* Flush temporary file to disk and close it. */
852 if (ffd->flush_to_disk)
853 SVN_ERR(svn_io_file_flush_to_disk(file, pool));
854 SVN_ERR(svn_io_file_close(file, pool));
855
856 return SVN_NO_ERROR;
857 }
858
859 /* After writing the new revprop file(s), call this function to move the
860 * file at TMP_PATH to FINAL_PATH and give it the permissions from
861 * PERMS_REFERENCE.
862 *
863 * Finally, delete all the temporary files given in FILES_TO_DELETE.
864 * The latter may be NULL.
865 *
866 * Use POOL for temporary allocations.
867 */
868 static svn_error_t *
switch_to_new_revprop(svn_fs_t * fs,const char * final_path,const char * tmp_path,const char * perms_reference,apr_array_header_t * files_to_delete,apr_pool_t * pool)869 switch_to_new_revprop(svn_fs_t *fs,
870 const char *final_path,
871 const char *tmp_path,
872 const char *perms_reference,
873 apr_array_header_t *files_to_delete,
874 apr_pool_t *pool)
875 {
876 fs_fs_data_t *ffd = fs->fsap_data;
877
878 SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference,
879 ffd->flush_to_disk, pool));
880
881 /* Clean up temporary files, if necessary. */
882 if (files_to_delete)
883 {
884 apr_pool_t *iterpool = svn_pool_create(pool);
885 int i;
886
887 for (i = 0; i < files_to_delete->nelts; ++i)
888 {
889 const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
890
891 svn_pool_clear(iterpool);
892 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
893 }
894
895 svn_pool_destroy(iterpool);
896 }
897 return SVN_NO_ERROR;
898 }
899
900 /* Write a pack file header to STREAM that starts at revision START_REVISION
901 * and contains the indexes [START,END) of SIZES.
902 */
903 static svn_error_t *
serialize_revprops_header(svn_stream_t * stream,svn_revnum_t start_revision,apr_array_header_t * sizes,int start,int end,apr_pool_t * pool)904 serialize_revprops_header(svn_stream_t *stream,
905 svn_revnum_t start_revision,
906 apr_array_header_t *sizes,
907 int start,
908 int end,
909 apr_pool_t *pool)
910 {
911 apr_pool_t *iterpool = svn_pool_create(pool);
912 int i;
913
914 SVN_ERR_ASSERT(start < end);
915
916 /* start revision and entry count */
917 SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
918 SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
919
920 /* the sizes array */
921 for (i = start; i < end; ++i)
922 {
923 /* Non-standard pool usage.
924 *
925 * We only allocate a few bytes each iteration -- even with a
926 * million iterations we would still be in good shape memory-wise.
927 */
928 apr_size_t size = APR_ARRAY_IDX(sizes, i, apr_size_t);
929 SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_SIZE_T_FMT "\n",
930 size));
931 }
932
933 /* the double newline char indicates the end of the header */
934 SVN_ERR(svn_stream_puts(stream, "\n"));
935
936 svn_pool_destroy(iterpool);
937 return SVN_NO_ERROR;
938 }
939
940 /* Writes the a pack file to FILE. It copies the serialized data
941 * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
942 *
943 * The data for the latter is taken from NEW_SERIALIZED. Note, that
944 * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
945 * taken in that case but only a subset of the old data will be copied.
946 *
947 * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
948 * POOL is used for temporary allocations.
949 */
950 static svn_error_t *
repack_revprops(svn_fs_t * fs,packed_revprops_t * revprops,int start,int end,int changed_index,svn_stringbuf_t * new_serialized,apr_size_t new_total_size,apr_file_t * file,apr_pool_t * pool)951 repack_revprops(svn_fs_t *fs,
952 packed_revprops_t *revprops,
953 int start,
954 int end,
955 int changed_index,
956 svn_stringbuf_t *new_serialized,
957 apr_size_t new_total_size,
958 apr_file_t *file,
959 apr_pool_t *pool)
960 {
961 fs_fs_data_t *ffd = fs->fsap_data;
962 svn_stream_t *stream;
963 int i;
964
965 /* create data empty buffers and the stream object */
966 svn_stringbuf_t *uncompressed
967 = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
968 svn_stringbuf_t *compressed
969 = svn_stringbuf_create_empty(pool);
970 stream = svn_stream_from_stringbuf(uncompressed, pool);
971
972 /* write the header*/
973 SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
974 revprops->sizes, start, end, pool));
975
976 /* append the serialized revprops */
977 for (i = start; i < end; ++i)
978 if (i == changed_index)
979 {
980 SVN_ERR(svn_stream_write(stream,
981 new_serialized->data,
982 &new_serialized->len));
983 }
984 else
985 {
986 apr_size_t size = APR_ARRAY_IDX(revprops->sizes, i, apr_size_t);
987 apr_size_t offset = APR_ARRAY_IDX(revprops->offsets, i, apr_size_t);
988
989 SVN_ERR(svn_stream_write(stream,
990 revprops->packed_revprops->data + offset,
991 &size));
992 }
993
994 /* flush the stream buffer (if any) to our underlying data buffer */
995 SVN_ERR(svn_stream_close(stream));
996
997 /* compress / store the data */
998 SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len,
999 compressed,
1000 ffd->compress_packed_revprops
1001 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
1002 : SVN_DELTA_COMPRESSION_LEVEL_NONE));
1003
1004 /* finally, write the content to the target file, flush and close it */
1005 SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len,
1006 NULL, pool));
1007 if (ffd->flush_to_disk)
1008 SVN_ERR(svn_io_file_flush_to_disk(file, pool));
1009 SVN_ERR(svn_io_file_close(file, pool));
1010
1011 return SVN_NO_ERROR;
1012 }
1013
1014 /* Allocate a new pack file name for revisions
1015 * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1]
1016 * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE,
1017 * auto-create that array if necessary. Return an open file *FILE that is
1018 * allocated in POOL.
1019 */
1020 static svn_error_t *
repack_file_open(apr_file_t ** file,svn_fs_t * fs,packed_revprops_t * revprops,int start,int end,apr_array_header_t ** files_to_delete,apr_pool_t * pool)1021 repack_file_open(apr_file_t **file,
1022 svn_fs_t *fs,
1023 packed_revprops_t *revprops,
1024 int start,
1025 int end,
1026 apr_array_header_t **files_to_delete,
1027 apr_pool_t *pool)
1028 {
1029 apr_int64_t tag;
1030 const char *tag_string;
1031 const char *new_filename;
1032 int i;
1033 int manifest_offset
1034 = (int)(revprops->start_revision - revprops->manifest_start);
1035
1036 /* get the old (= current) file name and enlist it for later deletion */
1037 const char *old_filename = APR_ARRAY_IDX(revprops->manifest,
1038 start + manifest_offset,
1039 const char*);
1040
1041 if (*files_to_delete == NULL)
1042 *files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
1043
1044 APR_ARRAY_PUSH(*files_to_delete, const char*)
1045 = svn_dirent_join(revprops->folder, old_filename, pool);
1046
1047 /* increase the tag part, i.e. the counter after the dot */
1048 tag_string = strchr(old_filename, '.');
1049 if (tag_string == NULL)
1050 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
1051 _("Packed file '%s' misses a tag"),
1052 old_filename);
1053
1054 SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
1055 new_filename = apr_psprintf(pool, "%ld.%" APR_INT64_T_FMT,
1056 revprops->start_revision + start,
1057 ++tag);
1058
1059 /* update the manifest to point to the new file */
1060 for (i = start; i < end; ++i)
1061 APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*)
1062 = new_filename;
1063
1064 /* open the file */
1065 SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder,
1066 new_filename,
1067 pool),
1068 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
1069
1070 return SVN_NO_ERROR;
1071 }
1072
1073 /* For revision REV in filesystem FS, set the revision properties to
1074 * PROPLIST. Return a new file in *TMP_PATH that the caller shall move
1075 * to *FINAL_PATH to make the change visible. Files to be deleted will
1076 * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
1077 * Use POOL for allocations.
1078 */
1079 static svn_error_t *
write_packed_revprop(const char ** final_path,const char ** tmp_path,apr_array_header_t ** files_to_delete,svn_fs_t * fs,svn_revnum_t rev,apr_hash_t * proplist,apr_pool_t * pool)1080 write_packed_revprop(const char **final_path,
1081 const char **tmp_path,
1082 apr_array_header_t **files_to_delete,
1083 svn_fs_t *fs,
1084 svn_revnum_t rev,
1085 apr_hash_t *proplist,
1086 apr_pool_t *pool)
1087 {
1088 fs_fs_data_t *ffd = fs->fsap_data;
1089 packed_revprops_t *revprops;
1090 svn_stream_t *stream;
1091 apr_file_t *file;
1092 svn_stringbuf_t *serialized;
1093 apr_size_t new_total_size;
1094 int changed_index;
1095
1096 /* read contents of the current pack file */
1097 SVN_ERR(read_pack_revprop(&revprops, fs, rev, TRUE, FALSE, pool));
1098
1099 /* serialize the new revprops */
1100 serialized = svn_stringbuf_create_empty(pool);
1101 stream = svn_stream_from_stringbuf(serialized, pool);
1102 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
1103 SVN_ERR(svn_stream_close(stream));
1104
1105 /* calculate the size of the new data */
1106 changed_index = (int)(rev - revprops->start_revision);
1107 new_total_size = revprops->total_size - revprops->serialized_size
1108 + serialized->len
1109 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
1110
1111 APR_ARRAY_IDX(revprops->sizes, changed_index, apr_size_t) = serialized->len;
1112
1113 /* can we put the new data into the same pack as the before? */
1114 if ( new_total_size < ffd->revprop_pack_size
1115 || revprops->sizes->nelts == 1)
1116 {
1117 /* simply replace the old pack file with new content as we do it
1118 * in the non-packed case */
1119
1120 *final_path = svn_dirent_join(revprops->folder, revprops->filename,
1121 pool);
1122 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
1123 svn_io_file_del_none, pool, pool));
1124 SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
1125 changed_index, serialized, new_total_size,
1126 file, pool));
1127 }
1128 else
1129 {
1130 /* split the pack file into two of roughly equal size */
1131 int right_count, left_count, i;
1132
1133 int left = 0;
1134 int right = revprops->sizes->nelts - 1;
1135 apr_size_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
1136 apr_size_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
1137
1138 /* let left and right side grow such that their size difference
1139 * is minimal after each step. */
1140 while (left <= right)
1141 if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_size_t)
1142 < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_size_t))
1143 {
1144 left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_size_t)
1145 + SVN_INT64_BUFFER_SIZE;
1146 ++left;
1147 }
1148 else
1149 {
1150 right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_size_t)
1151 + SVN_INT64_BUFFER_SIZE;
1152 --right;
1153 }
1154
1155 /* since the items need much less than SVN_INT64_BUFFER_SIZE
1156 * bytes to represent their length, the split may not be optimal */
1157 left_count = left;
1158 right_count = revprops->sizes->nelts - left;
1159
1160 /* if new_size is large, one side may exceed the pack size limit.
1161 * In that case, split before and after the modified revprop.*/
1162 if ( left_size > ffd->revprop_pack_size
1163 || right_size > ffd->revprop_pack_size)
1164 {
1165 left_count = changed_index;
1166 right_count = revprops->sizes->nelts - left_count - 1;
1167 }
1168
1169 /* write the new, split files */
1170 if (left_count)
1171 {
1172 SVN_ERR(repack_file_open(&file, fs, revprops, 0,
1173 left_count, files_to_delete, pool));
1174 SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
1175 changed_index, serialized, new_total_size,
1176 file, pool));
1177 }
1178
1179 if (left_count + right_count < revprops->sizes->nelts)
1180 {
1181 SVN_ERR(repack_file_open(&file, fs, revprops, changed_index,
1182 changed_index + 1, files_to_delete,
1183 pool));
1184 SVN_ERR(repack_revprops(fs, revprops, changed_index,
1185 changed_index + 1,
1186 changed_index, serialized, new_total_size,
1187 file, pool));
1188 }
1189
1190 if (right_count)
1191 {
1192 SVN_ERR(repack_file_open(&file, fs, revprops,
1193 revprops->sizes->nelts - right_count,
1194 revprops->sizes->nelts,
1195 files_to_delete, pool));
1196 SVN_ERR(repack_revprops(fs, revprops,
1197 revprops->sizes->nelts - right_count,
1198 revprops->sizes->nelts, changed_index,
1199 serialized, new_total_size, file,
1200 pool));
1201 }
1202
1203 /* write the new manifest */
1204 *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
1205 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
1206 svn_io_file_del_none, pool, pool));
1207 stream = svn_stream_from_aprfile2(file, TRUE, pool);
1208 for (i = 0; i < revprops->manifest->nelts; ++i)
1209 {
1210 const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
1211 const char*);
1212 SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename));
1213 }
1214 SVN_ERR(svn_stream_close(stream));
1215 if (ffd->flush_to_disk)
1216 SVN_ERR(svn_io_file_flush_to_disk(file, pool));
1217 SVN_ERR(svn_io_file_close(file, pool));
1218 }
1219
1220 return SVN_NO_ERROR;
1221 }
1222
1223 /* Set the revision property list of revision REV in filesystem FS to
1224 PROPLIST. Use POOL for temporary allocations. */
1225 svn_error_t *
svn_fs_fs__set_revision_proplist(svn_fs_t * fs,svn_revnum_t rev,apr_hash_t * proplist,apr_pool_t * pool)1226 svn_fs_fs__set_revision_proplist(svn_fs_t *fs,
1227 svn_revnum_t rev,
1228 apr_hash_t *proplist,
1229 apr_pool_t *pool)
1230 {
1231 svn_boolean_t is_packed;
1232 const char *final_path;
1233 const char *tmp_path;
1234 const char *perms_reference;
1235 apr_array_header_t *files_to_delete = NULL;
1236
1237 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
1238
1239 /* this info will not change while we hold the global FS write lock */
1240 is_packed = svn_fs_fs__is_packed_revprop(fs, rev);
1241
1242 /* Serialize the new revprop data */
1243 if (is_packed)
1244 SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
1245 fs, rev, proplist, pool));
1246 else
1247 SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
1248 fs, rev, proplist, pool));
1249
1250 /* Previous cache contents is invalid now. */
1251 svn_fs_fs__reset_revprop_cache(fs);
1252
1253 /* We use the rev file of this revision as the perms reference,
1254 * because when setting revprops for the first time, the revprop
1255 * file won't exist and therefore can't serve as its own reference.
1256 * (Whereas the rev file should already exist at this point.)
1257 */
1258 perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool);
1259
1260 /* Now, switch to the new revprop data. */
1261 SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
1262 files_to_delete, pool));
1263
1264 return SVN_NO_ERROR;
1265 }
1266
1267 /* Return TRUE, if for REVISION in FS, we can find the revprop pack file.
1268 * Use POOL for temporary allocations.
1269 * Set *MISSING, if the reason is a missing manifest or pack file.
1270 */
1271 svn_boolean_t
svn_fs_fs__packed_revprop_available(svn_boolean_t * missing,svn_fs_t * fs,svn_revnum_t revision,apr_pool_t * pool)1272 svn_fs_fs__packed_revprop_available(svn_boolean_t *missing,
1273 svn_fs_t *fs,
1274 svn_revnum_t revision,
1275 apr_pool_t *pool)
1276 {
1277 fs_fs_data_t *ffd = fs->fsap_data;
1278 svn_stringbuf_t *content = NULL;
1279
1280 /* try to read the manifest file */
1281 const char *folder
1282 = svn_fs_fs__path_revprops_pack_shard(fs, revision, pool);
1283 const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool);
1284
1285 svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content,
1286 missing,
1287 manifest_path,
1288 FALSE,
1289 pool);
1290
1291 /* if the manifest cannot be read, consider the pack files inaccessible
1292 * even if the file itself exists. */
1293 if (err)
1294 {
1295 svn_error_clear(err);
1296 return FALSE;
1297 }
1298
1299 if (*missing)
1300 return FALSE;
1301
1302 /* parse manifest content until we find the entry for REVISION.
1303 * Revision 0 is never packed. */
1304 revision = revision < ffd->max_files_per_dir
1305 ? revision - 1
1306 : revision % ffd->max_files_per_dir;
1307 while (content->data)
1308 {
1309 char *next = strchr(content->data, '\n');
1310 if (next)
1311 {
1312 *next = 0;
1313 ++next;
1314 }
1315
1316 if (revision-- == 0)
1317 {
1318 /* the respective pack file must exist (and be a file) */
1319 svn_node_kind_t kind;
1320 err = svn_io_check_path(svn_dirent_join(folder, content->data,
1321 pool),
1322 &kind, pool);
1323 if (err)
1324 {
1325 svn_error_clear(err);
1326 return FALSE;
1327 }
1328
1329 *missing = kind == svn_node_none;
1330 return kind == svn_node_file;
1331 }
1332
1333 content->data = next;
1334 }
1335
1336 return FALSE;
1337 }
1338
1339
1340 /****** Packing FSFS shards *********/
1341
1342 svn_error_t *
svn_fs_fs__copy_revprops(const char * pack_file_dir,const char * pack_filename,const char * shard_path,svn_revnum_t start_rev,svn_revnum_t end_rev,apr_array_header_t * sizes,apr_size_t total_size,int compression_level,svn_boolean_t flush_to_disk,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)1343 svn_fs_fs__copy_revprops(const char *pack_file_dir,
1344 const char *pack_filename,
1345 const char *shard_path,
1346 svn_revnum_t start_rev,
1347 svn_revnum_t end_rev,
1348 apr_array_header_t *sizes,
1349 apr_size_t total_size,
1350 int compression_level,
1351 svn_boolean_t flush_to_disk,
1352 svn_cancel_func_t cancel_func,
1353 void *cancel_baton,
1354 apr_pool_t *scratch_pool)
1355 {
1356 svn_stream_t *pack_stream;
1357 apr_file_t *pack_file;
1358 svn_revnum_t rev;
1359 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1360
1361 /* create empty data buffer and a write stream on top of it */
1362 svn_stringbuf_t *uncompressed
1363 = svn_stringbuf_create_ensure(total_size, scratch_pool);
1364 svn_stringbuf_t *compressed
1365 = svn_stringbuf_create_empty(scratch_pool);
1366 pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
1367
1368 /* write the pack file header */
1369 SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0,
1370 sizes->nelts, iterpool));
1371
1372 /* Some useful paths. */
1373 SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir,
1374 pack_filename,
1375 scratch_pool),
1376 APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
1377 scratch_pool));
1378
1379 /* Iterate over the revisions in this shard, squashing them together. */
1380 for (rev = start_rev; rev <= end_rev; rev++)
1381 {
1382 const char *path;
1383 svn_stream_t *stream;
1384 apr_file_t *file;
1385
1386 svn_pool_clear(iterpool);
1387
1388 /* Construct the file name. */
1389 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1390 iterpool);
1391
1392 /* Copy all the bits from the non-packed revprop file to the end of
1393 * the pack file. Use unbuffered apr_file_t since we're going to
1394 * write using 16kb chunks. */
1395 SVN_ERR(svn_io_file_open(&file, path, APR_READ, APR_OS_DEFAULT,
1396 iterpool));
1397 stream = svn_stream_from_aprfile2(file, FALSE, iterpool);
1398 SVN_ERR(svn_stream_copy3(stream, pack_stream,
1399 cancel_func, cancel_baton, iterpool));
1400 }
1401
1402 /* flush stream buffers to content buffer */
1403 SVN_ERR(svn_stream_close(pack_stream));
1404
1405 /* compress the content (or just store it for COMPRESSION_LEVEL 0) */
1406 SVN_ERR(svn__compress_zlib(uncompressed->data, uncompressed->len,
1407 compressed, compression_level));
1408
1409 /* write the pack file content to disk */
1410 SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len,
1411 NULL, scratch_pool));
1412 if (flush_to_disk)
1413 SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool));
1414 SVN_ERR(svn_io_file_close(pack_file, scratch_pool));
1415
1416 svn_pool_destroy(iterpool);
1417
1418 return SVN_NO_ERROR;
1419 }
1420
1421 svn_error_t *
svn_fs_fs__pack_revprops_shard(const char * pack_file_dir,const char * shard_path,apr_int64_t shard,int max_files_per_dir,apr_int64_t max_pack_size,int compression_level,svn_boolean_t flush_to_disk,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)1422 svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
1423 const char *shard_path,
1424 apr_int64_t shard,
1425 int max_files_per_dir,
1426 apr_int64_t max_pack_size,
1427 int compression_level,
1428 svn_boolean_t flush_to_disk,
1429 svn_cancel_func_t cancel_func,
1430 void *cancel_baton,
1431 apr_pool_t *scratch_pool)
1432 {
1433 const char *manifest_file_path, *pack_filename = NULL;
1434 apr_file_t *manifest_file;
1435 svn_stream_t *manifest_stream;
1436 svn_revnum_t start_rev, end_rev, rev;
1437 apr_size_t total_size;
1438 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1439 apr_array_header_t *sizes;
1440
1441 /* Sanitize config file values. */
1442 apr_size_t max_size = (apr_size_t)MIN(MAX(max_pack_size, 1),
1443 SVN_MAX_OBJECT_SIZE);
1444
1445 /* Some useful paths. */
1446 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
1447 scratch_pool);
1448
1449 /* Remove any existing pack file for this shard, since it is incomplete. */
1450 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
1451 scratch_pool));
1452
1453 /* Create the new directory and manifest file stream. */
1454 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
1455
1456 SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path,
1457 APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL,
1458 APR_OS_DEFAULT, scratch_pool));
1459 manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE,
1460 scratch_pool);
1461
1462 /* revisions to handle. Special case: revision 0 */
1463 start_rev = (svn_revnum_t) (shard * max_files_per_dir);
1464 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
1465 if (start_rev == 0)
1466 ++start_rev;
1467 /* Special special case: if max_files_per_dir is 1, then at this point
1468 start_rev == 1 and end_rev == 0 (!). Fortunately, everything just
1469 works. */
1470
1471 /* initialize the revprop size info */
1472 sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_size_t));
1473 total_size = 2 * SVN_INT64_BUFFER_SIZE;
1474
1475 /* Iterate over the revisions in this shard, determine their size and
1476 * squashing them together into pack files. */
1477 for (rev = start_rev; rev <= end_rev; rev++)
1478 {
1479 apr_finfo_t finfo;
1480 const char *path;
1481
1482 svn_pool_clear(iterpool);
1483
1484 /* Get the size of the file. */
1485 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1486 iterpool);
1487 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
1488
1489 /* If we already have started a pack file and this revprop cannot be
1490 * appended to it, write the previous pack file. Note this overflow
1491 * check works because we enforced MAX_SIZE <= SVN_MAX_OBJECT_SIZE. */
1492 if (sizes->nelts != 0
1493 && ( finfo.size > max_size
1494 || total_size > max_size
1495 || SVN_INT64_BUFFER_SIZE + finfo.size > max_size - total_size))
1496 {
1497 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1498 shard_path, start_rev, rev-1,
1499 sizes, total_size,
1500 compression_level, flush_to_disk,
1501 cancel_func, cancel_baton,
1502 iterpool));
1503
1504 /* next pack file starts empty again */
1505 apr_array_clear(sizes);
1506 total_size = 2 * SVN_INT64_BUFFER_SIZE;
1507 start_rev = rev;
1508 }
1509
1510 /* Update the manifest. Allocate a file name for the current pack
1511 * file if it is a new one */
1512 if (sizes->nelts == 0)
1513 pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev);
1514
1515 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n",
1516 pack_filename));
1517
1518 /* add to list of files to put into the current pack file */
1519 APR_ARRAY_PUSH(sizes, apr_size_t) = finfo.size;
1520 total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
1521 }
1522
1523 /* write the last pack file */
1524 if (sizes->nelts != 0)
1525 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1526 shard_path, start_rev, rev-1,
1527 sizes, (apr_size_t)total_size,
1528 compression_level, flush_to_disk,
1529 cancel_func, cancel_baton, iterpool));
1530
1531 /* flush the manifest file to disk and update permissions */
1532 SVN_ERR(svn_stream_close(manifest_stream));
1533 if (flush_to_disk)
1534 SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool));
1535 SVN_ERR(svn_io_file_close(manifest_file, iterpool));
1536 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
1537
1538 svn_pool_destroy(iterpool);
1539
1540 return SVN_NO_ERROR;
1541 }
1542
1543 svn_error_t *
svn_fs_fs__delete_revprops_shard(const char * shard_path,apr_int64_t shard,int max_files_per_dir,svn_cancel_func_t cancel_func,void * cancel_baton,apr_pool_t * scratch_pool)1544 svn_fs_fs__delete_revprops_shard(const char *shard_path,
1545 apr_int64_t shard,
1546 int max_files_per_dir,
1547 svn_cancel_func_t cancel_func,
1548 void *cancel_baton,
1549 apr_pool_t *scratch_pool)
1550 {
1551 if (shard == 0)
1552 {
1553 apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1554 int i;
1555
1556 /* delete all files except the one for revision 0 */
1557 for (i = 1; i < max_files_per_dir; ++i)
1558 {
1559 const char *path;
1560 svn_pool_clear(iterpool);
1561
1562 path = svn_dirent_join(shard_path,
1563 apr_psprintf(iterpool, "%d", i),
1564 iterpool);
1565 if (cancel_func)
1566 SVN_ERR(cancel_func(cancel_baton));
1567
1568 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
1569 }
1570
1571 svn_pool_destroy(iterpool);
1572 }
1573 else
1574 SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
1575 cancel_func, cancel_baton, scratch_pool));
1576
1577 return SVN_NO_ERROR;
1578 }
1579
1580