1
/* revprops.c --- everything needed to handle revprops in FSFS
3
* ====================================================================
4
* Licensed to the Apache Software Foundation (ASF) under one
5
* or more contributor license agreements. See the NOTICE file
6
* distributed with this work for additional information
7
* regarding copyright ownership. The ASF licenses this file
8
* to you under the Apache License, Version 2.0 (the
9
* "License"); you may not use this file except in compliance
10
* with the License. You may obtain a copy of the License at
12
* http://www.apache.org/licenses/LICENSE-2.0
14
* Unless required by applicable law or agreed to in writing,
15
* software distributed under the License is distributed on an
16
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17
* KIND, either express or implied. See the License for the
18
* specific language governing permissions and limitations
20
* ====================================================================
25
#include "svn_pools.h"
27
#include "svn_dirent_uri.h"
33
#include "private/svn_subr_private.h"
34
#include "private/svn_string_private.h"
35
#include "../libsvn_fs/fs-loader.h"
37
#include "svn_private_config.h"
39
/* Give writing processes 10 seconds to replace an existing revprop
40
file with a new one. After that time, we assume that the writing
41
process got aborted and that we have re-read revprops. */
42
#define REVPROP_CHANGE_TIMEOUT (10 * 1000000)
45
svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs,
46
svn_fs_upgrade_notify_t notify_func,
48
svn_cancel_func_t cancel_func,
50
apr_pool_t *scratch_pool)
52
fs_fs_data_t *ffd = fs->fsap_data;
53
const char *revprops_shard_path;
54
const char *revprops_pack_file_dir;
56
apr_int64_t first_unpacked_shard
57
= ffd->min_unpacked_rev / ffd->max_files_per_dir;
59
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
60
const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
62
int compression_level = ffd->compress_packed_revprops
63
? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
64
: SVN_DELTA_COMPRESSION_LEVEL_NONE;
66
/* first, pack all revprops shards to match the packed revision shards */
67
for (shard = 0; shard < first_unpacked_shard; ++shard)
69
svn_pool_clear(iterpool);
71
revprops_pack_file_dir = svn_dirent_join(revsprops_dir,
72
apr_psprintf(iterpool,
73
"%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD,
76
revprops_shard_path = svn_dirent_join(revsprops_dir,
77
apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
80
SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir,
82
shard, ffd->max_files_per_dir,
83
(int)(0.9 * ffd->revprop_pack_size),
85
cancel_func, cancel_baton,
88
SVN_ERR(notify_func(notify_baton, shard,
89
svn_fs_upgrade_pack_revprops, iterpool));
92
svn_pool_destroy(iterpool);
98
svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs,
99
svn_fs_upgrade_notify_t notify_func,
101
svn_cancel_func_t cancel_func,
103
apr_pool_t *scratch_pool)
105
fs_fs_data_t *ffd = fs->fsap_data;
106
const char *revprops_shard_path;
108
apr_int64_t first_unpacked_shard
109
= ffd->min_unpacked_rev / ffd->max_files_per_dir;
111
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
112
const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR,
115
/* delete the non-packed revprops shards afterwards */
116
for (shard = 0; shard < first_unpacked_shard; ++shard)
118
svn_pool_clear(iterpool);
120
revprops_shard_path = svn_dirent_join(revsprops_dir,
121
apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard),
123
SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path,
125
ffd->max_files_per_dir,
126
cancel_func, cancel_baton,
129
SVN_ERR(notify_func(notify_baton, shard,
130
svn_fs_upgrade_cleanup_revprops, iterpool));
133
svn_pool_destroy(iterpool);
138
/* Container for all data required to access the packed revprop file
139
* for a given REVISION. This structure will be filled incrementally
140
* by read_pack_revprops() its sub-routines.
142
typedef struct packed_revprops_t
144
/* revision number to read (not necessarily the first in the pack) */
145
svn_revnum_t revision;
147
/* current revprop generation. Used when populating the revprop cache */
148
apr_int64_t generation;
150
/* the actual revision properties */
151
apr_hash_t *properties;
153
/* their size when serialized to a single string
154
* (as found in PACKED_REVPROPS) */
155
apr_size_t serialized_size;
158
/* name of the pack file (without folder path) */
159
const char *filename;
161
/* packed shard folder path */
164
/* sum of values in SIZES */
165
apr_size_t total_size;
167
/* first revision in the pack (>= MANIFEST_START) */
168
svn_revnum_t start_revision;
170
/* size of the revprops in PACKED_REVPROPS */
171
apr_array_header_t *sizes;
173
/* offset of the revprops in PACKED_REVPROPS */
174
apr_array_header_t *offsets;
177
/* concatenation of the serialized representation of all revprops
178
* in the pack, i.e. the pack content without header and compression */
179
svn_stringbuf_t *packed_revprops;
181
/* First revision covered by MANIFEST.
182
* Will equal the shard start revision or 1, for the 1st shard. */
183
svn_revnum_t manifest_start;
185
/* content of the manifest.
186
* Maps long(rev - MANIFEST_START) to const char* pack file name */
187
apr_array_header_t *manifest;
190
/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES.
191
* Also, put them into the revprop cache, if activated, for future use.
192
* Three more parameters are being used to update the revprop cache: FS is
193
* our file system, the revprops belong to REVISION and the global revprop
194
* GENERATION is used as well.
196
* The returned hash will be allocated in POOL, SCRATCH_POOL is being used
197
* for temporary allocations.
200
parse_revprop(apr_hash_t **properties,
202
svn_revnum_t revision,
203
apr_int64_t generation,
204
svn_string_t *content,
206
apr_pool_t *scratch_pool)
208
svn_stream_t *stream = svn_stream_from_string(content, scratch_pool);
209
*properties = apr_hash_make(pool);
211
SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool),
212
apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.",
218
/* Read the non-packed revprops for revision REV in FS, put them into the
219
* revprop cache if activated and return them in *PROPERTIES. GENERATION
220
* is the current revprop generation.
222
* If the data could not be read due to an otherwise recoverable error,
223
* leave *PROPERTIES unchanged. No error will be returned in that case.
225
* Allocations will be done in POOL.
228
read_non_packed_revprop(apr_hash_t **properties,
231
apr_int64_t generation,
234
svn_stringbuf_t *content = NULL;
235
apr_pool_t *iterpool = svn_pool_create(pool);
236
svn_boolean_t missing = FALSE;
240
i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content;
243
svn_pool_clear(iterpool);
244
SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content,
246
svn_fs_fs__path_revprops(fs, rev, iterpool),
247
i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT ,
252
SVN_ERR(parse_revprop(properties, fs, rev, generation,
253
svn_stringbuf__morph_into_string(content),
256
svn_pool_clear(iterpool);
261
/* Return the minimum length of any packed revprop file name in REVPROPS. */
263
get_min_filename_len(packed_revprops_t *revprops)
265
char number_buffer[SVN_INT64_BUFFER_SIZE];
267
/* The revprop filenames have the format <REV>.<COUNT> - with <REV> being
268
* at least the first rev in the shard and <COUNT> having at least one
269
* digit. Thus, the minimum is 2 + #decimal places in the start rev.
271
return svn__i64toa(number_buffer, revprops->manifest_start) + 2;
274
/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST
275
* members. Use POOL for allocating results and SCRATCH_POOL for temporaries.
278
get_revprop_packname(svn_fs_t *fs,
279
packed_revprops_t *revprops,
281
apr_pool_t *scratch_pool)
283
fs_fs_data_t *ffd = fs->fsap_data;
284
svn_stringbuf_t *content = NULL;
285
const char *manifest_file_path;
287
char *buffer, *buffer_end;
288
const char **filenames, **filenames_end;
289
apr_size_t min_filename_len;
291
/* Determine the dimensions. Rev 0 is excluded from the first shard. */
292
rev_count = ffd->max_files_per_dir;
293
revprops->manifest_start
294
= revprops->revision - (revprops->revision % rev_count);
295
if (revprops->manifest_start == 0)
297
++revprops->manifest_start;
301
revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*));
303
/* No line in the file can be less than this number of chars long. */
304
min_filename_len = get_min_filename_len(revprops);
306
/* Read the content of the manifest file */
308
= svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool);
310
= svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
312
SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool));
314
/* There CONTENT must have a certain minimal size and there no
315
* unterminated lines at the end of the file. Both guarantees also
316
* simplify the parser loop below.
318
if ( content->len < rev_count * (min_filename_len + 1)
319
|| content->data[content->len - 1] != '\n')
320
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
321
_("Packed revprop manifest for r%ld not "
322
"properly terminated"), revprops->revision);
324
/* Chop (parse) the manifest CONTENT into filenames, one per line.
325
* We only have to replace all newlines with NUL and add all line
326
* starts to REVPROPS->MANIFEST.
328
* There must be exactly REV_COUNT lines and that is the number of
329
* lines we parse from BUFFER to FILENAMES. Set the end pointer for
330
* the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid
331
* BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN.
333
* Please note that this loop is performance critical for e.g. 'svn log'.
334
* It is run 1000x per revprop access, i.e. per revision and about
335
* 50 million times per sec (and CPU core).
337
for (filenames = (const char **)revprops->manifest->elts,
338
filenames_end = filenames + rev_count,
339
buffer = content->data,
340
buffer_end = buffer + content->len - min_filename_len;
341
(filenames < filenames_end) && (buffer < buffer_end);
344
/* BUFFER always points to the start of the next line / filename. */
347
/* Find the next EOL. This is guaranteed to stay within the CONTENT
348
* buffer because we left enough room after BUFFER_END and we know
349
* we will always see a newline as the last non-NUL char. */
350
buffer += min_filename_len;
351
while (*buffer != '\n')
354
/* Found EOL. Turn it into the filename terminator and move BUFFER
355
* to the start of the next line or CONTENT buffer end. */
360
/* We must have reached the end of both buffers. */
361
if (buffer < content->data + content->len)
362
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
363
_("Packed revprop manifest for r%ld "
364
"has too many entries"), revprops->revision);
366
if (filenames < filenames_end)
367
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
368
_("Packed revprop manifest for r%ld "
369
"has too few entries"), revprops->revision);
371
/* The target array has now exactly one entry per revision. */
372
revprops->manifest->nelts = rev_count;
374
/* Now get the file name */
375
idx = (int)(revprops->revision - revprops->manifest_start);
376
revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*);
381
/* Return TRUE, if revision R1 and R2 refer to the same shard in FS.
384
same_shard(svn_fs_t *fs,
388
fs_fs_data_t *ffd = fs->fsap_data;
389
return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir);
392
/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS,
393
* fill the START_REVISION member, and make PACKED_REVPROPS point to the
394
* first serialized revprop. If READ_ALL is set, initialize the SIZES
395
* and OFFSETS members as well.
397
* Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as
398
* well as the SERIALIZED_SIZE member. If revprop caching has been
399
* enabled, parse all revprops in the pack and cache them.
402
parse_packed_revprops(svn_fs_t *fs,
403
packed_revprops_t *revprops,
404
svn_boolean_t read_all,
406
apr_pool_t *scratch_pool)
408
svn_stream_t *stream;
409
apr_int64_t first_rev, count, i;
411
const char *header_end;
412
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
414
/* decompress (even if the data is only "stored", there is still a
415
* length header to remove) */
416
svn_stringbuf_t *compressed = revprops->packed_revprops;
417
svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool);
418
SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX));
420
/* read first revision number and number of revisions in the pack */
421
stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
422
SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream,
424
SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream,
427
/* Check revision range for validity. */
428
if ( !same_shard(fs, revprops->revision, first_rev)
429
|| !same_shard(fs, revprops->revision, first_rev + count - 1)
431
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
432
_("Revprop pack for revision r%ld"
433
" contains revprops for r%ld .. r%ld"),
435
(svn_revnum_t)first_rev,
436
(svn_revnum_t)(first_rev + count -1));
438
/* Since start & end are in the same shard, it is enough to just test
439
* the FIRST_REV for being actually packed. That will also cover the
440
* special case of rev 0 never being packed. */
441
if (!svn_fs_fs__is_packed_revprop(fs, first_rev))
442
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
443
_("Revprop pack for revision r%ld"
444
" starts at non-packed revisions r%ld"),
445
revprops->revision, (svn_revnum_t)first_rev);
447
/* make PACKED_REVPROPS point to the first char after the header.
448
* This is where the serialized revprops are. */
449
header_end = strstr(uncompressed->data, "\n\n");
450
if (header_end == NULL)
451
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
452
_("Header end not found"));
454
offset = header_end - uncompressed->data + 2;
456
revprops->packed_revprops = svn_stringbuf_create_empty(pool);
457
revprops->packed_revprops->data = uncompressed->data + offset;
458
revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset);
459
revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset);
461
/* STREAM still points to the first entry in the sizes list. */
462
revprops->start_revision = (svn_revnum_t)first_rev;
465
/* Init / construct REVPROPS members. */
466
revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset));
467
revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset));
470
/* Now parse, revision by revision, the size and content of each
471
* revisions' revprops. */
472
for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i)
475
svn_string_t serialized;
476
svn_revnum_t revision = (svn_revnum_t)(first_rev + i);
477
svn_pool_clear(iterpool);
479
/* read & check the serialized size */
480
SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream,
482
if (size + offset > (apr_int64_t)revprops->packed_revprops->len)
483
return svn_error_create(SVN_ERR_FS_CORRUPT, NULL,
484
_("Packed revprop size exceeds pack file size"));
486
/* Parse this revprops list, if necessary */
487
serialized.data = revprops->packed_revprops->data + offset;
488
serialized.len = (apr_size_t)size;
490
if (revision == revprops->revision)
492
SVN_ERR(parse_revprop(&revprops->properties, fs, revision,
493
revprops->generation, &serialized,
495
revprops->serialized_size = serialized.len;
497
/* If we only wanted the revprops for REVISION then we are done. */
504
/* fill REVPROPS data structures */
505
APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len;
506
APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset;
508
revprops->total_size += serialized.len;
510
offset += serialized.len;
516
/* In filesystem FS, read the packed revprops for revision REV into
517
* *REVPROPS. Use GENERATION to populate the revprop cache, if enabled.
518
* If you want to modify revprop contents / update REVPROPS, READ_ALL
519
* must be set. Otherwise, only the properties of REV are being provided.
520
* Allocate data in POOL.
523
read_pack_revprop(packed_revprops_t **revprops,
526
apr_int64_t generation,
527
svn_boolean_t read_all,
530
apr_pool_t *iterpool = svn_pool_create(pool);
531
svn_boolean_t missing = FALSE;
533
packed_revprops_t *result;
536
/* someone insisted that REV is packed. Double-check if necessary */
537
if (!svn_fs_fs__is_packed_revprop(fs, rev))
538
SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool));
540
if (!svn_fs_fs__is_packed_revprop(fs, rev))
541
return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
542
_("No such packed revision %ld"), rev);
544
/* initialize the result data structure */
545
result = apr_pcalloc(pool, sizeof(*result));
546
result->revision = rev;
547
result->generation = generation;
549
/* try to read the packed revprops. This may require retries if we have
550
* concurrent writers. */
552
i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops;
555
const char *file_path;
556
svn_pool_clear(iterpool);
558
/* there might have been concurrent writes.
559
* Re-read the manifest and the pack file.
561
SVN_ERR(get_revprop_packname(fs, result, pool, iterpool));
562
file_path = svn_dirent_join(result->folder,
565
SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops,
568
i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT,
572
/* the file content should be available now */
573
if (!result->packed_revprops)
574
return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL,
575
_("Failed to read revprop pack file for r%ld"), rev);
577
/* parse it. RESULT will be complete afterwards. */
578
err = parse_packed_revprops(fs, result, read_all, pool, iterpool);
579
svn_pool_destroy(iterpool);
581
return svn_error_createf(SVN_ERR_FS_CORRUPT, err,
582
_("Revprop pack file for r%ld is corrupt"), rev);
589
/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P.
591
* Allocations will be done in POOL.
594
svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p,
599
fs_fs_data_t *ffd = fs->fsap_data;
600
apr_int64_t generation = 0;
605
/* should they be available at all? */
606
SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
608
/* if REV had not been packed when we began, try reading it from the
609
* non-packed shard. If that fails, we will fall through to packed
611
if (!svn_fs_fs__is_packed_revprop(fs, rev))
613
svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev,
617
if (!APR_STATUS_IS_ENOENT(err->apr_err)
618
|| ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT)
619
return svn_error_trace(err);
621
svn_error_clear(err);
622
*proplist_p = NULL; /* in case read_non_packed_revprop changed it */
626
/* if revprop packing is available and we have not read the revprops, yet,
627
* try reading them from a packed shard. If that fails, REV is most
628
* likely invalid (or its revprops highly contested). */
629
if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p)
631
packed_revprops_t *revprops;
632
SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool));
633
*proplist_p = revprops->properties;
636
/* The revprops should have been there. Did we get them? */
638
return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
639
_("Could not read revprops for revision %ld"),
645
/* Serialize the revision property list PROPLIST of revision REV in
646
* filesystem FS to a non-packed file. Return the name of that temporary
647
* file in *TMP_PATH and the file path that it must be moved to in
650
* Use POOL for allocations.
653
write_non_packed_revprop(const char **final_path,
654
const char **tmp_path,
657
apr_hash_t *proplist,
661
svn_stream_t *stream;
662
*final_path = svn_fs_fs__path_revprops(fs, rev, pool);
664
/* ### do we have a directory sitting around already? we really shouldn't
665
### have to get the dirname here. */
666
SVN_ERR(svn_io_open_unique_file3(&file, tmp_path,
667
svn_dirent_dirname(*final_path, pool),
668
svn_io_file_del_none, pool, pool));
669
stream = svn_stream_from_aprfile2(file, TRUE, pool);
670
SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
671
SVN_ERR(svn_stream_close(stream));
673
/* Flush temporary file to disk and close it. */
674
SVN_ERR(svn_io_file_flush_to_disk(file, pool));
675
SVN_ERR(svn_io_file_close(file, pool));
680
/* After writing the new revprop file(s), call this function to move the
681
* file at TMP_PATH to FINAL_PATH and give it the permissions from
684
* Finally, delete all the temporary files given in FILES_TO_DELETE.
685
* The latter may be NULL.
687
* Use POOL for temporary allocations.
690
switch_to_new_revprop(svn_fs_t *fs,
691
const char *final_path,
692
const char *tmp_path,
693
const char *perms_reference,
694
apr_array_header_t *files_to_delete,
697
SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference,
700
/* Clean up temporary files, if necessary. */
703
apr_pool_t *iterpool = svn_pool_create(pool);
706
for (i = 0; i < files_to_delete->nelts; ++i)
708
const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*);
710
svn_pool_clear(iterpool);
711
SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
714
svn_pool_destroy(iterpool);
719
/* Write a pack file header to STREAM that starts at revision START_REVISION
720
* and contains the indexes [START,END) of SIZES.
723
serialize_revprops_header(svn_stream_t *stream,
724
svn_revnum_t start_revision,
725
apr_array_header_t *sizes,
730
apr_pool_t *iterpool = svn_pool_create(pool);
733
SVN_ERR_ASSERT(start < end);
735
/* start revision and entry count */
736
SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision));
737
SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start));
739
/* the sizes array */
740
for (i = start; i < end; ++i)
742
/* Non-standard pool usage.
744
* We only allocate a few bytes each iteration -- even with a
745
* million iterations we would still be in good shape memory-wise.
747
apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t);
748
SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n",
752
/* the double newline char indicates the end of the header */
753
SVN_ERR(svn_stream_printf(stream, iterpool, "\n"));
755
svn_pool_destroy(iterpool);
759
/* Writes the a pack file to FILE. It copies the serialized data
760
* from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX.
762
* The data for the latter is taken from NEW_SERIALIZED. Note, that
763
* CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is
764
* taken in that case but only a subset of the old data will be copied.
766
* NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size.
767
* POOL is used for temporary allocations.
770
repack_revprops(svn_fs_t *fs,
771
packed_revprops_t *revprops,
775
svn_stringbuf_t *new_serialized,
776
apr_off_t new_total_size,
780
fs_fs_data_t *ffd = fs->fsap_data;
781
svn_stream_t *stream;
784
/* create data empty buffers and the stream object */
785
svn_stringbuf_t *uncompressed
786
= svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool);
787
svn_stringbuf_t *compressed
788
= svn_stringbuf_create_empty(pool);
789
stream = svn_stream_from_stringbuf(uncompressed, pool);
791
/* write the header*/
792
SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start,
793
revprops->sizes, start, end, pool));
795
/* append the serialized revprops */
796
for (i = start; i < end; ++i)
797
if (i == changed_index)
799
SVN_ERR(svn_stream_write(stream,
800
new_serialized->data,
801
&new_serialized->len));
806
= (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t);
808
= (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t);
810
SVN_ERR(svn_stream_write(stream,
811
revprops->packed_revprops->data + offset,
815
/* flush the stream buffer (if any) to our underlying data buffer */
816
SVN_ERR(svn_stream_close(stream));
818
/* compress / store the data */
819
SVN_ERR(svn__compress(uncompressed,
821
ffd->compress_packed_revprops
822
? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
823
: SVN_DELTA_COMPRESSION_LEVEL_NONE));
825
/* finally, write the content to the target file, flush and close it */
826
SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len,
828
SVN_ERR(svn_io_file_flush_to_disk(file, pool));
829
SVN_ERR(svn_io_file_close(file, pool));
834
/* Allocate a new pack file name for revisions
835
* [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1]
836
* of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE,
837
* auto-create that array if necessary. Return an open file *FILE that is
841
repack_file_open(apr_file_t **file,
843
packed_revprops_t *revprops,
846
apr_array_header_t **files_to_delete,
850
const char *tag_string;
851
svn_string_t *new_filename;
854
= (int)(revprops->start_revision - revprops->manifest_start);
856
/* get the old (= current) file name and enlist it for later deletion */
857
const char *old_filename = APR_ARRAY_IDX(revprops->manifest,
858
start + manifest_offset,
861
if (*files_to_delete == NULL)
862
*files_to_delete = apr_array_make(pool, 3, sizeof(const char*));
864
APR_ARRAY_PUSH(*files_to_delete, const char*)
865
= svn_dirent_join(revprops->folder, old_filename, pool);
867
/* increase the tag part, i.e. the counter after the dot */
868
tag_string = strchr(old_filename, '.');
869
if (tag_string == NULL)
870
return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL,
871
_("Packed file '%s' misses a tag"),
874
SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1));
875
new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT,
876
revprops->start_revision + start,
879
/* update the manifest to point to the new file */
880
for (i = start; i < end; ++i)
881
APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*)
882
= new_filename->data;
885
SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder,
888
APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool));
893
/* For revision REV in filesystem FS, set the revision properties to
894
* PROPLIST. Return a new file in *TMP_PATH that the caller shall move
895
* to *FINAL_PATH to make the change visible. Files to be deleted will
896
* be listed in *FILES_TO_DELETE which may remain unchanged / unallocated.
897
* Use POOL for allocations.
900
write_packed_revprop(const char **final_path,
901
const char **tmp_path,
902
apr_array_header_t **files_to_delete,
905
apr_hash_t *proplist,
908
fs_fs_data_t *ffd = fs->fsap_data;
909
packed_revprops_t *revprops;
910
apr_int64_t generation = 0;
911
svn_stream_t *stream;
913
svn_stringbuf_t *serialized;
914
apr_off_t new_total_size;
917
/* read contents of the current pack file */
918
SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool));
920
/* serialize the new revprops */
921
serialized = svn_stringbuf_create_empty(pool);
922
stream = svn_stream_from_stringbuf(serialized, pool);
923
SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool));
924
SVN_ERR(svn_stream_close(stream));
926
/* calculate the size of the new data */
927
changed_index = (int)(rev - revprops->start_revision);
928
new_total_size = revprops->total_size - revprops->serialized_size
930
+ (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE;
932
APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len;
934
/* can we put the new data into the same pack as the before? */
935
if ( new_total_size < ffd->revprop_pack_size
936
|| revprops->sizes->nelts == 1)
938
/* simply replace the old pack file with new content as we do it
939
* in the non-packed case */
941
*final_path = svn_dirent_join(revprops->folder, revprops->filename,
943
SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
944
svn_io_file_del_none, pool, pool));
945
SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts,
946
changed_index, serialized, new_total_size,
951
/* split the pack file into two of roughly equal size */
952
int right_count, left_count, i;
955
int right = revprops->sizes->nelts - 1;
956
apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE;
957
apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE;
959
/* let left and right side grow such that their size difference
960
* is minimal after each step. */
961
while (left <= right)
962
if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
963
< right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t))
965
left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t)
966
+ SVN_INT64_BUFFER_SIZE;
971
right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)
972
+ SVN_INT64_BUFFER_SIZE;
976
/* since the items need much less than SVN_INT64_BUFFER_SIZE
977
* bytes to represent their length, the split may not be optimal */
979
right_count = revprops->sizes->nelts - left;
981
/* if new_size is large, one side may exceed the pack size limit.
982
* In that case, split before and after the modified revprop.*/
983
if ( left_size > ffd->revprop_pack_size
984
|| right_size > ffd->revprop_pack_size)
986
left_count = changed_index;
987
right_count = revprops->sizes->nelts - left_count - 1;
990
/* write the new, split files */
993
SVN_ERR(repack_file_open(&file, fs, revprops, 0,
994
left_count, files_to_delete, pool));
995
SVN_ERR(repack_revprops(fs, revprops, 0, left_count,
996
changed_index, serialized, new_total_size,
1000
if (left_count + right_count < revprops->sizes->nelts)
1002
SVN_ERR(repack_file_open(&file, fs, revprops, changed_index,
1003
changed_index + 1, files_to_delete,
1005
SVN_ERR(repack_revprops(fs, revprops, changed_index,
1007
changed_index, serialized, new_total_size,
1013
SVN_ERR(repack_file_open(&file, fs, revprops,
1014
revprops->sizes->nelts - right_count,
1015
revprops->sizes->nelts,
1016
files_to_delete, pool));
1017
SVN_ERR(repack_revprops(fs, revprops,
1018
revprops->sizes->nelts - right_count,
1019
revprops->sizes->nelts, changed_index,
1020
serialized, new_total_size, file,
1024
/* write the new manifest */
1025
*final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool);
1026
SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder,
1027
svn_io_file_del_none, pool, pool));
1029
for (i = 0; i < revprops->manifest->nelts; ++i)
1031
const char *filename = APR_ARRAY_IDX(revprops->manifest, i,
1033
SVN_ERR(svn_io_file_write_full(file, filename, strlen(filename),
1035
SVN_ERR(svn_io_file_putc('\n', file, pool));
1038
SVN_ERR(svn_io_file_flush_to_disk(file, pool));
1039
SVN_ERR(svn_io_file_close(file, pool));
1042
return SVN_NO_ERROR;
1045
/* Set the revision property list of revision REV in filesystem FS to
1046
PROPLIST. Use POOL for temporary allocations. */
1048
svn_fs_fs__set_revision_proplist(svn_fs_t *fs,
1050
apr_hash_t *proplist,
1053
svn_boolean_t is_packed;
1054
const char *final_path;
1055
const char *tmp_path;
1056
const char *perms_reference;
1057
apr_array_header_t *files_to_delete = NULL;
1059
SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool));
1061
/* this info will not change while we hold the global FS write lock */
1062
is_packed = svn_fs_fs__is_packed_revprop(fs, rev);
1064
/* Serialize the new revprop data */
1066
SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete,
1067
fs, rev, proplist, pool));
1069
SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path,
1070
fs, rev, proplist, pool));
1072
/* We use the rev file of this revision as the perms reference,
1073
* because when setting revprops for the first time, the revprop
1074
* file won't exist and therefore can't serve as its own reference.
1075
* (Whereas the rev file should already exist at this point.)
1077
perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool);
1079
/* Now, switch to the new revprop data. */
1080
SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference,
1081
files_to_delete, pool));
1083
return SVN_NO_ERROR;
1086
/* Return TRUE, if for REVISION in FS, we can find the revprop pack file.
1087
* Use POOL for temporary allocations.
1088
* Set *MISSING, if the reason is a missing manifest or pack file.
1091
svn_fs_fs__packed_revprop_available(svn_boolean_t *missing,
1093
svn_revnum_t revision,
1096
fs_fs_data_t *ffd = fs->fsap_data;
1097
svn_stringbuf_t *content = NULL;
1099
/* try to read the manifest file */
1101
= svn_fs_fs__path_revprops_pack_shard(fs, revision, pool);
1102
const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool);
1104
svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content,
1110
/* if the manifest cannot be read, consider the pack files inaccessible
1111
* even if the file itself exists. */
1114
svn_error_clear(err);
1121
/* parse manifest content until we find the entry for REVISION.
1122
* Revision 0 is never packed. */
1123
revision = revision < ffd->max_files_per_dir
1125
: revision % ffd->max_files_per_dir;
1126
while (content->data)
1128
char *next = strchr(content->data, '\n');
1135
if (revision-- == 0)
1137
/* the respective pack file must exist (and be a file) */
1138
svn_node_kind_t kind;
1139
err = svn_io_check_path(svn_dirent_join(folder, content->data,
1144
svn_error_clear(err);
1148
*missing = kind == svn_node_none;
1149
return kind == svn_node_file;
1152
content->data = next;
1159
/****** Packing FSFS shards *********/
1162
svn_fs_fs__copy_revprops(const char *pack_file_dir,
1163
const char *pack_filename,
1164
const char *shard_path,
1165
svn_revnum_t start_rev,
1166
svn_revnum_t end_rev,
1167
apr_array_header_t *sizes,
1168
apr_size_t total_size,
1169
int compression_level,
1170
svn_cancel_func_t cancel_func,
1172
apr_pool_t *scratch_pool)
1174
svn_stream_t *pack_stream;
1175
apr_file_t *pack_file;
1177
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1179
/* create empty data buffer and a write stream on top of it */
1180
svn_stringbuf_t *uncompressed
1181
= svn_stringbuf_create_ensure(total_size, scratch_pool);
1182
svn_stringbuf_t *compressed
1183
= svn_stringbuf_create_empty(scratch_pool);
1184
pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool);
1186
/* write the pack file header */
1187
SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0,
1188
sizes->nelts, iterpool));
1190
/* Some useful paths. */
1191
SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir,
1194
APR_WRITE | APR_CREATE, APR_OS_DEFAULT,
1197
/* Iterate over the revisions in this shard, squashing them together. */
1198
for (rev = start_rev; rev <= end_rev; rev++)
1201
svn_stream_t *stream;
1203
svn_pool_clear(iterpool);
1205
/* Construct the file name. */
1206
path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1209
/* Copy all the bits from the non-packed revprop file to the end of
1211
SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool));
1212
SVN_ERR(svn_stream_copy3(stream, pack_stream,
1213
cancel_func, cancel_baton, iterpool));
1216
/* flush stream buffers to content buffer */
1217
SVN_ERR(svn_stream_close(pack_stream));
1219
/* compress the content (or just store it for COMPRESSION_LEVEL 0) */
1220
SVN_ERR(svn__compress(uncompressed, compressed, compression_level));
1222
/* write the pack file content to disk */
1223
SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len,
1224
NULL, scratch_pool));
1225
SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool));
1226
SVN_ERR(svn_io_file_close(pack_file, scratch_pool));
1228
svn_pool_destroy(iterpool);
1230
return SVN_NO_ERROR;
1234
svn_fs_fs__pack_revprops_shard(const char *pack_file_dir,
1235
const char *shard_path,
1237
int max_files_per_dir,
1238
apr_off_t max_pack_size,
1239
int compression_level,
1240
svn_cancel_func_t cancel_func,
1242
apr_pool_t *scratch_pool)
1244
const char *manifest_file_path, *pack_filename = NULL;
1245
apr_file_t *manifest_file;
1246
svn_stream_t *manifest_stream;
1247
svn_revnum_t start_rev, end_rev, rev;
1248
apr_off_t total_size;
1249
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1250
apr_array_header_t *sizes;
1252
/* Some useful paths. */
1253
manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST,
1256
/* Remove any existing pack file for this shard, since it is incomplete. */
1257
SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton,
1260
/* Create the new directory and manifest file stream. */
1261
SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool));
1263
SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path,
1264
APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL,
1265
APR_OS_DEFAULT, scratch_pool));
1266
manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE,
1269
/* revisions to handle. Special case: revision 0 */
1270
start_rev = (svn_revnum_t) (shard * max_files_per_dir);
1271
end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1);
1274
/* Special special case: if max_files_per_dir is 1, then at this point
1275
start_rev == 1 and end_rev == 0 (!). Fortunately, everything just
1278
/* initialize the revprop size info */
1279
sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t));
1280
total_size = 2 * SVN_INT64_BUFFER_SIZE;
1282
/* Iterate over the revisions in this shard, determine their size and
1283
* squashing them together into pack files. */
1284
for (rev = start_rev; rev <= end_rev; rev++)
1289
svn_pool_clear(iterpool);
1291
/* Get the size of the file. */
1292
path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev),
1294
SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool));
1296
/* if we already have started a pack file and this revprop cannot be
1297
* appended to it, write the previous pack file. */
1298
if (sizes->nelts != 0 &&
1299
total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size)
1301
SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1302
shard_path, start_rev, rev-1,
1303
sizes, (apr_size_t)total_size,
1304
compression_level, cancel_func,
1305
cancel_baton, iterpool));
1307
/* next pack file starts empty again */
1308
apr_array_clear(sizes);
1309
total_size = 2 * SVN_INT64_BUFFER_SIZE;
1313
/* Update the manifest. Allocate a file name for the current pack
1314
* file if it is a new one */
1315
if (sizes->nelts == 0)
1316
pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev);
1318
SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n",
1321
/* add to list of files to put into the current pack file */
1322
APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size;
1323
total_size += SVN_INT64_BUFFER_SIZE + finfo.size;
1326
/* write the last pack file */
1327
if (sizes->nelts != 0)
1328
SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename,
1329
shard_path, start_rev, rev-1,
1330
sizes, (apr_size_t)total_size,
1331
compression_level, cancel_func,
1332
cancel_baton, iterpool));
1334
/* flush the manifest file to disk and update permissions */
1335
SVN_ERR(svn_stream_close(manifest_stream));
1336
SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool));
1337
SVN_ERR(svn_io_file_close(manifest_file, iterpool));
1338
SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool));
1340
svn_pool_destroy(iterpool);
1342
return SVN_NO_ERROR;
1346
svn_fs_fs__delete_revprops_shard(const char *shard_path,
1348
int max_files_per_dir,
1349
svn_cancel_func_t cancel_func,
1351
apr_pool_t *scratch_pool)
1355
apr_pool_t *iterpool = svn_pool_create(scratch_pool);
1358
/* delete all files except the one for revision 0 */
1359
for (i = 1; i < max_files_per_dir; ++i)
1362
svn_pool_clear(iterpool);
1364
path = svn_dirent_join(shard_path,
1365
apr_psprintf(iterpool, "%d", i),
1368
SVN_ERR((*cancel_func)(cancel_baton));
1370
SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool));
1373
svn_pool_destroy(iterpool);
1376
SVN_ERR(svn_io_remove_dir2(shard_path, TRUE,
1377
cancel_func, cancel_baton, scratch_pool));
1379
return SVN_NO_ERROR;
added
removed
subversion/libsvn_fs_fs/revprops.c
