1
/* fs_x.c --- filesystem operations specific to fs_x
3
* ====================================================================
4
* Licensed to the Apache Software Foundation (ASF) under one
5
* or more contributor license agreements. See the NOTICE file
6
* distributed with this work for additional information
7
* regarding copyright ownership. The ASF licenses this file
8
* to you under the Apache License, Version 2.0 (the
9
* "License"); you may not use this file except in compliance
10
* with the License. You may obtain a copy of the License at
12
* http://www.apache.org/licenses/LICENSE-2.0
14
* Unless required by applicable law or agreed to in writing,
15
* software distributed under the License is distributed on an
16
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17
* KIND, either express or implied. See the License for the
18
* specific language governing permissions and limitations
20
* ====================================================================
28
#include "svn_props.h"
30
#include "svn_dirent_uri.h"
31
#include "svn_sorts.h"
32
#include "svn_version.h"
34
#include "cached_data.h"
36
#include "rep-cache.h"
38
#include "transaction.h"
43
#include "private/svn_fs_util.h"
44
#include "private/svn_string_private.h"
45
#include "private/svn_subr_private.h"
46
#include "../libsvn_fs/fs-loader.h"
48
#include "svn_private_config.h"
50
/* The default maximum number of files per directory to store in the
51
rev and revprops directory. The number below is somewhat arbitrary,
52
and can be overridden by defining the macro while compiling; the
53
figure of 1000 is reasonable for VFAT filesystems, which are by far
54
the worst performers in this area. */
55
#ifndef SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR
56
#define SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR 1000
59
/* Begin deltification after a node history exceeded this this limit.
60
Useful values are 4 to 64 with 16 being a good compromise between
61
computational overhead and repository size savings.
62
Should be a power of 2.
63
Values < 2 will result in standard skip-delta behavior. */
64
#define SVN_FS_X_MAX_LINEAR_DELTIFICATION 16
66
/* Finding a deltification base takes operations proportional to the
67
number of changes being skipped. To prevent exploding runtime
68
during commits, limit the deltification range to this value.
69
Should be a power of 2 minus one.
70
Values < 1 disable deltification. */
71
#define SVN_FS_X_MAX_DELTIFICATION_WALK 1023
76
/* Check that BUF, a nul-terminated buffer of text from format file PATH,
77
contains only digits at OFFSET and beyond, raising an error if not.
79
Uses SCRATCH_POOL for temporary allocation. */
81
check_format_file_buffer_numeric(const char *buf,
84
apr_pool_t *scratch_pool)
86
return svn_fs_x__check_file_buffer_numeric(buf, offset, path, "Format",
90
/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format
91
number is not the same as a format number supported by this
94
check_format(int format)
96
/* Put blacklisted versions here. */
98
/* We support all formats from 1-current simultaneously */
99
if (1 <= format && format <= SVN_FS_X__FORMAT_NUMBER)
102
return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
103
_("Expected FS format between '1' and '%d'; found format '%d'"),
104
SVN_FS_X__FORMAT_NUMBER, format);
107
/* Read the format file at PATH and set *PFORMAT to the format version found
108
* and *MAX_FILES_PER_DIR to the shard size. Use SCRATCH_POOL for temporary
111
read_format(int *pformat,
112
int *max_files_per_dir,
114
apr_pool_t *scratch_pool)
116
svn_stream_t *stream;
117
svn_stringbuf_t *content;
118
svn_stringbuf_t *buf;
119
svn_boolean_t eos = FALSE;
121
SVN_ERR(svn_stringbuf_from_file2(&content, path, scratch_pool));
122
stream = svn_stream_from_stringbuf(content, scratch_pool);
123
SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, scratch_pool));
124
if (buf->len == 0 && eos)
126
/* Return a more useful error message. */
127
return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
128
_("Can't read first line of format file '%s'"),
129
svn_dirent_local_style(path, scratch_pool));
132
/* Check that the first line contains only digits. */
133
SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, scratch_pool));
134
SVN_ERR(svn_cstring_atoi(pformat, buf->data));
136
/* Check that we support this format at all */
137
SVN_ERR(check_format(*pformat));
139
/* Read any options. */
140
SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, scratch_pool));
141
if (!eos && strncmp(buf->data, "layout sharded ", 15) == 0)
143
/* Check that the argument is numeric. */
144
SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path,
146
SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15));
149
return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL,
150
_("'%s' contains invalid filesystem format option '%s'"),
151
svn_dirent_local_style(path, scratch_pool), buf->data);
156
/* Write the format number and maximum number of files per directory
157
to a new format file in PATH, possibly expecting to overwrite a
158
previously existing file.
160
Use SCRATCH_POOL for temporary allocation. */
162
svn_fs_x__write_format(svn_fs_t *fs,
163
svn_boolean_t overwrite,
164
apr_pool_t *scratch_pool)
167
const char *path = svn_fs_x__path_format(fs, scratch_pool);
168
svn_fs_x__data_t *ffd = fs->fsap_data;
170
SVN_ERR_ASSERT(1 <= ffd->format && ffd->format <= SVN_FS_X__FORMAT_NUMBER);
172
sb = svn_stringbuf_createf(scratch_pool, "%d\n", ffd->format);
173
svn_stringbuf_appendcstr(sb, apr_psprintf(scratch_pool,
174
"layout sharded %d\n",
175
ffd->max_files_per_dir));
177
/* svn_io_write_version_file() does a load of magic to allow it to
178
replace version files that already exist. We only need to do
179
that when we're allowed to overwrite an existing file. */
182
/* Create the file */
183
SVN_ERR(svn_io_file_create(path, sb->data, scratch_pool));
187
SVN_ERR(svn_io_write_atomic(path, sb->data, sb->len,
188
NULL /* copy_perms_path */, scratch_pool));
191
/* And set the perms to make it read only */
192
return svn_io_set_file_read_only(path, FALSE, scratch_pool);
195
/* Check that BLOCK_SIZE is a valid block / page size, i.e. it is within
196
* the range of what the current system may address in RAM and it is a
197
* power of 2. Assume that the element size within the block is ITEM_SIZE.
198
* Use SCRATCH_POOL for temporary allocations.
201
verify_block_size(apr_int64_t block_size,
202
apr_size_t item_size,
204
apr_pool_t *scratch_pool)
208
return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
209
_("%s is too small for fsfs.conf setting '%s'."),
210
apr_psprintf(scratch_pool,
215
if (block_size > SVN_MAX_OBJECT_SIZE / item_size)
216
return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
217
_("%s is too large for fsfs.conf setting '%s'."),
218
apr_psprintf(scratch_pool,
223
/* Ensure it is a power of two.
224
* For positive X, X & (X-1) will reset the lowest bit set.
225
* If the result is 0, at most one bit has been set. */
226
if (0 != (block_size & (block_size - 1)))
227
return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE, NULL,
228
_("%s is invalid for fsfs.conf setting '%s' "
229
"because it is not a power of 2."),
230
apr_psprintf(scratch_pool,
238
/* Read the configuration information of the file system at FS_PATH
239
* and set the respective values in FFD. Use pools as usual.
242
read_config(svn_fs_x__data_t *ffd,
244
apr_pool_t *result_pool,
245
apr_pool_t *scratch_pool)
247
svn_config_t *config;
248
apr_int64_t compression_level;
250
SVN_ERR(svn_config_read3(&config,
251
svn_dirent_join(fs_path, PATH_CONFIG, scratch_pool),
252
FALSE, FALSE, FALSE, scratch_pool));
254
/* Initialize ffd->rep_sharing_allowed. */
255
SVN_ERR(svn_config_get_bool(config, &ffd->rep_sharing_allowed,
256
CONFIG_SECTION_REP_SHARING,
257
CONFIG_OPTION_ENABLE_REP_SHARING, TRUE));
259
/* Initialize deltification settings in ffd. */
260
SVN_ERR(svn_config_get_int64(config, &ffd->max_deltification_walk,
261
CONFIG_SECTION_DELTIFICATION,
262
CONFIG_OPTION_MAX_DELTIFICATION_WALK,
263
SVN_FS_X_MAX_DELTIFICATION_WALK));
264
SVN_ERR(svn_config_get_int64(config, &ffd->max_linear_deltification,
265
CONFIG_SECTION_DELTIFICATION,
266
CONFIG_OPTION_MAX_LINEAR_DELTIFICATION,
267
SVN_FS_X_MAX_LINEAR_DELTIFICATION));
268
SVN_ERR(svn_config_get_int64(config, &compression_level,
269
CONFIG_SECTION_DELTIFICATION,
270
CONFIG_OPTION_COMPRESSION_LEVEL,
271
SVN_DELTA_COMPRESSION_LEVEL_DEFAULT));
272
ffd->delta_compression_level
273
= (int)MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE, compression_level),
274
SVN_DELTA_COMPRESSION_LEVEL_MAX);
276
/* Initialize revprop packing settings in ffd. */
277
SVN_ERR(svn_config_get_bool(config, &ffd->compress_packed_revprops,
278
CONFIG_SECTION_PACKED_REVPROPS,
279
CONFIG_OPTION_COMPRESS_PACKED_REVPROPS,
281
SVN_ERR(svn_config_get_int64(config, &ffd->revprop_pack_size,
282
CONFIG_SECTION_PACKED_REVPROPS,
283
CONFIG_OPTION_REVPROP_PACK_SIZE,
284
ffd->compress_packed_revprops
288
ffd->revprop_pack_size *= 1024;
290
/* I/O settings in ffd. */
291
SVN_ERR(svn_config_get_int64(config, &ffd->block_size,
293
CONFIG_OPTION_BLOCK_SIZE,
295
SVN_ERR(svn_config_get_int64(config, &ffd->l2p_page_size,
297
CONFIG_OPTION_L2P_PAGE_SIZE,
299
SVN_ERR(svn_config_get_int64(config, &ffd->p2l_page_size,
301
CONFIG_OPTION_P2L_PAGE_SIZE,
304
/* Don't accept unreasonable or illegal values.
305
* Block size and P2L page size are in kbytes;
306
* L2P blocks are arrays of apr_off_t. */
307
SVN_ERR(verify_block_size(ffd->block_size, 0x400,
308
CONFIG_OPTION_BLOCK_SIZE, scratch_pool));
309
SVN_ERR(verify_block_size(ffd->p2l_page_size, 0x400,
310
CONFIG_OPTION_P2L_PAGE_SIZE, scratch_pool));
311
SVN_ERR(verify_block_size(ffd->l2p_page_size, sizeof(apr_off_t),
312
CONFIG_OPTION_L2P_PAGE_SIZE, scratch_pool));
314
/* convert kBytes to bytes */
315
ffd->block_size *= 0x400;
316
ffd->p2l_page_size *= 0x400;
317
/* L2P pages are in entries - not in (k)Bytes */
320
SVN_ERR(svn_config_get_bool(config, &ffd->pack_after_commit,
321
CONFIG_SECTION_DEBUG,
322
CONFIG_OPTION_PACK_AFTER_COMMIT,
325
/* memcached configuration */
326
SVN_ERR(svn_cache__make_memcache_from_config(&ffd->memcache, config,
327
result_pool, scratch_pool));
329
SVN_ERR(svn_config_get_bool(config, &ffd->fail_stop,
330
CONFIG_SECTION_CACHES, CONFIG_OPTION_FAIL_STOP,
336
/* Write FS' initial configuration file.
337
* Use SCRATCH_POOL for temporary allocations. */
339
write_config(svn_fs_t *fs,
340
apr_pool_t *scratch_pool)
342
#define NL APR_EOL_STR
343
static const char * const fsx_conf_contents =
344
"### This file controls the configuration of the FSX filesystem." NL
346
"[" SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS "]" NL
347
"### These options name memcached servers used to cache internal FSX" NL
348
"### data. See http://www.danga.com/memcached/ for more information on" NL
349
"### memcached. To use memcached with FSX, run one or more memcached" NL
350
"### servers, and specify each of them as an option like so:" NL
351
"# first-server = 127.0.0.1:11211" NL
352
"# remote-memcached = mymemcached.corp.example.com:11212" NL
353
"### The option name is ignored; the value is of the form HOST:PORT." NL
354
"### memcached servers can be shared between multiple repositories;" NL
355
"### however, if you do this, you *must* ensure that repositories have" NL
356
"### distinct UUIDs and paths, or else cached data from one repository" NL
357
"### might be used by another accidentally. Note also that memcached has" NL
358
"### no authentication for reads or writes, so you must ensure that your" NL
359
"### memcached servers are only accessible by trusted users." NL
361
"[" CONFIG_SECTION_CACHES "]" NL
362
"### When a cache-related error occurs, normally Subversion ignores it" NL
363
"### and continues, logging an error if the server is appropriately" NL
364
"### configured (and ignoring it with file:// access). To make" NL
365
"### Subversion never ignore cache errors, uncomment this line." NL
366
"# " CONFIG_OPTION_FAIL_STOP " = true" NL
368
"[" CONFIG_SECTION_REP_SHARING "]" NL
369
"### To conserve space, the filesystem can optionally avoid storing" NL
370
"### duplicate representations. This comes at a slight cost in" NL
371
"### performance, as maintaining a database of shared representations can" NL
372
"### increase commit times. The space savings are dependent upon the size" NL
373
"### of the repository, the number of objects it contains and the amount of" NL
374
"### duplication between them, usually a function of the branching and" NL
375
"### merging process." NL
377
"### The following parameter enables rep-sharing in the repository. It can" NL
378
"### be switched on and off at will, but for best space-saving results" NL
379
"### should be enabled consistently over the life of the repository." NL
380
"### 'svnadmin verify' will check the rep-cache regardless of this setting." NL
381
"### rep-sharing is enabled by default." NL
382
"# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL
384
"[" CONFIG_SECTION_DELTIFICATION "]" NL
385
"### To conserve space, the filesystem stores data as differences against" NL
386
"### existing representations. This comes at a slight cost in performance," NL
387
"### as calculating differences can increase commit times. Reading data" NL
388
"### will also create higher CPU load and the data will be fragmented." NL
389
"### Since deltification tends to save significant amounts of disk space," NL
390
"### the overall I/O load can actually be lower." NL
392
"### The options in this section allow for tuning the deltification" NL
393
"### strategy. Their effects on data size and server performance may vary" NL
394
"### from one repository to another." NL
396
"### During commit, the server may need to walk the whole change history of" NL
397
"### of a given node to find a suitable deltification base. This linear" NL
398
"### process can impact commit times, svnadmin load and similar operations." NL
399
"### This setting limits the depth of the deltification history. If the" NL
400
"### threshold has been reached, the node will be stored as fulltext and a" NL
401
"### new deltification history begins." NL
402
"### Note, this is unrelated to svn log." NL
403
"### Very large values rarely provide significant additional savings but" NL
404
"### can impact performance greatly - in particular if directory" NL
405
"### deltification has been activated. Very small values may be useful in" NL
406
"### repositories that are dominated by large, changing binaries." NL
407
"### Should be a power of two minus 1. A value of 0 will effectively" NL
408
"### disable deltification." NL
409
"### For 1.9, the default value is 1023." NL
410
"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL
412
"### The skip-delta scheme used by FSX tends to repeatably store redundant" NL
413
"### delta information where a simple delta against the latest version is" NL
414
"### often smaller. By default, 1.9+ will therefore use skip deltas only" NL
415
"### after the linear chain of deltas has grown beyond the threshold" NL
416
"### specified by this setting." NL
417
"### Values up to 64 can result in some reduction in repository size for" NL
418
"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL
419
"### numbers can reduce those costs at the cost of more disk space. For" NL
420
"### rarely read repositories or those containing larger binaries, this may" NL
421
"### present a better trade-off." NL
422
"### Should be a power of two. A value of 1 or smaller will cause the" NL
423
"### exclusive use of skip-deltas." NL
424
"### For 1.8, the default value is 16." NL
425
"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL
427
"### After deltification, we compress the data through zlib to minimize on-" NL
428
"### disk size. That can be an expensive and ineffective process. This" NL
429
"### setting controls the usage of zlib in future revisions." NL
430
"### Revisions with highly compressible data in them may shrink in size" NL
431
"### if the setting is increased but may take much longer to commit. The" NL
432
"### time taken to uncompress that data again is widely independent of the" NL
433
"### compression level." NL
434
"### Compression will be ineffective if the incoming content is already" NL
435
"### highly compressed. In that case, disabling the compression entirely" NL
436
"### will speed up commits as well as reading the data. Repositories with" NL
437
"### many small compressible files (source code) but also a high percentage" NL
438
"### of large incompressible ones (artwork) may benefit from compression" NL
439
"### levels lowered to e.g. 1." NL
440
"### Valid values are 0 to 9 with 9 providing the highest compression ratio" NL
441
"### and 0 disabling it altogether." NL
442
"### The default value is 5." NL
443
"# " CONFIG_OPTION_COMPRESSION_LEVEL " = 5" NL
445
"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL
446
"### This parameter controls the size (in kBytes) of packed revprop files." NL
447
"### Revprops of consecutive revisions will be concatenated into a single" NL
448
"### file up to but not exceeding the threshold given here. However, each" NL
449
"### pack file may be much smaller and revprops of a single revision may be" NL
450
"### much larger than the limit set here. The threshold will be applied" NL
451
"### before optional compression takes place." NL
452
"### Large values will reduce disk space usage at the expense of increased" NL
453
"### latency and CPU usage reading and changing individual revprops. They" NL
454
"### become an advantage when revprop caching has been enabled because a" NL
455
"### lot of data can be read in one go. Values smaller than 4 kByte will" NL
456
"### not improve latency any further and quickly render revprop packing" NL
457
"### ineffective." NL
458
"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL
459
"### pack files and 256 kBytes when compression has been enabled." NL
460
"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL
462
"### To save disk space, packed revprop files may be compressed. Standard" NL
463
"### revprops tend to allow for very effective compression. Reading and" NL
464
"### even more so writing, become significantly more CPU intensive. With" NL
465
"### revprop caching enabled, the overhead can be offset by reduced I/O" NL
466
"### unless you often modify revprops after packing." NL
467
"### Compressing packed revprops is enabled by default." NL
468
"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = true" NL
470
"[" CONFIG_SECTION_IO "]" NL
471
"### Parameters in this section control the data access granularity in" NL
472
"### format 7 repositories and later. The defaults should translate into" NL
473
"### decent performance over a wide range of setups." NL
475
"### When a specific piece of information needs to be read from disk, a" NL
476
"### data block is being read at once and its contents are being cached." NL
477
"### If the repository is being stored on a RAID, the block size should be" NL
478
"### either 50% or 100% of RAID block size / granularity. Also, your file" NL
479
"### system blocks/clusters should be properly aligned and sized. In that" NL
480
"### setup, each access will hit only one disk (minimizes I/O load) but" NL
481
"### uses all the data provided by the disk in a single access." NL
482
"### For SSD-based storage systems, slightly lower values around 16 kB" NL
483
"### may improve latency while still maximizing throughput." NL
484
"### Can be changed at any time but must be a power of 2." NL
485
"### block-size is given in kBytes and with a default of 64 kBytes." NL
486
"# " CONFIG_OPTION_BLOCK_SIZE " = 64" NL
488
"### The log-to-phys index maps data item numbers to offsets within the" NL
489
"### rev or pack file. This index is organized in pages of a fixed maximum" NL
490
"### capacity. To access an item, the page table and the respective page" NL
491
"### must be read." NL
492
"### This parameter only affects revisions with thousands of changed paths." NL
493
"### If you have several extremely large revisions (~1 mio changes), think" NL
494
"### about increasing this setting. Reducing the value will rarely result" NL
495
"### in a net speedup." NL
496
"### This is an expert setting. Must be a power of 2." NL
497
"### l2p-page-size is 8192 entries by default." NL
498
"# " CONFIG_OPTION_L2P_PAGE_SIZE " = 8192" NL
500
"### The phys-to-log index maps positions within the rev or pack file to" NL
501
"### to data items, i.e. describes what piece of information is being" NL
502
"### stored at any particular offset. The index describes the rev file" NL
503
"### in chunks (pages) and keeps a global list of all those pages. Large" NL
504
"### pages mean a shorter page table but a larger per-page description of" NL
505
"### data items in it. The latency sweet spot depends on the change size" NL
506
"### distribution but covers a relatively wide range." NL
507
"### If the repository contains very large files, i.e. individual changes" NL
508
"### of tens of MB each, increasing the page size will shorten the index" NL
509
"### file at the expense of a slightly increased latency in sections with" NL
510
"### smaller changes." NL
511
"### For source code repositories, this should be about 16x the block-size." NL
512
"### Must be a power of 2." NL
513
"### p2l-page-size is given in kBytes and with a default of 1024 kBytes." NL
514
"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024" NL
517
return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG,
519
fsx_conf_contents, scratch_pool);
522
/* Read FS's UUID file and store the data in the FS struct. */
524
read_uuid(svn_fs_t *fs,
525
apr_pool_t *scratch_pool)
527
svn_fs_x__data_t *ffd = fs->fsap_data;
528
apr_file_t *uuid_file;
529
char buf[APR_UUID_FORMATTED_LENGTH + 2];
532
/* Read the repository uuid. */
533
SVN_ERR(svn_io_file_open(&uuid_file, svn_fs_x__path_uuid(fs, scratch_pool),
534
APR_READ | APR_BUFFERED, APR_OS_DEFAULT,
538
SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, scratch_pool));
539
fs->uuid = apr_pstrdup(fs->pool, buf);
541
/* Read the instance ID. */
543
SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit,
545
ffd->instance_id = apr_pstrdup(fs->pool, buf);
547
SVN_ERR(svn_io_file_close(uuid_file, scratch_pool));
553
svn_fs_x__read_format_file(svn_fs_t *fs,
554
apr_pool_t *scratch_pool)
556
svn_fs_x__data_t *ffd = fs->fsap_data;
557
int format, max_files_per_dir;
559
/* Read info from format file. */
560
SVN_ERR(read_format(&format, &max_files_per_dir,
561
svn_fs_x__path_format(fs, scratch_pool), scratch_pool));
563
/* Now that we've got *all* info, store / update values in FFD. */
564
ffd->format = format;
565
ffd->max_files_per_dir = max_files_per_dir;
571
svn_fs_x__open(svn_fs_t *fs,
573
apr_pool_t *scratch_pool)
575
svn_fs_x__data_t *ffd = fs->fsap_data;
576
fs->path = apr_pstrdup(fs->pool, path);
578
/* Read the FS format file. */
579
SVN_ERR(svn_fs_x__read_format_file(fs, scratch_pool));
581
/* Read in and cache the repository uuid. */
582
SVN_ERR(read_uuid(fs, scratch_pool));
584
/* Read the min unpacked revision. */
585
SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs, scratch_pool));
587
/* Read the configuration file. */
588
SVN_ERR(read_config(ffd, fs->path, fs->pool, scratch_pool));
590
return svn_error_trace(svn_fs_x__read_current(&ffd->youngest_rev_cache,
594
/* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying
595
* parameters over between them. */
596
typedef struct upgrade_baton_t
599
svn_fs_upgrade_notify_t notify_func;
601
svn_cancel_func_t cancel_func;
605
/* Upgrade the FS given in upgrade_baton_t *)BATON to the latest format
606
* version. Apply options an invoke callback from that BATON.
607
* Temporary allocations are to be made from SCRATCH_POOL.
609
* At the moment, this is a simple placeholder as we don't support upgrades
610
* from experimental FSX versions.
613
upgrade_body(void *baton,
614
apr_pool_t *scratch_pool)
616
upgrade_baton_t *upgrade_baton = baton;
617
svn_fs_t *fs = upgrade_baton->fs;
618
int format, max_files_per_dir;
619
const char *format_path = svn_fs_x__path_format(fs, scratch_pool);
621
/* Read the FS format number and max-files-per-dir setting. */
622
SVN_ERR(read_format(&format, &max_files_per_dir, format_path,
625
/* If we're already up-to-date, there's nothing else to be done here. */
626
if (format == SVN_FS_X__FORMAT_NUMBER)
635
svn_fs_x__upgrade(svn_fs_t *fs,
636
svn_fs_upgrade_notify_t notify_func,
638
svn_cancel_func_t cancel_func,
640
apr_pool_t *scratch_pool)
642
upgrade_baton_t baton;
644
baton.notify_func = notify_func;
645
baton.notify_baton = notify_baton;
646
baton.cancel_func = cancel_func;
647
baton.cancel_baton = cancel_baton;
649
return svn_fs_x__with_all_locks(fs, upgrade_body, (void *)&baton,
655
svn_fs_x__youngest_rev(svn_revnum_t *youngest_p,
657
apr_pool_t *scratch_pool)
659
svn_fs_x__data_t *ffd = fs->fsap_data;
660
SVN_ERR(svn_fs_x__read_current(youngest_p, fs, scratch_pool));
661
ffd->youngest_rev_cache = *youngest_p;
667
svn_fs_x__ensure_revision_exists(svn_revnum_t rev,
669
apr_pool_t *scratch_pool)
671
svn_fs_x__data_t *ffd = fs->fsap_data;
673
if (! SVN_IS_VALID_REVNUM(rev))
674
return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
675
_("Invalid revision number '%ld'"), rev);
678
/* Did the revision exist the last time we checked the current
680
if (rev <= ffd->youngest_rev_cache)
683
SVN_ERR(svn_fs_x__read_current(&ffd->youngest_rev_cache, fs, scratch_pool));
686
if (rev <= ffd->youngest_rev_cache)
689
return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL,
690
_("No such revision %ld"), rev);
695
svn_fs_x__file_length(svn_filesize_t *length,
696
svn_fs_x__noderev_t *noderev)
698
if (noderev->data_rep)
699
*length = noderev->data_rep->expanded_size;
707
svn_fs_x__file_text_rep_equal(svn_fs_x__representation_t *a,
708
svn_fs_x__representation_t *b)
710
svn_boolean_t a_empty = a == NULL || a->expanded_size == 0;
711
svn_boolean_t b_empty = b == NULL || b->expanded_size == 0;
713
/* This makes sure that neither rep will be NULL later on */
714
if (a_empty && b_empty)
717
if (a_empty != b_empty)
720
/* Same physical representation? Note that these IDs are always up-to-date
721
instead of e.g. being set lazily. */
722
if (svn_fs_x__id_eq(&a->id, &b->id))
725
/* Contents are equal if the checksums match. These are also always known.
727
return memcmp(a->md5_digest, b->md5_digest, sizeof(a->md5_digest)) == 0
728
&& memcmp(a->sha1_digest, b->sha1_digest, sizeof(a->sha1_digest)) == 0;
732
svn_fs_x__prop_rep_equal(svn_boolean_t *equal,
734
svn_fs_x__noderev_t *a,
735
svn_fs_x__noderev_t *b,
736
svn_boolean_t strict,
737
apr_pool_t *scratch_pool)
739
svn_fs_x__representation_t *rep_a = a->prop_rep;
740
svn_fs_x__representation_t *rep_b = b->prop_rep;
741
apr_hash_t *proplist_a;
742
apr_hash_t *proplist_b;
744
/* Mainly for a==b==NULL */
751
/* Committed property lists can be compared quickly */
753
&& svn_fs_x__is_revision(rep_a->id.change_set)
754
&& svn_fs_x__is_revision(rep_b->id.change_set))
756
/* MD5 must be given. Having the same checksum is good enough for
757
accepting the prop lists as equal. */
758
*equal = memcmp(rep_a->md5_digest, rep_b->md5_digest,
759
sizeof(rep_a->md5_digest)) == 0;
763
/* Same path in same txn? */
764
if (svn_fs_x__id_eq(&a->noderev_id, &b->noderev_id))
770
/* Skip the expensive bits unless we are in strict mode.
771
Simply assume that there is a different. */
778
/* At least one of the reps has been modified in a txn.
779
Fetch and compare them. */
780
SVN_ERR(svn_fs_x__get_proplist(&proplist_a, fs, a, scratch_pool,
782
SVN_ERR(svn_fs_x__get_proplist(&proplist_b, fs, b, scratch_pool,
785
*equal = svn_fs__prop_lists_equal(proplist_a, proplist_b, scratch_pool);
791
svn_fs_x__file_checksum(svn_checksum_t **checksum,
792
svn_fs_x__noderev_t *noderev,
793
svn_checksum_kind_t kind,
794
apr_pool_t *result_pool)
798
if (noderev->data_rep)
805
case svn_checksum_md5:
806
temp.digest = noderev->data_rep->md5_digest;
809
case svn_checksum_sha1:
810
if (! noderev->data_rep->has_sha1)
813
temp.digest = noderev->data_rep->sha1_digest;
820
*checksum = svn_checksum_dup(&temp, result_pool);
826
svn_fs_x__representation_t *
827
svn_fs_x__rep_copy(svn_fs_x__representation_t *rep,
828
apr_pool_t *result_pool)
833
return apr_pmemdup(result_pool, rep, sizeof(*rep));
837
/* Write out the zeroth revision for filesystem FS.
838
Perform temporary allocations in SCRATCH_POOL. */
840
write_revision_zero(svn_fs_t *fs,
841
apr_pool_t *scratch_pool)
843
/* Use an explicit sub-pool to have full control over temp file lifetimes.
844
* Since we have it, use it for everything else as well. */
845
apr_pool_t *subpool = svn_pool_create(scratch_pool);
846
const char *path_revision_zero = svn_fs_x__path_rev(fs, 0, subpool);
847
apr_hash_t *proplist;
850
apr_array_header_t *index_entries;
851
svn_fs_x__p2l_entry_t *entry;
852
svn_fs_x__revision_file_t *rev_file;
853
const char *l2p_proto_index, *p2l_proto_index;
855
/* Construct a skeleton r0 with no indexes. */
856
svn_string_t *noderev_str = svn_string_create("id: 2+0\n"
864
svn_string_t *changes_str = svn_string_create("\n",
866
svn_string_t *r0 = svn_string_createf(subpool, "%s%s",
870
/* Write skeleton r0 to disk. */
871
SVN_ERR(svn_io_file_create(path_revision_zero, r0->data, subpool));
873
/* Construct the index P2L contents: describe the 2 items we have.
874
Be sure to create them in on-disk order. */
875
index_entries = apr_array_make(subpool, 2, sizeof(entry));
877
entry = apr_pcalloc(subpool, sizeof(*entry));
879
entry->size = (apr_off_t)noderev_str->len;
880
entry->type = SVN_FS_X__ITEM_TYPE_NODEREV;
881
entry->item_count = 1;
882
entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
883
entry->items[0].change_set = 0;
884
entry->items[0].number = SVN_FS_X__ITEM_INDEX_ROOT_NODE;
885
APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
887
entry = apr_pcalloc(subpool, sizeof(*entry));
888
entry->offset = (apr_off_t)noderev_str->len;
889
entry->size = (apr_off_t)changes_str->len;
890
entry->type = SVN_FS_X__ITEM_TYPE_CHANGES;
891
entry->item_count = 1;
892
entry->items = apr_pcalloc(subpool, sizeof(*entry->items));
893
entry->items[0].change_set = 0;
894
entry->items[0].number = SVN_FS_X__ITEM_INDEX_CHANGES;
895
APR_ARRAY_PUSH(index_entries, svn_fs_x__p2l_entry_t *) = entry;
897
/* Now re-open r0, create proto-index files from our entries and
898
rewrite the index section of r0. */
899
SVN_ERR(svn_fs_x__open_pack_or_rev_file_writable(&rev_file, fs, 0,
901
SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index, fs,
902
rev_file, index_entries,
904
SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index, fs,
907
SVN_ERR(svn_fs_x__add_index_data(fs, rev_file->file, l2p_proto_index,
908
p2l_proto_index, 0, subpool));
909
SVN_ERR(svn_fs_x__close_revision_file(rev_file));
911
SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool));
913
/* Set a date on revision 0. */
914
date.data = svn_time_to_cstring(apr_time_now(), fs->pool);
915
date.len = strlen(date.data);
916
proplist = apr_hash_make(fs->pool);
917
svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date);
918
return svn_fs_x__set_revision_proplist(fs, 0, proplist, fs->pool);
922
svn_fs_x__create_file_tree(svn_fs_t *fs,
926
apr_pool_t *scratch_pool)
928
svn_fs_x__data_t *ffd = fs->fsap_data;
930
fs->path = apr_pstrdup(fs->pool, path);
931
ffd->format = format;
933
/* Use an appropriate sharding mode if supported by the format. */
934
ffd->max_files_per_dir = shard_size;
936
/* Create the revision data directories. */
937
SVN_ERR(svn_io_make_dir_recursively(
938
svn_fs_x__path_rev_shard(fs, 0, scratch_pool),
941
/* Create the revprops directory. */
942
SVN_ERR(svn_io_make_dir_recursively(
943
svn_fs_x__path_revprops_shard(fs, 0, scratch_pool),
946
/* Create the transaction directory. */
947
SVN_ERR(svn_io_make_dir_recursively(
948
svn_fs_x__path_txns_dir(fs, scratch_pool),
951
/* Create the protorevs directory. */
952
SVN_ERR(svn_io_make_dir_recursively(
953
svn_fs_x__path_txn_proto_revs(fs, scratch_pool),
956
/* Create the 'current' file. */
957
SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_current(fs, scratch_pool),
959
SVN_ERR(svn_fs_x__write_current(fs, 0, scratch_pool));
961
/* Create the 'uuid' file. */
962
SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs, scratch_pool),
964
SVN_ERR(svn_fs_x__set_uuid(fs, NULL, NULL, scratch_pool));
966
/* Create the fsfs.conf file. */
967
SVN_ERR(write_config(fs, scratch_pool));
968
SVN_ERR(read_config(ffd, fs->path, fs->pool, scratch_pool));
970
/* Add revision 0. */
971
SVN_ERR(write_revision_zero(fs, scratch_pool));
973
/* Create the min unpacked rev file. */
974
SVN_ERR(svn_io_file_create(
975
svn_fs_x__path_min_unpacked_rev(fs, scratch_pool),
976
"0\n", scratch_pool));
978
/* Create the txn-current file if the repository supports
979
the transaction sequence file. */
980
SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(fs, scratch_pool),
981
"0\n", scratch_pool));
982
SVN_ERR(svn_io_file_create_empty(
983
svn_fs_x__path_txn_current_lock(fs, scratch_pool),
986
/* Initialize the revprop caching info. */
987
SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, scratch_pool));
989
ffd->youngest_rev_cache = 0;
994
svn_fs_x__create(svn_fs_t *fs,
996
apr_pool_t *scratch_pool)
998
int format = SVN_FS_X__FORMAT_NUMBER;
999
svn_fs_x__data_t *ffd = fs->fsap_data;
1001
fs->path = apr_pstrdup(fs->pool, path);
1002
/* See if compatibility with older versions was explicitly requested. */
1005
svn_version_t *compatible_version;
1006
SVN_ERR(svn_fs__compatible_version(&compatible_version, fs->config,
1009
/* select format number */
1010
switch(compatible_version->minor)
1020
case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL,
1021
_("FSX is not compatible with Subversion prior to 1.9"));
1023
default:format = SVN_FS_X__FORMAT_NUMBER;
1027
/* Actual FS creation. */
1028
SVN_ERR(svn_fs_x__create_file_tree(fs, path, format,
1029
SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR,
1032
/* This filesystem is ready. Stamp it with a format number. */
1033
SVN_ERR(svn_fs_x__write_format(fs, FALSE, scratch_pool));
1035
ffd->youngest_rev_cache = 0;
1036
return SVN_NO_ERROR;
1040
svn_fs_x__set_uuid(svn_fs_t *fs,
1042
const char *instance_id,
1043
apr_pool_t *scratch_pool)
1045
svn_fs_x__data_t *ffd = fs->fsap_data;
1046
const char *uuid_path = svn_fs_x__path_uuid(fs, scratch_pool);
1047
svn_stringbuf_t *contents = svn_stringbuf_create_empty(scratch_pool);
1050
uuid = svn_uuid_generate(scratch_pool);
1053
instance_id = svn_uuid_generate(scratch_pool);
1055
svn_stringbuf_appendcstr(contents, uuid);
1056
svn_stringbuf_appendcstr(contents, "\n");
1057
svn_stringbuf_appendcstr(contents, instance_id);
1058
svn_stringbuf_appendcstr(contents, "\n");
1060
/* We use the permissions of the 'current' file, because the 'uuid'
1061
file does not exist during repository creation. */
1062
SVN_ERR(svn_io_write_atomic(uuid_path, contents->data, contents->len,
1064
svn_fs_x__path_current(fs, scratch_pool),
1067
fs->uuid = apr_pstrdup(fs->pool, uuid);
1068
ffd->instance_id = apr_pstrdup(fs->pool, instance_id);
1070
return SVN_NO_ERROR;
1073
/** Node origin lazy cache. */
1075
/* If directory PATH does not exist, create it and give it the same
1076
permissions as FS_path.*/
1078
svn_fs_x__ensure_dir_exists(const char *path,
1079
const char *fs_path,
1080
apr_pool_t *scratch_pool)
1082
svn_error_t *err = svn_io_dir_make(path, APR_OS_DEFAULT, scratch_pool);
1083
if (err && APR_STATUS_IS_EEXIST(err->apr_err))
1085
svn_error_clear(err);
1086
return SVN_NO_ERROR;
1090
/* We successfully created a new directory. Dup the permissions
1092
return svn_io_copy_perms(fs_path, path, scratch_pool);
1099
svn_fs_x__revision_prop(svn_string_t **value_p,
1102
const char *propname,
1103
apr_pool_t *result_pool,
1104
apr_pool_t *scratch_pool)
1108
SVN_ERR(svn_fs__check_fs(fs, TRUE));
1109
SVN_ERR(svn_fs_x__get_revision_proplist(&table, fs, rev, FALSE,
1110
scratch_pool, scratch_pool));
1112
*value_p = svn_string_dup(svn_hash_gets(table, propname), result_pool);
1114
return SVN_NO_ERROR;
1118
/* Baton used for change_rev_prop_body below. */
1119
typedef struct change_rev_prop_baton_t {
1123
const svn_string_t *const *old_value_p;
1124
const svn_string_t *value;
1125
} change_rev_prop_baton_t;
1127
/* The work-horse for svn_fs_x__change_rev_prop, called with the FS
1128
write lock. This implements the svn_fs_x__with_write_lock()
1129
'body' callback type. BATON is a 'change_rev_prop_baton_t *'. */
1130
static svn_error_t *
1131
change_rev_prop_body(void *baton,
1132
apr_pool_t *scratch_pool)
1134
change_rev_prop_baton_t *cb = baton;
1137
/* Read current revprop values from disk (never from cache).
1138
Even if somehow the cache got out of sync, we want to make sure that
1139
we read, update and write up-to-date data. */
1140
SVN_ERR(svn_fs_x__get_revision_proplist(&table, cb->fs, cb->rev, TRUE,
1141
scratch_pool, scratch_pool));
1143
if (cb->old_value_p)
1145
const svn_string_t *wanted_value = *cb->old_value_p;
1146
const svn_string_t *present_value = svn_hash_gets(table, cb->name);
1147
if ((!wanted_value != !present_value)
1148
|| (wanted_value && present_value
1149
&& !svn_string_compare(wanted_value, present_value)))
1151
/* What we expected isn't what we found. */
1152
return svn_error_createf(SVN_ERR_FS_PROP_BASEVALUE_MISMATCH, NULL,
1153
_("revprop '%s' has unexpected value in "
1159
svn_hash_sets(table, cb->name, cb->value);
1161
return svn_fs_x__set_revision_proplist(cb->fs, cb->rev, table,
1166
svn_fs_x__change_rev_prop(svn_fs_t *fs,
1169
const svn_string_t *const *old_value_p,
1170
const svn_string_t *value,
1171
apr_pool_t *scratch_pool)
1173
change_rev_prop_baton_t cb;
1175
SVN_ERR(svn_fs__check_fs(fs, TRUE));
1180
cb.old_value_p = old_value_p;
1183
return svn_fs_x__with_write_lock(fs, change_rev_prop_body, &cb,
1189
svn_fs_x__info_format(int *fs_format,
1190
svn_version_t **supports_version,
1192
apr_pool_t *result_pool,
1193
apr_pool_t *scratch_pool)
1195
svn_fs_x__data_t *ffd = fs->fsap_data;
1196
*fs_format = ffd->format;
1197
*supports_version = apr_palloc(result_pool, sizeof(svn_version_t));
1199
(*supports_version)->major = SVN_VER_MAJOR;
1200
(*supports_version)->minor = 9;
1201
(*supports_version)->patch = 0;
1202
(*supports_version)->tag = "";
1204
switch (ffd->format)
1209
# if SVN_FS_X__FORMAT_NUMBER != 1
1210
# error "Need to add a 'case' statement here"
1215
return SVN_NO_ERROR;
1219
svn_fs_x__info_config_files(apr_array_header_t **files,
1221
apr_pool_t *result_pool,
1222
apr_pool_t *scratch_pool)
1224
*files = apr_array_make(result_pool, 1, sizeof(const char *));
1225
APR_ARRAY_PUSH(*files, const char *) = svn_dirent_join(fs->path, PATH_CONFIG,
1227
return SVN_NO_ERROR;
added
removed
subversion/libsvn_fs_x/fs_x.c
