1
/* caching.c : in-memory caching
3
* ====================================================================
4
* Licensed to the Apache Software Foundation (ASF) under one
5
* or more contributor license agreements. See the NOTICE file
6
* distributed with this work for additional information
7
* regarding copyright ownership. The ASF licenses this file
8
* to you under the Apache License, Version 2.0 (the
9
* "License"); you may not use this file except in compliance
10
* with the License. You may obtain a copy of the License at
12
* http://www.apache.org/licenses/LICENSE-2.0
14
* Unless required by applicable law or agreed to in writing,
15
* software distributed under the License is distributed on an
16
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17
* KIND, either express or implied. See the License for the
18
* specific language governing permissions and limitations
20
* ====================================================================
31
#include "temp_serializer.h"
33
#include "../libsvn_fs/fs-loader.h"
35
#include "svn_config.h"
36
#include "svn_cache_config.h"
38
#include "svn_private_config.h"
40
#include "svn_pools.h"
42
#include "private/svn_debug.h"
43
#include "private/svn_subr_private.h"
45
/* Take the ORIGINAL string and replace all occurrences of ":" without
46
* limiting the key space. Allocate the result in RESULT_POOL.
49
normalize_key_part(const char *original,
50
apr_pool_t *result_pool)
53
apr_size_t len = strlen(original);
54
svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len,
57
for (i = 0; i < len; ++i)
62
case ':': svn_stringbuf_appendbytes(normalized, "%_", 2);
64
case '%': svn_stringbuf_appendbytes(normalized, "%%", 2);
66
default : svn_stringbuf_appendbyte(normalized, c);
70
return normalized->data;
73
/* *CACHE_TXDELTAS, *CACHE_FULLTEXTS and *CACHE_REVPROPS flags will be set
74
according to FS->CONFIG. *CACHE_NAMESPACE receives the cache prefix
77
Allocate CACHE_NAMESPACE in RESULT_POOL. */
79
read_config(const char **cache_namespace,
80
svn_boolean_t *cache_txdeltas,
81
svn_boolean_t *cache_fulltexts,
82
svn_boolean_t *cache_revprops,
84
apr_pool_t *result_pool)
86
/* No cache namespace by default. I.e. all FS instances share the
87
* cached data. If you specify different namespaces, the data will
88
* share / compete for the same cache memory but keys will not match
89
* across namespaces and, thus, cached data will not be shared between
92
* Since the namespace will be concatenated with other elements to form
93
* the complete key prefix, we must make sure that the resulting string
94
* is unique and cannot be created by any other combination of elements.
97
= normalize_key_part(svn_hash__get_cstring(fs->config,
98
SVN_FS_CONFIG_FSFS_CACHE_NS,
102
/* don't cache text deltas by default.
103
* Once we reconstructed the fulltexts from the deltas,
104
* these deltas are rarely re-used. Therefore, only tools
105
* like svnadmin will activate this to speed up operations
109
= svn_hash__get_bool(fs->config,
110
SVN_FS_CONFIG_FSFS_CACHE_DELTAS,
113
/* by default, cache fulltexts.
114
* Most SVN tools care about reconstructed file content.
115
* Thus, this is a reasonable default.
116
* SVN admin tools may set that to FALSE because fulltexts
117
* won't be re-used rendering the cache less effective
118
* by squeezing wanted data out.
121
= svn_hash__get_bool(fs->config,
122
SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS,
125
/* don't cache revprops by default.
126
* Revprop caching significantly speeds up operations like
127
* svn ls -v. However, it requires synchronization that may
128
* not be available or efficient in the current server setup.
129
* Option "2" is equivalent to "1".
131
if (strcmp(svn_hash__get_cstring(fs->config,
132
SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
135
= svn_hash__get_bool(fs->config,
136
SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
139
*cache_revprops = TRUE;
145
/* Implements svn_cache__error_handler_t
146
* This variant clears the error after logging it.
149
warn_and_continue_on_cache_errors(svn_error_t *err,
153
svn_fs_t *fs = baton;
154
(fs->warning)(fs->warning_baton, err);
155
svn_error_clear(err);
160
/* Implements svn_cache__error_handler_t
161
* This variant logs the error and passes it on to the callers.
164
warn_and_fail_on_cache_errors(svn_error_t *err,
168
svn_fs_t *fs = baton;
169
(fs->warning)(fs->warning_baton, err);
173
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
174
/* Baton to be used for the dump_cache_statistics() pool cleanup function, */
175
typedef struct dump_cache_baton_t
177
/* the pool about to be cleaned up. Will be used for temp. allocations. */
180
/* the cache to dump the statistics for */
182
} dump_cache_baton_t;
184
/* APR pool cleanup handler that will printf the statistics of the
185
cache referenced by the baton in BATON_VOID. */
187
dump_cache_statistics(void *baton_void)
189
dump_cache_baton_t *baton = baton_void;
191
apr_status_t result = APR_SUCCESS;
192
svn_cache__info_t info;
193
svn_string_t *text_stats;
194
apr_array_header_t *lines;
197
svn_error_t *err = svn_cache__get_info(baton->cache,
202
/* skip unused caches */
203
if (! err && (info.gets > 0 || info.sets > 0))
205
text_stats = svn_cache__format_info(&info, TRUE, baton->pool);
206
lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool);
208
for (i = 0; i < lines->nelts; ++i)
210
const char *line = APR_ARRAY_IDX(lines, i, const char *);
212
SVN_DBG(("%s\n", line));
217
/* process error returns */
220
result = err->apr_err;
221
svn_error_clear(err);
228
dump_global_cache_statistics(void *baton_void)
230
apr_pool_t *pool = baton_void;
232
svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool);
233
svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool);
234
apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n",
238
for (i = 0; i < lines->nelts; ++i)
240
const char *line = APR_ARRAY_IDX(lines, i, const char *);
242
SVN_DBG(("%s\n", line));
249
#endif /* SVN_DEBUG_CACHE_DUMP_STATS */
251
/* This function sets / registers the required callbacks for a given
252
* not transaction-specific CACHE object in FS, if CACHE is not NULL.
254
* All these svn_cache__t instances shall be handled uniformly. Unless
255
* ERROR_HANDLER is NULL, register it for the given CACHE in FS.
258
init_callbacks(svn_cache__t *cache,
260
svn_cache__error_handler_t error_handler,
265
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
267
/* schedule printing the access statistics upon pool cleanup,
268
* i.e. end of FSX session.
270
dump_cache_baton_t *baton;
272
baton = apr_palloc(pool, sizeof(*baton));
274
baton->cache = cache;
276
apr_pool_cleanup_register(pool,
278
dump_cache_statistics,
279
apr_pool_cleanup_null);
283
SVN_ERR(svn_cache__set_error_handler(cache,
293
/* Sets *CACHE_P to cache instance based on provided options.
294
* Creates memcache if MEMCACHE is not NULL. Creates membuffer cache if
295
* MEMBUFFER is not NULL. Fallbacks to inprocess cache if MEMCACHE and
296
* MEMBUFFER are NULL and pages is non-zero. Sets *CACHE_P to NULL
297
* otherwise. Use the given PRIORITY class for the new cache. If it
298
* is 0, then use the default priority class.
300
* Unless NO_HANDLER is true, register an error handler that reports errors
301
* as warnings to the FS warning callback.
303
* Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL.
306
create_cache(svn_cache__t **cache_p,
307
svn_memcache_t *memcache,
308
svn_membuffer_t *membuffer,
310
apr_int64_t items_per_page,
311
svn_cache__serialize_func_t serializer,
312
svn_cache__deserialize_func_t deserializer,
315
apr_uint32_t priority,
317
svn_boolean_t no_handler,
318
apr_pool_t *result_pool,
319
apr_pool_t *scratch_pool)
321
svn_cache__error_handler_t error_handler = no_handler
323
: warn_and_fail_on_cache_errors;
325
priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY;
329
SVN_ERR(svn_cache__create_memcache(cache_p, memcache,
330
serializer, deserializer, klen,
331
prefix, result_pool));
332
error_handler = no_handler
334
: warn_and_continue_on_cache_errors;
338
SVN_ERR(svn_cache__create_membuffer_cache(
339
cache_p, membuffer, serializer, deserializer,
340
klen, prefix, priority, FALSE, result_pool, scratch_pool));
344
SVN_ERR(svn_cache__create_inprocess(
345
cache_p, serializer, deserializer, klen, pages,
346
items_per_page, FALSE, prefix, result_pool));
353
SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool));
359
svn_fs_x__initialize_caches(svn_fs_t *fs,
360
apr_pool_t *scratch_pool)
362
svn_fs_x__data_t *ffd = fs->fsap_data;
363
const char *prefix = apr_pstrcat(scratch_pool,
365
"/", normalize_key_part(fs->path,
369
svn_membuffer_t *membuffer;
370
svn_boolean_t no_handler = ffd->fail_stop;
371
svn_boolean_t cache_txdeltas;
372
svn_boolean_t cache_fulltexts;
373
svn_boolean_t cache_revprops;
374
const char *cache_namespace;
376
/* Evaluating the cache configuration. */
377
SVN_ERR(read_config(&cache_namespace,
384
prefix = apr_pstrcat(scratch_pool, "ns:", cache_namespace, ":", prefix,
387
membuffer = svn_cache__get_global_membuffer_cache();
389
/* General rules for assigning cache priorities:
391
* - Data that can be reconstructed from other elements has low prio
392
* (e.g. fulltexts, directories etc.)
393
* - Index data required to find any of the other data has high prio
394
* (e.g. noderevs, L2P and P2L index pages)
395
* - everthing else should use default prio
398
#ifdef SVN_DEBUG_CACHE_DUMP_STATS
400
/* schedule printing the global access statistics upon pool cleanup,
401
* i.e. end of FSX session.
404
apr_pool_cleanup_register(fs->pool,
406
dump_global_cache_statistics,
407
apr_pool_cleanup_null);
410
/* Rough estimate: revision DAG nodes have size around 320 bytes, so
411
* let's put 16 on a page. */
412
SVN_ERR(create_cache(&(ffd->rev_node_cache),
416
svn_fs_x__dag_serialize,
417
svn_fs_x__dag_deserialize,
419
apr_pstrcat(scratch_pool, prefix, "DAG", SVN_VA_NULL),
420
SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
423
fs->pool, scratch_pool));
425
/* 1st level DAG node cache */
426
ffd->dag_node_cache = svn_fs_x__create_dag_cache(fs->pool);
428
/* Very rough estimate: 1K per directory. */
429
SVN_ERR(create_cache(&(ffd->dir_cache),
433
svn_fs_x__serialize_dir_entries,
434
svn_fs_x__deserialize_dir_entries,
435
sizeof(svn_fs_x__id_t),
436
apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL),
437
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
440
fs->pool, scratch_pool));
442
/* Only 16 bytes per entry (a revision number + the corresponding offset).
443
Since we want ~8k pages, that means 512 entries per page. */
444
SVN_ERR(create_cache(&(ffd->packed_offset_cache),
448
svn_fs_x__serialize_manifest,
449
svn_fs_x__deserialize_manifest,
450
sizeof(svn_revnum_t),
451
apr_pstrcat(scratch_pool, prefix, "PACK-MANIFEST",
453
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
456
fs->pool, scratch_pool));
458
/* initialize node revision cache, if caching has been enabled */
459
SVN_ERR(create_cache(&(ffd->node_revision_cache),
462
32, 32, /* ~200 byte / entry; 1k entries total */
463
svn_fs_x__serialize_node_revision,
464
svn_fs_x__deserialize_node_revision,
465
sizeof(svn_fs_x__pair_cache_key_t),
466
apr_pstrcat(scratch_pool, prefix, "NODEREVS",
468
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
471
fs->pool, scratch_pool));
473
/* initialize representation header cache, if caching has been enabled */
474
SVN_ERR(create_cache(&(ffd->rep_header_cache),
477
1, 1000, /* ~8 bytes / entry; 1k entries total */
478
svn_fs_x__serialize_rep_header,
479
svn_fs_x__deserialize_rep_header,
480
sizeof(svn_fs_x__representation_cache_key_t),
481
apr_pstrcat(scratch_pool, prefix, "REPHEADER",
483
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
486
fs->pool, scratch_pool));
488
/* initialize node change list cache, if caching has been enabled */
489
SVN_ERR(create_cache(&(ffd->changes_cache),
492
1, 8, /* 1k / entry; 8 entries total, rarely used */
493
svn_fs_x__serialize_changes,
494
svn_fs_x__deserialize_changes,
495
sizeof(svn_revnum_t),
496
apr_pstrcat(scratch_pool, prefix, "CHANGES",
501
fs->pool, scratch_pool));
503
/* if enabled, cache fulltext and other derived information */
506
SVN_ERR(create_cache(&(ffd->fulltext_cache),
509
0, 0, /* Do not use inprocess cache */
510
/* Values are svn_stringbuf_t */
512
sizeof(svn_fs_x__pair_cache_key_t),
513
apr_pstrcat(scratch_pool, prefix, "TEXT",
515
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
518
fs->pool, scratch_pool));
520
SVN_ERR(create_cache(&(ffd->properties_cache),
523
0, 0, /* Do not use inprocess cache */
524
svn_fs_x__serialize_properties,
525
svn_fs_x__deserialize_properties,
526
sizeof(svn_fs_x__pair_cache_key_t),
527
apr_pstrcat(scratch_pool, prefix, "PROP",
529
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
532
fs->pool, scratch_pool));
534
SVN_ERR(create_cache(&(ffd->mergeinfo_cache),
537
0, 0, /* Do not use inprocess cache */
538
svn_fs_x__serialize_mergeinfo,
539
svn_fs_x__deserialize_mergeinfo,
541
apr_pstrcat(scratch_pool, prefix, "MERGEINFO",
546
fs->pool, scratch_pool));
548
SVN_ERR(create_cache(&(ffd->mergeinfo_existence_cache),
551
0, 0, /* Do not use inprocess cache */
552
/* Values are svn_stringbuf_t */
555
apr_pstrcat(scratch_pool, prefix, "HAS_MERGEINFO",
560
fs->pool, scratch_pool));
564
ffd->fulltext_cache = NULL;
565
ffd->properties_cache = NULL;
566
ffd->mergeinfo_cache = NULL;
567
ffd->mergeinfo_existence_cache = NULL;
570
/* if enabled, cache revprops */
573
SVN_ERR(create_cache(&(ffd->revprop_cache),
576
0, 0, /* Do not use inprocess cache */
577
svn_fs_x__serialize_properties,
578
svn_fs_x__deserialize_properties,
579
sizeof(svn_fs_x__pair_cache_key_t),
580
apr_pstrcat(scratch_pool, prefix, "REVPROP",
582
SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
585
fs->pool, scratch_pool));
589
ffd->revprop_cache = NULL;
592
/* if enabled, cache text deltas and their combinations */
595
SVN_ERR(create_cache(&(ffd->txdelta_window_cache),
598
0, 0, /* Do not use inprocess cache */
599
svn_fs_x__serialize_txdelta_window,
600
svn_fs_x__deserialize_txdelta_window,
601
sizeof(svn_fs_x__window_cache_key_t),
602
apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW",
604
SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
607
fs->pool, scratch_pool));
609
SVN_ERR(create_cache(&(ffd->combined_window_cache),
612
0, 0, /* Do not use inprocess cache */
613
/* Values are svn_stringbuf_t */
615
sizeof(svn_fs_x__window_cache_key_t),
616
apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW",
618
SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
621
fs->pool, scratch_pool));
625
ffd->txdelta_window_cache = NULL;
626
ffd->combined_window_cache = NULL;
629
SVN_ERR(create_cache(&(ffd->noderevs_container_cache),
632
16, 4, /* Important, largish objects */
633
svn_fs_x__serialize_noderevs_container,
634
svn_fs_x__deserialize_noderevs_container,
635
sizeof(svn_fs_x__pair_cache_key_t),
636
apr_pstrcat(scratch_pool, prefix, "NODEREVSCNT",
638
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
641
fs->pool, scratch_pool));
642
SVN_ERR(create_cache(&(ffd->changes_container_cache),
645
0, 0, /* Do not use inprocess cache */
646
svn_fs_x__serialize_changes_container,
647
svn_fs_x__deserialize_changes_container,
648
sizeof(svn_fs_x__pair_cache_key_t),
649
apr_pstrcat(scratch_pool, prefix, "CHANGESCNT",
654
fs->pool, scratch_pool));
655
SVN_ERR(create_cache(&(ffd->reps_container_cache),
658
0, 0, /* Do not use inprocess cache */
659
svn_fs_x__serialize_reps_container,
660
svn_fs_x__deserialize_reps_container,
661
sizeof(svn_fs_x__pair_cache_key_t),
662
apr_pstrcat(scratch_pool, prefix, "REPSCNT",
667
fs->pool, scratch_pool));
669
SVN_ERR(create_cache(&(ffd->l2p_header_cache),
672
64, 16, /* entry size varies but we must cover
673
a reasonable number of revisions (1k) */
674
svn_fs_x__serialize_l2p_header,
675
svn_fs_x__deserialize_l2p_header,
676
sizeof(svn_fs_x__pair_cache_key_t),
677
apr_pstrcat(scratch_pool, prefix, "L2P_HEADER",
679
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
682
fs->pool, scratch_pool));
683
SVN_ERR(create_cache(&(ffd->l2p_page_cache),
686
64, 16, /* entry size varies but we must cover
687
a reasonable number of revisions (1k) */
688
svn_fs_x__serialize_l2p_page,
689
svn_fs_x__deserialize_l2p_page,
690
sizeof(svn_fs_x__page_cache_key_t),
691
apr_pstrcat(scratch_pool, prefix, "L2P_PAGE",
693
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
696
fs->pool, scratch_pool));
697
SVN_ERR(create_cache(&(ffd->p2l_header_cache),
700
4, 1, /* Large entries. Rarely used. */
701
svn_fs_x__serialize_p2l_header,
702
svn_fs_x__deserialize_p2l_header,
703
sizeof(svn_fs_x__pair_cache_key_t),
704
apr_pstrcat(scratch_pool, prefix, "P2L_HEADER",
706
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
709
fs->pool, scratch_pool));
710
SVN_ERR(create_cache(&(ffd->p2l_page_cache),
713
4, 16, /* Variably sized entries. Rarely used. */
714
svn_fs_x__serialize_p2l_page,
715
svn_fs_x__deserialize_p2l_page,
716
sizeof(svn_fs_x__page_cache_key_t),
717
apr_pstrcat(scratch_pool, prefix, "P2L_PAGE",
719
SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
722
fs->pool, scratch_pool));