« back to all changes in this revision
Viewing changes to filecache/strategy.py
- browse files at revision 57
- compare with another revision
- download diff
- download tarball
- view history from revision 57
- Committer: Rick Harris
- Date: 2010-04-28 00:01:40 UTC
- Revision ID: git-v1:2e59ea24954f66594b5d7688bea4d8e266649ad7
Removed delete lock and semaphore Closes #26
added
removed
filecache/strategy.py
1
1
import time
2
2
from math import ceil
3
3
from pprint import pformat
4
5
from sharedfile import SharedFileNotFound
6
4
7
from lib.utils import logger
5
8
from filecache import FileCache, FileCacheException
6
9
9
12
Provides a least-recently-used file cache.
10
13
11
14
Tune keys_pct and size_pct to avoid thrashing near the cut-off point.
12
13
Use pruning_interval to avoid having to the prune the cache on every cache
14
miss. Note that anything value other than one may cause the cache to
15
exceed the stated max_keys or max_size. Take this into account.
16
15
"""
17
16
def _prune(self):
18
17
start_time = time.time()
19
num_keys = 0
20
# N.B. prune already as the delete lock
21
cache_stats = self._unsafe_stat()
18
19
cache_stats = self._stat()
22
20
logger.debug("cache_stats for pruning %s" % pformat(cache_stats))
23
21
24
22
lru_keys = self._make_lru_keys(cache_stats)
25
num_keys += self._prune_max_keys(cache_stats, lru_keys)
26
num_keys += self._prune_max_size(cache_stats, lru_keys)
27
28
logger.info("pruned %s keys in %.2f secs" %
29
(num_keys, (time.time() - start_time)))
23
24
pruned_keys, skipped_keys = self._prune_max_keys(cache_stats, lru_keys)
25
pruned_size, skipped_size = self._prune_max_size(cache_stats, lru_keys)
26
27
pruned = pruned_keys + pruned_size
28
skipped = skipped_keys + skipped_size
29
30
logger.info("pruned %s keys (skipped %s) in %.2f secs" %
31
(pruned, skipped, (time.time() - start_time)))
30
32
31
33
def _make_lru_keys(self, cache_stats):
32
34
""" sort keys newest (highest) -> oldest (lowest) """
38
40
39
41
def _prune_max_keys(self, cache_stats, lru_keys):
40
42
if self.config.max_keys is None:
41
return 0
43
return 0, 0
42
44
43
45
num_keys = len(lru_keys)
44
46
ok_keys = ceil(self.config.keys_pct * self.config.max_keys)
45
47
to_free = num_keys - ok_keys
46
48
if to_free <= 0:
47
return 0
49
return 0, 0
48
50
49
freed = 0
50
51
logger.debug("%i/%i (max: %i) freeing %i items" %
51
52
(num_keys, ok_keys, self.config.max_keys, to_free))
52
53
54
pruned = skipped = 0
53
55
while lru_keys and to_free > 0:
54
56
key = lru_keys.pop()
55
57
56
if self.config.prune_dry_run:
57
logger.debug("would have pruned '%s'" % key)
58
try:
59
self.delete(key)
60
except SharedFileNotFound:
61
logger.debug("skipping pruning of key '%s', not found" % key)
62
skipped += 1
58
63
else:
59
# since the delete lock has been acquired, we need to call
60
# the lock-less delete
61
self._unsafe_delete(key)
62
63
to_free -= 1
64
freed += 1
64
to_free -= 1
65
pruned += 1
65
66
66
return freed
67
return pruned, skipped
67
68
68
69
def _prune_max_size(self, cache_stats, lru_keys):
69
70
if self.config.max_size is None:
70
return 0
71
return 0, 0
71
72
72
73
cache_size = sum(v['size'] for v in cache_stats.values())
73
74
ok_size = ceil(self.config.size_pct * self.config.max_size)
74
75
to_free = cache_size - ok_size
75
76
76
77
if to_free <= 0:
77
return 0
78
return 0, 0
78
79
79
freed = 0
80
80
logger.debug("%ib/%ib (max: %ib) trying to free %i bytes" %
81
81
(cache_size, ok_size, self.config.max_size, to_free))
82
82
83
pruned = skipped = 0
83
84
while lru_keys and to_free > 0:
84
85
key = lru_keys.pop()
85
86
del_size = cache_stats[key]['size']
86
87
87
if self.config.prune_dry_run:
88
logger.debug("would have pruned '%s' for %i bytes"
89
% (key, del_size))
88
try:
89
self.delete(key)
90
except SharedFileNotFound:
91
logger.debug("skipping pruning of key '%s', not found" % key)
92
skipped += 1
90
93
else:
91
# since the delete lock has been acquired, we need to call
92
# the lock-less delete
93
self._unsafe_delete(key)
94
95
to_free -= del_size
96
freed += 1
97
98
return freed
94
to_free -= del_size
95
pruned += 1
96
97
return pruned, skipped
99
98
100
99
def _add_configs(self, cfg):
101
100
cfg.add_config('max_keys', type=int)
104
103
cfg.add_config('max_size', type=int)
105
104
cfg.add_config('size_pct', type=float, default=0.75)
106
105
107
cfg.add_config('prune_dry_run', type=bool, default=False)
108
109
110
111
106
class ExpiryFileCache(FileCache):
112
107
"""
113
108
Provides an expiry file cache
130
125
if expiry is None:
131
126
return True
132
127
133
age = time.time() - self._key_stat(key)["mtime"]
128
age = time.time() - self._make_entry(key).stat()["mtime"]
134
129
logger.debug("key '%s' age %.2f secs with expiry %.2f"
135
130
% (key, age, expiry))
136
131
return age > expiry
137
132
138
133
def _prune(self):
139
134
start_time = time.time()
140
num_keys = 0
141
135
# N.B. prune already as the delete lock
142
cache_stats = self._unsafe_stat()
136
cache_stats = self._stat()
143
137
logger.debug("cache_stats for pruning %s" % pformat(cache_stats))
138
139
pruned = skipped = 0
144
140
for key in cache_stats:
145
141
if self._check_stale(key):
146
self._unsafe_delete(key)
147
num_keys += 1
142
try:
143
self.delete(key)
144
except SharedFileNotFound:
145
logger.debug("skipping pruning of key '%s', not found"
146
% key)
147
skipped += 1
148
else:
149
pruned += 1
148
150
149
logger.info("pruned %s keys in %.2f secs" %
150
(num_keys, (time.time() - start_time)))
151
logger.info("pruned %s keys (skipped %s) in %.2f secs" %
152
(pruned, skipped, (time.time() - start_time)))
151
153
152
154
153
155
def _add_configs(self, cfg):
154
156
cfg.add_config('default_expiry', type=int, default=None)
155
cfg.add_config('prune_dry_run', type=bool, default=False)
156
157
Loggerhead is a web-based interface for Breezy
Version: 2.0.1