« back to all changes in this revision
Viewing changes to filecache/__init__.py
- browse files at revision 47
- compare with another revision
- download diff
- download tarball
- view history from revision 47
- Committer: Rick Harris
- Date: 2010-04-23 00:48:59 UTC
- Revision ID: git-v1:017b7d851a640a83a3222ea53d15a7634e546d3a
Added SharedFileDict Closes #22
- files modified:
- filecache/__init__.py
added
removed
filecache/__init__.py
10
10
from pprint import pformat
11
11
import md5 # use hashlib
12
12
from tempfile import gettempdir
13
import lib.kval
13
14
import lib.kval as kval
14
15
from lib.utils import safe_mkdir, safe_rmdir, logger
15
16
from lib.filesem import FileSemaphore
16
17
from lib.filelock import FileLock, FileLockAcquired
17
18
from lib.sharedfile import (SharedFile, SharedFileNotFound,
18
SharedFileRollbackWithDelete)
19
SharedFileRollbackWithDelete, SharedFileDict)
19
20
20
21
__version__ = "0.2.1b"
21
22
26
27
pass
27
28
28
29
29
class FileCacheEntryMetadata(dict):
30
""" Provies a dictionary interface for metadata """
31
#TODO: This is really an AtomicDict, extact it
32
def __init__(self, manager, *args, **kwargs):
33
super(FileCacheEntryMetadata, self).__init__(*args, **kwargs)
34
self.manager = manager
35
36
def __getitem__(self, key):
37
f = SharedFile(self.manager.entry.metadata_path)
38
try:
39
f.open(mode='r')
40
except SharedFileNotFound:
41
raise KeyError
42
try:
43
self.update(self.manager.metadata_serializer.load(f))
44
return super(FileCacheEntryMetadata, self).__getitem__(key)
45
except:
46
f.request_rollback()
47
raise
48
finally:
49
f.close()
50
51
def __setitem__(self, key, value):
52
def updater(data):
53
super(FileCacheEntryMetadata, self).__setitem__(key, value)
54
55
try:
56
self._atomic_update({key: value}, updater)
57
except SharedFileNotFound:
58
self._atomic_write({key: value}, updater)
59
60
logger.debug("metadata key '%s' is now %s" % (key, value))
61
62
def __delitem__(self, key):
63
def updater(data):
64
super(FileCacheEntryMetadata, self).__delitem__(key)
65
if not self:
66
# no keys left, rm the metadata file
67
raise SharedFileRollbackWithDelete
68
try:
69
self._atomic_update(None, updater)
70
except SharedFileNotFound:
71
raise KeyError("key '%s' not found" % key)
72
73
def update(self, data):
74
def updater(data):
75
super(FileCacheEntryMetadata, self).update(data)
76
try:
77
self._atomic_update(data, updater)
78
except SharedFileNotFound:
79
self._atomic_write(data)
80
81
def _atomic_write(self, data=None, update_fn=None):
82
""" Munge data dict, then write it to disk """
83
f = SharedFile(self.manager.entry.metadata_path)
84
f.open(mode='w')
85
try:
86
if update_fn is not None:
87
update_fn(data)
88
self.manager.metadata_serializer.dump(self, f)
89
except:
90
f.request_rollback()
91
raise
92
finally:
93
f.close()
94
95
def _atomic_update(self, data=None, update_fn=None):
96
""" Read data, munge it, write it back, atomically. If the file
97
doesn't exist already, use _atomic_write()
98
"""
99
data = data or {}
100
f = SharedFile(self.manager.entry.metadata_path)
101
f.open(mode='rw')
102
try:
103
try:
104
super(FileCacheEntryMetadata, self).update(
105
self.manager.metadata_serializer.load(f))
106
if update_fn is not None:
107
update_fn(data)
108
self.manager.metadata_serializer.dump(self, f)
109
except SharedFileRollbackWithDelete:
110
f.request_rollback(delete=True)
111
except:
112
f.request_rollback()
113
raise
114
finally:
115
f.close()
116
117
#TODO: __contains__, __len__, keys()
118
119
class FileCacheEntryMetadataManager(object):
120
""" Provides a means to attach metadata to a cache entry """
121
def __init__(self, entry, metadata_serializer=lib.kval):
122
"""
123
entry - pointer back to cache entry
124
metadata_serializer - used to read/write metadata to file
125
"""
126
self.entry = entry
127
self.metadata_serializer = metadata_serializer
128
129
def set(self, metadata):
130
if not metadata:
131
return
132
133
f = SharedFile(self.entry.metadata_path)
134
f.open(mode='w')
135
try:
136
self.metadata_serializer.dump(metadata, f)
137
except:
138
f.request_rollback()
139
raise
140
finally:
141
f.close()
142
143
logger.debug("'%s' metadata is now %s" %
144
(self.entry.key, pformat(metadata)))
145
146
def get(self):
147
f = SharedFile(self.entry.metadata_path)
148
try:
149
f.open(mode='r')
150
except SharedFileNotFound:
151
return FileCacheEntryMetadata(self)
152
153
try:
154
return FileCacheEntryMetadata(
155
self, self.metadata_serializer.load(f))
156
finally:
157
f.close()
158
159
def delete(self, wait=True):
160
logger.debug("deleting metadata for '%s'" % self.entry.key)
161
try:
162
SharedFile(self.entry.metadata_path).delete(wait=wait)
163
except SharedFileNotFound:
164
pass
165
166
30
class FileCacheEntry(SharedFile):
167
31
""" Represents a cached file
168
32
170
34
metadata_serializer, the default being kval ("a=b").
171
35
"""
172
36
173
def __init__(self, cache, key, overwrite=True, wait=True, timeout=10,
174
delay=0.5):
37
def __init__(self, cache, key, wait=True, timeout=10, delay=0.5):
175
38
""" create a new cache entry
176
39
177
40
cache - points back to parent cache
182
45
path = os.path.join(self.cache.path, self.key)
183
46
184
47
super(FileCacheEntry, self).__init__(
185
path, overwrite=overwrite, wait=wait, timeout=timeout, delay=delay)
48
path, wait=wait, timeout=timeout, delay=delay)
186
49
187
50
self.metadata_path = os.path.join(self.cache.path,
188
51
".%s.metadata" % self.key)
189
self.metadata_manager = FileCacheEntryMetadataManager(self)
52
53
self.metadata_file_dict = SharedFileDict(
54
self.metadata_path, serializer=kval)
190
55
191
56
def delete(self, wait=True):
192
57
del self.metadata
193
58
SharedFile.delete(self, wait=wait)
194
59
60
def _unsafe_delete(self):
61
self.metadata._unsafe_delete()
62
SharedFile._unsafe_delete(self)
63
195
64
def _get_metadata(self):
196
return self.metadata_manager.get()
65
return self.metadata_file_dict
66
197
67
def _set_metadata(self, metadata):
198
self.metadata_manager.set(metadata)
68
self.metadata_file_dict.set(metadata)
69
logger.debug("'%s' metadata is now %s" %
70
(self.key, pformat(metadata)))
71
199
72
def _del_metadata(self):
200
self.metadata_manager.delete()
73
logger.debug("deleting metadata for '%s'" % self.key)
74
self.metadata_file_dict.clear()
201
75
202
76
metadata = property(_get_metadata, _set_metadata, _del_metadata)
203
77
388
262
return False
389
263
return self._check_valid(key)
390
264
391
def _make_entry(self, key, overwrite=True, wait=True, timeout=10,
392
delay=0.5):
393
return FileCacheEntry(self, key, overwrite=overwrite, wait=wait,
394
timeout=timeout, delay=delay)
265
def _make_entry(self, key, wait=True, timeout=10, delay=0.5):
266
return FileCacheEntry(self, key, wait=wait, timeout=timeout,
267
delay=delay)
395
268
396
269
def _rmdir_path(self):
397
270
safe_rmdir(self.path)
527
400
logger.debug("cached nx entry for key '%s'" % entry.key)
528
401
529
402
def _unsafe_delete(self, key):
530
self._make_entry(key).delete()
403
self._make_entry(key)._unsafe_delete()
531
404
532
405
def _make_cache_lock_path(self, suffix):
533
406
# All users of the cache need to agree upon the names of the global
Loggerhead is a web-based interface for Breezy
Version: 2.0.1