1
# Copyright 2013-2014 Lars Wirzenius
3
# This program is free software: you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation, either version 3 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16
# =*= License: GPL-3+ =*=
22
class KeyValueStore(object):
27
def get_value(self, key, default):
32
def set_value(self, key, value):
33
self._map[key] = value
35
def remove_value(self, key):
39
return self._map.items()
42
other = KeyValueStore()
43
for key, value in self.items():
44
other.set_value(key, value)
48
class LockableKeyValueStore(object):
52
self.data = KeyValueStore()
56
assert not self.locked
57
self.stashed = self.data
58
self.data = self.data.copy()
63
self.data = self.stashed
72
def get_value(self, key, default):
73
return self.data.get_value(key, default)
75
def set_value(self, key, value):
76
self.data.set_value(key, value)
78
def remove_value(self, key):
79
self.data.remove_value(key)
82
return self.data.items()
85
class Counter(object):
95
class DummyClient(object):
97
def __init__(self, name):
100
self.generation_counter = Counter()
101
self.data = LockableKeyValueStore()
104
return self.data.locked
108
raise obnamlib.RepositoryClientLockingFailed(
109
client_name=self.name)
112
def _require_lock(self):
113
if not self.data.locked:
114
raise obnamlib.RepositoryClientNotLocked(client_name=self.name)
120
def force_lock(self):
126
self.data.set_value('current-generation', None)
129
def get_key(self, key):
130
return self.data.get_value(key, '')
132
def set_key(self, key, value):
134
self.data.set_value(key, value)
136
def get_generation_ids(self):
137
key = 'generation-ids'
138
return self.data.get_value(key, [])
140
def create_generation(self):
142
if self.data.get_value('current-generation', None) is not None:
143
raise obnamlib.RepositoryClientGenerationUnfinished(
144
client_name=self.name)
145
generation_id = (self.name, self.generation_counter.next())
146
ids = self.data.get_value('generation-ids', [])
147
self.data.set_value('generation-ids', ids + [generation_id])
148
self.data.set_value('current-generation', generation_id)
151
prev_gen_id = ids[-1]
152
for key, value in self.data.items():
153
if self._is_filekey(key):
154
x, gen_id, filename = key
155
if gen_id == prev_gen_id:
156
value = self.data.get_value(key, None)
158
self._filekey(generation_id, filename), value)
159
elif self._is_filekeykey(key):
160
x, gen_id, filename, k = key
161
if gen_id == prev_gen_id:
162
value = self.data.get_value(key, None)
164
self._filekeykey(generation_id, filename, k),
166
elif self._is_filechunkskey(key):
167
x, gen_id, filename = key
168
if gen_id == prev_gen_id:
169
value = self.data.get_value(key, [])
171
self._filechunkskey(generation_id, filename),
176
def _require_generation(self, gen_id):
177
ids = self.data.get_value('generation-ids', [])
178
if gen_id not in ids:
179
raise obnamlib.RepositoryGenerationDoesNotExist(
180
client_name=self.name, gen_id=gen_id)
182
def get_generation_key(self, gen_id, key):
183
return self.data.get_value(gen_id + (key,), '')
185
def set_generation_key(self, gen_id, key, value):
187
self.data.set_value(gen_id + (key,), value)
189
def remove_generation(self, gen_id):
191
self._require_generation(gen_id)
192
ids = self.data.get_value('generation-ids', [])
193
self.data.set_value('generation-ids', [x for x in ids if x != gen_id])
195
def get_generation_chunk_ids(self, gen_id):
197
for key, value in self.data.items():
198
if self._is_filechunkskey(key) and key[1] == gen_id:
199
chunk_ids.extend(value)
202
def interpret_generation_spec(self, genspec):
203
ids = self.data.get_value('generation-ids', [])
205
raise obnamlib.RepositoryClientHasNoGenerations(
206
client_name=self.name)
207
if genspec == 'latest':
211
gen_number = int(genspec)
212
if (self.name, gen_number) in ids:
213
return (self.name, gen_number)
214
raise obnamlib.RepositoryGenerationDoesNotExist(
215
client_name=self.name, gen_id=genspec)
217
def make_generation_spec(self, generation_id):
218
name, gen_number = generation_id
219
return str(gen_number)
221
def _filekey(self, gen_id, filename):
222
return ('file', gen_id, filename)
224
def _is_filekey(self, key):
225
return (type(key) is tuple and len(key) == 3 and key[0] == 'file')
227
def file_exists(self, gen_id, filename):
228
return self.data.get_value(self._filekey(gen_id, filename), False)
230
def add_file(self, gen_id, filename):
231
self.data.set_value(self._filekey(gen_id, filename), True)
233
def remove_file(self, gen_id, filename):
235
for key, value in self.data.items():
237
self._is_filekey(key) or
238
self._is_filekeykey(key) or
239
self._is_filechunkskey(key))
241
if key[1] == gen_id and key[2] == filename:
245
self.data.remove_value(k)
247
def _filekeykey(self, gen_id, filename, key):
248
return ('filekey', gen_id, filename, key)
250
def _is_filekeykey(self, key):
251
return (type(key) is tuple and len(key) == 4 and key[0] == 'filekey')
253
def _require_file(self, gen_id, filename):
254
if not self.file_exists(gen_id, filename):
255
raise obnamlib.RepositoryFileDoesNotExistInGeneration(
256
client_name=self.name,
257
genspec=self.make_generation_spec(gen_id),
260
def get_file_key(self, gen_id, filename, key):
261
self._require_generation(gen_id)
262
self._require_file(gen_id, filename)
263
if key in obnamlib.REPO_FILE_INTEGER_KEYS:
267
return self.data.get_value(
268
self._filekeykey(gen_id, filename, key), default)
270
def set_file_key(self, gen_id, filename, key, value):
271
self._require_generation(gen_id)
272
self._require_file(gen_id, filename)
273
self.data.set_value(self._filekeykey(gen_id, filename, key), value)
275
def _filechunkskey(self, gen_id, filename):
276
return ('filechunks', gen_id, filename)
278
def _is_filechunkskey(self, key):
280
type(key) is tuple and len(key) == 3 and key[0] == 'filechunks')
282
def get_file_chunk_ids(self, gen_id, filename):
283
self._require_generation(gen_id)
284
self._require_file(gen_id, filename)
285
return self.data.get_value(self._filechunkskey(gen_id, filename), [])
287
def append_file_chunk_id(self, gen_id, filename, chunk_id):
288
self._require_generation(gen_id)
289
self._require_file(gen_id, filename)
290
chunk_ids = self.get_file_chunk_ids(gen_id, filename)
292
self._filechunkskey(gen_id, filename),
293
chunk_ids + [chunk_id])
295
def clear_file_chunk_ids(self, gen_id, filename):
296
self._require_generation(gen_id)
297
self._require_file(gen_id, filename)
298
self.data.set_value(self._filechunkskey(gen_id, filename), [])
300
def get_file_children(self, gen_id, filename):
302
if filename.endswith('/'):
305
prefix = filename + '/'
306
for key, value in self.data.items():
307
if not self._is_filekey(key):
309
x, y, candidate = key
310
if candidate == filename:
312
if not candidate.startswith(prefix): # pragma: no cover
314
if '/' in candidate[len(prefix):]:
316
children.append(candidate)
320
class DummyClientList(object):
323
self.data = LockableKeyValueStore()
327
raise obnamlib.RepositoryClientListLockingFailed()
331
if not self.data.locked:
332
raise obnamlib.RepositoryClientListNotLocked()
336
if not self.data.locked:
337
raise obnamlib.RepositoryClientListNotLocked()
344
def _require_lock(self):
345
if not self.data.locked:
346
raise obnamlib.RepositoryClientListNotLocked()
349
return [k for k, v in self.data.items() if v is not None]
351
def __getitem__(self, client_name):
352
client = self.data.get_value(client_name, None)
354
raise obnamlib.RepositoryClientDoesNotExist(
355
client_name=client_name)
358
def add(self, client_name):
360
if self.data.get_value(client_name, None) is not None:
361
raise obnamlib.RepositoryClientAlreadyExists(
362
client_name=client_name)
363
self.data.set_value(client_name, DummyClient(client_name))
365
def remove(self, client_name):
367
if self.data.get_value(client_name, None) is None:
368
raise obnamlib.RepositoryClientDoesNotExist(
369
client_name=client_name)
370
self.data.set_value(client_name, None)
372
def rename(self, old_client_name, new_client_name):
374
client = self.data.get_value(old_client_name, None)
376
raise obnamlib.RepositoryClientDoesNotExist(
377
client_name=old_client_name)
378
if self.data.get_value(new_client_name, None) is not None:
379
raise obnamlib.RepositoryClientAlreadyExists(
380
client_name=new_client_name)
381
self.data.set_value(old_client_name, None)
382
self.data.set_value(new_client_name, client)
384
def get_client_by_generation_id(self, gen_id):
385
client_name, generation_number = gen_id
386
return self[client_name]
388
def get_client_encryption_key_id(self, client_name):
389
client = self.data.get_value(client_name, None)
391
raise obnamlib.RepositoryClientDoesNotExist(
392
client_name=client_name)
395
def set_client_encryption_key_id(self, client_name, key_id):
396
client = self.data.get_value(client_name, None)
398
raise obnamlib.RepositoryClientDoesNotExist(
399
client_name=client_name)
400
client.key_id = key_id
403
class ChunkStore(object):
406
self.next_chunk_id = Counter()
409
def put_chunk_content(self, content):
410
chunk_id = self.next_chunk_id.next()
411
self.chunks[chunk_id] = content
414
def get_chunk_content(self, chunk_id):
415
if chunk_id not in self.chunks:
416
raise obnamlib.RepositoryChunkDoesNotExist(chunk_id=str(chunk_id))
417
return self.chunks[chunk_id]
419
def has_chunk(self, chunk_id):
420
return chunk_id in self.chunks
422
def remove_chunk(self, chunk_id):
423
if chunk_id not in self.chunks:
424
raise obnamlib.RepositoryChunkDoesNotExist(chunk_id=str(chunk_id))
425
del self.chunks[chunk_id]
427
def get_chunk_ids(self):
428
return self.chunks.keys()
431
class ChunkIndexes(object):
434
self.data = LockableKeyValueStore()
438
raise obnamlib.RepositoryChunkIndexesLockingFailed()
441
def _require_lock(self):
442
if not self.data.locked:
443
raise obnamlib.RepositoryChunkIndexesNotLocked()
457
def prepare(self, chunk_content):
460
def put_chunk(self, chunk_id, token_is_chunk_content, client_id):
462
self.data.set_value(chunk_id, token_is_chunk_content)
464
def find_chunk(self, chunk_content):
466
for chunk_id, stored_content in self.data.items():
467
if stored_content == chunk_content:
468
chunk_ids.append(chunk_id)
470
raise obnamlib.RepositoryChunkContentNotInIndexes()
473
def remove_chunk(self, chunk_id, client_id):
475
self.data.set_value(chunk_id, None)
478
class RepositoryFormatDummy(obnamlib.RepositoryInterface):
480
'''Simplistic repository format for testing.
482
This class exists to exercise the RepositoryInterfaceTests class.
489
self._client_list = DummyClientList()
490
self._chunk_store = ChunkStore()
491
self._chunk_indexes = ChunkIndexes()
497
def set_fs(self, fs):
506
def get_client_names(self):
507
return self._client_list.names()
509
def lock_client_list(self):
510
self._client_list.lock()
512
def unlock_client_list(self):
513
self._client_list.unlock()
515
def commit_client_list(self):
516
self._client_list.commit()
518
def got_client_list_lock(self):
519
return self._client_list.data.locked
521
def force_client_list_lock(self):
522
self._client_list.force()
524
def add_client(self, client_name):
525
self._client_list.add(client_name)
527
def remove_client(self, client_name):
528
self._client_list.remove(client_name)
530
def rename_client(self, old_client_name, new_client_name):
531
self._client_list.rename(old_client_name, new_client_name)
533
def get_client_encryption_key_id(self, client_name):
534
return self._client_list.get_client_encryption_key_id(client_name)
536
def set_client_encryption_key_id(self, client_name, key_id):
537
self._client_list.set_client_encryption_key_id(client_name, key_id)
539
def client_is_locked(self, client_name):
540
return self._client_list[client_name].is_locked()
542
def lock_client(self, client_name):
543
self._client_list[client_name].lock()
545
def unlock_client(self, client_name):
546
self._client_list[client_name].unlock()
548
def commit_client(self, client_name):
549
self._client_list[client_name].commit()
551
def got_client_lock(self, client_name):
552
return self._client_list[client_name].data.locked
554
def force_client_lock(self, client_name):
555
self._client_list[client_name].force_lock()
557
def get_allowed_client_keys(self):
558
return [obnamlib.REPO_CLIENT_TEST_KEY]
560
def get_client_key(self, client_name, key):
561
return self._client_list[client_name].get_key(key)
563
def set_client_key(self, client_name, key, value):
564
if key not in self.get_allowed_client_keys():
565
raise obnamlib.RepositoryClientKeyNotAllowed(
567
client_name=client_name,
568
key_name=obnamlib.repo_key_name(key))
569
self._client_list[client_name].set_key(key, value)
571
def get_client_generation_ids(self, client_name):
572
return self._client_list[client_name].get_generation_ids()
574
def get_client_extra_data_directory(self, client_name):
577
def create_generation(self, client_name):
578
return self._client_list[client_name].create_generation()
580
def get_allowed_generation_keys(self):
582
obnamlib.REPO_GENERATION_TEST_KEY,
583
obnamlib.REPO_GENERATION_STARTED,
584
obnamlib.REPO_GENERATION_ENDED,
587
def get_generation_key(self, generation_id, key):
588
client = self._client_list.get_client_by_generation_id(generation_id)
589
return client.get_generation_key(generation_id, key)
591
def set_generation_key(self, generation_id, key, value):
592
client = self._client_list.get_client_by_generation_id(generation_id)
593
if key not in self.get_allowed_generation_keys():
594
raise obnamlib.RepositoryGenerationKeyNotAllowed(
596
client_name=client.name,
597
key_name=obnamlib.repo_key_name(key))
598
return client.set_generation_key(generation_id, key, value)
600
def remove_generation(self, generation_id):
601
client = self._client_list.get_client_by_generation_id(generation_id)
602
client.remove_generation(generation_id)
604
def get_generation_chunk_ids(self, generation_id):
605
client = self._client_list.get_client_by_generation_id(generation_id)
606
return client.get_generation_chunk_ids(generation_id)
608
def interpret_generation_spec(self, client_name, genspec):
609
client = self._client_list[client_name]
610
return client.interpret_generation_spec(genspec)
612
def make_generation_spec(self, generation_id):
613
client = self._client_list.get_client_by_generation_id(generation_id)
614
return client.make_generation_spec(generation_id)
616
def file_exists(self, generation_id, filename):
617
client = self._client_list.get_client_by_generation_id(generation_id)
618
return client.file_exists(generation_id, filename)
620
def add_file(self, generation_id, filename):
621
client = self._client_list.get_client_by_generation_id(generation_id)
622
return client.add_file(generation_id, filename)
624
def remove_file(self, generation_id, filename):
625
client = self._client_list.get_client_by_generation_id(generation_id)
626
return client.remove_file(generation_id, filename)
628
def get_file_key(self, generation_id, filename, key):
629
client = self._client_list.get_client_by_generation_id(generation_id)
630
if key not in self.get_allowed_file_keys():
631
raise obnamlib.RepositoryFileKeyNotAllowed(
633
client_name=client.name,
634
key_name=obnamlib.repo_key_name(key))
635
return client.get_file_key(generation_id, filename, key)
637
def set_file_key(self, generation_id, filename, key, value):
638
client = self._client_list.get_client_by_generation_id(generation_id)
639
if key not in self.get_allowed_file_keys():
640
raise obnamlib.RepositoryFileKeyNotAllowed(
642
client_name=client.name,
643
key_name=obnamlib.repo_key_name(key))
644
client.set_file_key(generation_id, filename, key, value)
646
def get_allowed_file_keys(self):
648
obnamlib.REPO_FILE_TEST_KEY,
649
obnamlib.REPO_FILE_MODE,
650
obnamlib.REPO_FILE_MTIME_SEC,
651
obnamlib.REPO_FILE_MTIME_NSEC,
652
obnamlib.REPO_FILE_ATIME_SEC,
653
obnamlib.REPO_FILE_ATIME_NSEC,
654
obnamlib.REPO_FILE_NLINK,
655
obnamlib.REPO_FILE_SIZE,
656
obnamlib.REPO_FILE_UID,
657
obnamlib.REPO_FILE_GID,
658
obnamlib.REPO_FILE_BLOCKS,
659
obnamlib.REPO_FILE_DEV,
660
obnamlib.REPO_FILE_INO,
661
obnamlib.REPO_FILE_USERNAME,
662
obnamlib.REPO_FILE_GROUPNAME,
663
obnamlib.REPO_FILE_SYMLINK_TARGET,
664
obnamlib.REPO_FILE_XATTR_BLOB,
665
obnamlib.REPO_FILE_MD5,
668
def get_file_chunk_ids(self, generation_id, filename):
669
client = self._client_list.get_client_by_generation_id(generation_id)
670
return client.get_file_chunk_ids(generation_id, filename)
672
def append_file_chunk_id(self, generation_id, filename, chunk_id):
673
client = self._client_list.get_client_by_generation_id(generation_id)
674
return client.append_file_chunk_id(generation_id, filename, chunk_id)
676
def clear_file_chunk_ids(self, generation_id, filename):
677
client = self._client_list.get_client_by_generation_id(generation_id)
678
client.clear_file_chunk_ids(generation_id, filename)
680
def get_file_children(self, generation_id, filename):
681
client = self._client_list.get_client_by_generation_id(generation_id)
682
return client.get_file_children(generation_id, filename)
684
def put_chunk_content(self, content):
685
return self._chunk_store.put_chunk_content(content)
687
def get_chunk_content(self, chunk_id):
688
return self._chunk_store.get_chunk_content(chunk_id)
690
def has_chunk(self, chunk_id):
691
return self._chunk_store.has_chunk(chunk_id)
693
def remove_chunk(self, chunk_id):
694
self._chunk_store.remove_chunk(chunk_id)
696
def get_chunk_ids(self):
697
return self._chunk_store.get_chunk_ids()
699
def lock_chunk_indexes(self):
700
self._chunk_indexes.lock()
702
def unlock_chunk_indexes(self):
703
self._chunk_indexes.unlock()
705
def commit_chunk_indexes(self):
706
self._chunk_indexes.commit()
708
def got_chunk_indexes_lock(self):
709
return self._chunk_indexes.data.locked
711
def force_chunk_indexes_lock(self):
712
self._chunk_indexes.force()
714
def prepare_chunk_for_indexes(self, chunk_content):
715
return self._chunk_indexes.prepare(chunk_content)
717
def put_chunk_into_indexes(self, chunk_id, token, client_id):
718
self._chunk_indexes.put_chunk(chunk_id, token, client_id)
720
def find_chunk_ids_by_content(self, chunk_content):
721
return self._chunk_indexes.find_chunk(chunk_content)
723
def remove_chunk_from_indexes(self, chunk_id, client_id):
724
return self._chunk_indexes.remove_chunk(chunk_id, client_id)
726
def validate_chunk_content(self, chunk_id):
729
def get_fsck_work_items(self):
added
removed
obnamlib/repo_dummy.py
