34
38
class MultiBackend(duplicity.backend.Backend):
35
"""Store files across multiple remote stores. URL is a path to a local file containing URLs/other config defining the remote store"""
39
"""Store files across multiple remote stores. URL is a path to a local file
40
containing URLs/other config defining the remote store"""
37
42
# the stores we are managing
40
# when we write, we "stripe" via a simple round-robin across
45
# Set of known query paramaters
46
__knownQueryParameters = frozenset([
51
# the mode of operation to follow
52
# can be one of 'stripe' or 'mirror' currently
54
__mode_allowedSet = frozenset([
59
# the write error handling logic
60
# can be one of the following:
61
# * continue - default, on failure continues to next source
62
# * abort - stop all further operations
63
__onfail_mode = 'continue'
64
__onfail_mode_allowedSet = frozenset([
69
# when we write in stripe mode, we "stripe" via a simple round-robin across
41
70
# remote stores. It's hard to get too much more sophisticated
42
71
# since we can't rely on the backend to give us any useful meta
43
72
# data (e.g. sizes of files, capacity of the store (quotas)) to do
44
73
# a better job of balancing load across stores.
77
def get_query_params(parsed_url):
78
# Reparse so the query string is available
79
reparsed_url = urlparse.urlparse(parsed_url.geturl())
80
if len(reparsed_url.query) == 0:
83
queryMultiDict = urlparse.parse_qs(reparsed_url.query, strict_parsing=True)
84
except ValueError as e:
85
log.Log(_("MultiBackend: Could not parse query string %s: %s ")
86
% (reparsed_url.query, e),
88
raise BackendException('Could not parse query string')
90
# Convert the multi-dict to a single dictionary
91
# while checking to make sure that no unrecognized values are found
92
for name, valueList in queryMultiDict.items():
93
if len(valueList) != 1:
94
log.Log(_("MultiBackend: Invalid query string %s: more than one value for %s")
95
% (reparsed_url.query, name),
97
raise BackendException('Invalid query string')
98
if name not in MultiBackend.__knownQueryParameters:
99
log.Log(_("MultiBackend: Invalid query string %s: unknown parameter %s")
100
% (reparsed_url.query, name),
102
raise BackendException('Invalid query string')
104
queryDict[name] = valueList[0]
47
107
def __init__(self, parsed_url):
48
108
duplicity.backend.Backend.__init__(self, parsed_url)
140
queryParams = MultiBackend.get_query_params(parsed_url)
142
if 'mode' in queryParams:
143
self.__mode = queryParams['mode']
145
if 'onfail' in queryParams:
146
self.__onfail_mode = queryParams['onfail']
148
if self.__mode not in MultiBackend.__mode_allowedSet:
149
log.Log(_("MultiBackend: illegal value for %s: %s")
150
% ('mode', self.__mode), log.ERROR)
151
raise BackendException("MultiBackend: invalid mode value")
153
if self.__onfail_mode not in MultiBackend.__onfail_mode_allowedSet:
154
log.Log(_("MultiBackend: illegal value for %s: %s")
155
% ('onfail', self.__onfail_mode), log.ERROR)
156
raise BackendException("MultiBackend: invalid onfail value")
81
159
with open(parsed_url.path) as f:
82
160
configs = json.load(f)
83
161
except IOError as e:
162
log.Log(_("MultiBackend: Url %s")
163
% (parsed_url.geturl()),
84
166
log.Log(_("MultiBackend: Could not load config file %s: %s ")
85
167
% (parsed_url.path, e),
108
192
def _put(self, source_path, remote_filename):
193
# Store an indication of whether any of these passed
195
# Mirror mode always starts at zero
196
if self.__mode == 'mirror':
197
self.__write_cursor = 0
109
199
first = self.__write_cursor
111
201
store = self.__stores[self.__write_cursor]
117
207
% (self.__write_cursor, store.backend.parsed_url.url_string),
119
209
store.put(source_path, remote_filename)
120
211
self.__write_cursor = next
212
# No matter what, if we loop around, break this loop
215
# If in stripe mode, don't continue to the next
216
if self.__mode == 'stripe':
122
218
except Exception as e:
123
219
log.Log(_("MultiBackend: failed to write to store #%s (%s), try #%s, Exception: %s")
124
220
% (self.__write_cursor, store.backend.parsed_url.url_string, next, e),
126
222
self.__write_cursor = next
128
if (self.__write_cursor == first):
224
# If we consider write failure as abort, abort
225
if self.__onfail_mode == 'abort':
226
log.Log(_("MultiBackend: failed to write %s. Aborting process.")
229
raise BackendException("failed to write")
231
# If we've looped around, and none of them passed, fail
232
if (self.__write_cursor == first) and not passed:
129
233
log.Log(_("MultiBackend: failed to write %s. Tried all backing stores and none succeeded")
159
263
% (s.backend.parsed_url.url_string, l),
161
265
lists.append(s.list())
162
# combine the lists into a single flat list:
163
result = [item for sublist in lists for item in sublist]
266
# combine the lists into a single flat list w/o duplicates via set:
267
result = list({item for sublist in lists for item in sublist})
164
268
log.Log(_("MultiBackend: combined list: %s")
169
273
def _delete(self, filename):
274
# Store an indication on whether any passed
170
276
# since the backend operations will be retried, we can't
171
277
# simply try to get from the store, if not found, move to the
172
278
# next store (since each failure will be retried n times
178
284
if filename in list:
179
285
s._do_delete(filename)
287
# In stripe mode, only one item will have the file
288
if self.__mode == 'stripe':
181
290
log.Log(_("MultiBackend: failed to delete %s from %s")
182
291
% (filename, s.backend.parsed_url.url_string),
184
log.Log(_("MultiBackend: failed to delete %s. Tried all backing stores and none succeeded")
187
# raise BackendException("failed to delete")
294
log.Log(_("MultiBackend: failed to delete %s. Tried all backing stores and none succeeded")
297
# raise BackendException("failed to delete")
189
299
duplicity.backend.register_backend('multi', MultiBackend)
added
removed
duplicity/backends/multibackend.py
