« back to all changes in this revision
Viewing changes to duplicity/backend.py
- browse files at revision 978.2.1
- compare with another revision
- download diff
- download tarball
- view history from revision 978.2.1
- Committer: Michael Terry
- Date: 2014-04-21 19:21:45 UTC
- mto: This revision was merged to the branch mainline in revision 981.
- Revision ID: michael.terry@canonical.com-20140421192145-b1vlb0hppnn8jrtl
Checkpoint
- files added:
- duplicity/backends/README
- files modified:
- bin/duplicity
added
removed
duplicity/backend.py
31
31
import re
32
32
import getpass
33
33
import gettext
34
import types
34
35
import urllib
35
36
import urlparse
36
37
38
39
from duplicity import file_naming
39
40
from duplicity import globals
40
41
from duplicity import log
42
from duplicity import path
41
43
from duplicity import progress
44
from duplicity import util
42
45
43
46
from duplicity.util import exception_traceback
44
47
45
from duplicity.errors import BackendException, FatalBackendError
48
from duplicity.errors import BackendException
49
from duplicity.errors import FatalBackendException
46
50
from duplicity.errors import TemporaryLoadException
47
51
from duplicity.errors import ConflictingScheme
48
52
from duplicity.errors import InvalidBackendURL
296
300
# Replace the full network location with the stripped copy.
297
301
return parsed_url.geturl().replace(parsed_url.netloc, straight_netloc, 1)
298
302
299
300
# Decorator for backend operation functions to simplify writing one that
301
# retries. Make sure to add a keyword argument 'raise_errors' to your function
302
# and if it is true, raise an exception on an error. If false, fatal-log it.
303
def retry(fn):
304
def iterate(*args):
305
for n in range(1, globals.num_retries):
306
try:
307
kwargs = {"raise_errors" : True}
308
return fn(*args, **kwargs)
309
except Exception as e:
310
log.Warn(_("Attempt %s failed: %s: %s")
311
% (n, e.__class__.__name__, str(e)))
312
log.Debug(_("Backtrace of previous error: %s")
313
% exception_traceback())
314
if isinstance(e, TemporaryLoadException):
315
time.sleep(30) # wait longer before trying again
316
else:
317
time.sleep(10) # wait a bit before trying again
318
# Now try one last time, but fatal-log instead of raising errors
319
kwargs = {"raise_errors" : False}
320
return fn(*args, **kwargs)
321
return iterate
322
323
# same as above, a bit dumber and always dies fatally if last trial fails
324
# hence no need for the raise_errors var ;), we really catch everything here
325
# as we don't know what the underlying code comes up with and we really *do*
326
# want to retry globals.num_retries times under all circumstances
327
def retry_fatal(fn):
328
def _retry_fatal(self, *args):
303
def get_code_from_exception(backend, e):
304
if isinstance(e, BackendException) and e.code != log.ErrorCode.backend_error:
305
return e.code
306
elif hasattr(backend, '_error_code'):
307
return backend._error_code(e) or log.ErrorCode.backend_error
308
else:
309
return log.ErrorCode.backend_error
310
311
def _retry(self, num_retries, fn, *args):
312
for n in range(1, num_retries):
329
313
try:
330
n = 0
314
return fn(self, *args)
315
except FatalBackendException as e:
316
# die on fatal errors
317
raise e
318
except Exception as e:
319
# retry on anything else
320
log.Warn(_("Attempt %s failed. %s: %s")
321
% (n, e.__class__.__name__, str(e)))
322
log.Debug(_("Backtrace of previous error: %s")
323
% exception_traceback())
324
if get_code_from_exception(self, e) == log.ErrorCode.backend_not_found:
325
# If we tried to do something, but the file just isn't there,
326
# no need to retry.
327
return
328
if isinstance(e, TemporaryLoadException):
329
time.sleep(1)#90) # wait longer before trying again
330
else:
331
time.sleep(1)#30) # wait a bit before trying again
332
if hasattr(self, '_retry_cleanup'):
333
self._retry_cleanup()
334
335
def retry(fatal=True):
336
# Decorators with arguments introduce a new level of indirection. So we
337
# have to return a decorator function (which itself returns a function!)
338
def handle_fatal_error(backend, e, n):
339
code = get_code_from_exception(backend, e)
340
def make_filename(f):
341
if isinstance(f, path.ROPath):
342
return util.escape(f.name)
343
else:
344
return util.escape(f)
345
extra = ' '.join([fn.__name__] + [make_filename(x) for x in args if x])
346
log.FatalError(_("Giving up after %s attempts. %s: %s")
347
% (n, e.__class__.__name__,
348
str(e)), code=code, extra=extra)
349
350
def outer_retry(fn):
351
def inner_retry(self, *args):
331
352
for n in range(1, globals.num_retries):
332
353
try:
333
self.retry_count = n
334
354
return fn(self, *args)
335
except FatalBackendError as e:
355
except FatalBackendException as e:
336
356
# die on fatal errors
337
357
raise e
338
358
except Exception as e:
339
359
# retry on anything else
340
log.Warn(_("Attempt %s failed. %s: %s")
341
% (n, e.__class__.__name__, str(e)))
342
360
log.Debug(_("Backtrace of previous error: %s")
343
361
% exception_traceback())
344
time.sleep(10) # wait a bit before trying again
345
# final trial, die on exception
346
self.retry_count = n+1
347
return fn(self, *args)
348
except Exception as e:
349
log.Debug(_("Backtrace of previous error: %s")
350
% exception_traceback())
351
log.FatalError(_("Giving up after %s attempts. %s: %s")
352
% (self.retry_count, e.__class__.__name__, str(e)),
353
log.ErrorCode.backend_error)
354
self.retry_count = 0
362
at_end = n == globals.num_retries
363
if get_code_from_exception(self, e) == log.ErrorCode.backend_not_found:
364
# If we tried to do something, but the file just isn't there,
365
# no need to retry.
366
at_end = True
367
if at_end and fatal:
368
handle_fatal_error(self, e, n)
369
else:
370
log.Warn(_("Attempt %s failed. %s: %s")
371
% (n, e.__class__.__name__, str(e)))
372
if not at_end:
373
if isinstance(e, TemporaryLoadException):
374
time.sleep(1)#90) # wait longer before trying again
375
else:
376
time.sleep(1)#30) # wait a bit before trying again
377
if hasattr(self, '_retry_cleanup'):
378
self._retry_cleanup()
355
379
356
return _retry_fatal
380
return inner_retry
381
return outer_retry
357
382
358
383
class Backend(object):
359
384
"""
360
385
Represents a generic duplicity backend, capable of storing and
361
386
retrieving files.
362
387
363
Concrete sub-classes are expected to implement:
364
365
- put
366
- get
367
- list
368
- delete
369
- close (if needed)
370
371
Optional:
372
373
- move
388
See README in backends directory for information on how to write a backend.
374
389
"""
375
390
376
391
def __init__(self, parsed_url):
377
392
self.parsed_url = parsed_url
378
393
379
def put(self, source_path, remote_filename = None):
394
def __do_put(self, source_path, remote_filename):
395
if hasattr(self, '_put'):
396
log.Info(_("Writing %s") % remote_filename)
397
self._put(source_path, remote_filename)
398
else:
399
raise NotImplementedError()
400
401
@retry(fatal=True)
402
def put(self, source_path, remote_filename=None):
380
403
"""
381
404
Transfer source_path (Path object) to remote_filename (string)
382
405
383
406
If remote_filename is None, get the filename from the last
384
407
path component of pathname.
385
408
"""
386
raise NotImplementedError()
409
if not remote_filename:
410
remote_filename = source_path.get_filename()
411
self.__do_put(source_path, remote_filename)
387
412
388
def move(self, source_path, remote_filename = None):
413
@retry(fatal=True)
414
def move(self, source_path, remote_filename=None):
389
415
"""
390
416
Move source_path (Path object) to remote_filename (string)
391
417
392
418
Same as put(), but unlinks source_path in the process. This allows the
393
419
local backend to do this more efficiently using rename.
394
420
"""
395
self.put(source_path, remote_filename)
396
source_path.delete()
421
if not remote_filename:
422
remote_filename = source_path.get_filename()
423
if not hasattr(self, '_move') or not self._move(source_path, remote_filename):
424
self.__do_put(source_path, remote_filename)
425
source_path.delete()
397
426
427
@retry(fatal=True)
398
428
def get(self, remote_filename, local_path):
399
429
"""Retrieve remote_filename and place in local_path"""
400
raise NotImplementedError()
430
if hasattr(self, '_get'):
431
self._get(remote_filename, local_path)
432
if not local_path.exists():
433
raise BackendException(_("File %s not found locally after get "
434
"from backend") % util.ufn(local_path.name))
435
local_path.setdata()
436
else:
437
raise NotImplementedError()
401
438
439
@retry(fatal=True)
402
440
def list(self):
403
441
"""
404
442
Return list of filenames (byte strings) present in backend
409
447
# There shouldn't be any encoding errors for files we care
410
448
# about, since duplicity filenames are ascii. But user files
411
449
# may be in the same directory. So just replace characters.
412
return filename.encode(sys.getfilesystemencoding(), 'replace')
450
# We don't know what encoding the remote backend may have given
451
# us, but utf8 is a pretty good guess.
452
return filename.encode('utf8', 'replace')
413
453
else:
414
454
return filename
415
455
424
464
"""
425
465
Delete each filename in filename_list, in order if possible.
426
466
"""
427
raise NotImplementedError()
467
assert type(filename_list) is not types.StringType
468
if hasattr(self, '_delete_list'):
469
self._do_delete_list(filename_list)
470
elif hasattr(self, '_delete'):
471
for filename in filename_list:
472
self._do_delete(filename)
473
else:
474
raise NotImplementedError()
475
476
@retry(fatal=False)
477
def _do_delete_list(self, filename_list):
478
self._delete_list(filename_list)
479
480
@retry(fatal=False)
481
def _do_delete(self, filename):
482
self._delete(filename)
428
483
429
484
# Should never cause FatalError.
430
485
# Returns a dictionary of dictionaries. The outer dictionary maps
435
490
# if None, error querying file
436
491
#
437
492
# Returned dictionary is guaranteed to contain a metadata dictionary for
438
# each filename, but not all metadata are guaranteed to be present.
439
def query_info(self, filename_list, raise_errors=True):
493
# each filename, and all metadata are guaranteed to be present.
494
def query_info(self, filename_list):
440
495
"""
441
496
Return metadata about each filename in filename_list
442
497
"""
443
498
info = {}
444
if hasattr(self, '_query_list_info'):
445
info = self._query_list_info(filename_list)
446
elif hasattr(self, '_query_file_info'):
499
if hasattr(self, '_query_list'):
500
info = self._do_query_list(filename_list)
501
elif hasattr(self, '_query'):
447
502
for filename in filename_list:
448
info[filename] = self._query_file_info(filename)
503
info[filename] = self._do_query(filename)
449
504
450
505
# Fill out any missing entries (may happen if backend has no support
451
506
# or its query_list support is lazy)
452
507
for filename in filename_list:
453
if filename not in info:
508
if filename not in info or info[filename] is None:
454
509
info[filename] = {}
455
456
return info
510
for metadata in ['size']:
511
info[filename].setdefault(metadata, None)
512
513
return info
514
515
@retry(fatal=False)
516
def _do_query_list(self, filename_list):
517
info = self._query_list(filename_list)
518
if info is None:
519
info = {}
520
return info
521
522
@retry(fatal=False)
523
def _do_query(self, filename):
524
return self._query(filename)
525
526
def close(self):
527
"""
528
Close the backend, releasing any resources held and
529
invalidating any file objects obtained from the backend.
530
"""
531
if hasattr(self, '_close'):
532
self._close()
457
533
458
534
""" use getpass by default, inherited backends may overwrite this behaviour """
459
535
use_getpass = True
493
569
else:
494
570
return commandline
495
571
496
"""
497
DEPRECATED:
498
run_command(_persist) - legacy wrappers for subprocess_popen(_persist)
499
"""
500
def run_command(self, commandline):
501
return self.subprocess_popen(commandline)
502
def run_command_persist(self, commandline):
503
return self.subprocess_popen_persist(commandline)
504
505
"""
506
DEPRECATED:
507
popen(_persist) - legacy wrappers for subprocess_popen(_persist)
508
"""
509
def popen(self, commandline):
510
result, stdout, stderr = self.subprocess_popen(commandline)
511
return stdout
512
def popen_persist(self, commandline):
513
result, stdout, stderr = self.subprocess_popen_persist(commandline)
514
return stdout
515
516
def _subprocess_popen(self, commandline):
572
def __subprocess_popen(self, commandline):
517
573
"""
518
574
For internal use.
519
575
Execute the given command line, interpreted as a shell command.
534
590
"""
535
591
private = self.munge_password(commandline)
536
592
log.Info(_("Reading results of '%s'") % private)
537
result, stdout, stderr = self._subprocess_popen(commandline)
593
result, stdout, stderr = self.__subprocess_popen(commandline)
538
594
if result != 0:
539
595
raise BackendException("Error running '%s'" % private)
540
596
return result, stdout, stderr
558
614
if n > 1:
559
615
time.sleep(30)
560
616
log.Info(_("Reading results of '%s'") % private)
561
result, stdout, stderr = self._subprocess_popen(commandline)
617
result, stdout, stderr = self.__subprocess_popen(commandline)
562
618
if result == 0:
563
619
return result, stdout, stderr
564
620
645
701
fout = self.get_fileobj_write(filename, parseresults)
646
702
fout.write(buffer)
647
703
assert not fout.close()
648
649
def close(self):
650
"""
651
Close the backend, releasing any resources held and
652
invalidating any file objects obtained from the backend.
653
"""
654
pass
Loggerhead is a web-based interface for Breezy
Version: 2.0.1