1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
#
# Copyright 2002 Ben Escoto <ben@emerose.org>
# Copyright 2007 Kenneth Loafman <kenneth@loafman.com>
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
Miscellaneous utilities.
"""
from future_builtins import map
import errno
import os
import sys
import string
import traceback
from lockfile import FileLock, UnlockError
from duplicity import tarfile
import duplicity.globals as globals
import duplicity.log as log
def exception_traceback(limit = 50):
"""
@return A string representation in typical Python format of the
currently active/raised exception.
"""
type, value, tb = sys.exc_info()
lines = traceback.format_tb(tb, limit)
lines.extend(traceback.format_exception_only(type, value))
msg = "Traceback (innermost last):\n"
msg = msg + "%-20s %s" % (string.join(lines[:-1], ""), lines[-1])
return uexc(msg)
def escape(string):
"Convert a (bytes) filename to a format suitable for logging (quoted utf8)"
string = ufn(string).encode('unicode-escape', 'replace')
return u"'%s'" % string.decode('utf8', 'replace')
def ufn(filename):
"Convert a (bytes) filename to unicode for printing"
assert not isinstance(filename, unicode)
return filename.decode(sys.getfilesystemencoding(), 'replace')
def uindex(index):
"Convert an index (a tuple of path parts) to unicode for printing"
if index:
return os.path.join(*map(ufn, index))
else:
return u'.'
def uexc(e):
# Exceptions in duplicity often have path names in them, which if they are
# non-ascii will cause a UnicodeDecodeError when implicitly decoding to
# unicode. So we decode manually, using the filesystem encoding.
# 99.99% of the time, this will be a fine encoding to use.
return ufn(str(e))
def maybe_ignore_errors(fn):
"""
Execute fn. If the global configuration setting ignore_errors is
set to True, catch errors and log them but do continue (and return
None).
@param fn: A callable.
@return Whatever fn returns when called, or None if it failed and ignore_errors is true.
"""
try:
return fn()
except Exception as e:
if globals.ignore_errors:
log.Warn(_("IGNORED_ERROR: Warning: ignoring error as requested: %s: %s")
% (e.__class__.__name__, uexc(e)))
return None
else:
raise
class BlackHoleList(list):
def append(self, x):
pass
class FakeTarFile:
debug = 0
def __iter__(self):
return iter([])
def close(self):
pass
def make_tarfile(mode, fp):
# We often use 'empty' tarfiles for signatures that haven't been filled out
# yet. So we want to ignore ReadError exceptions, which are used to signal
# this.
try:
tf = tarfile.TarFile("arbitrary", mode, fp)
# Now we cause TarFile to not cache TarInfo objects. It would end up
# consuming a lot of memory over the lifetime of our long-lasting
# signature files otherwise.
tf.members = BlackHoleList()
return tf
except tarfile.ReadError:
return FakeTarFile()
def get_tarinfo_name(ti):
# Python versions before 2.6 ensure that directories end with /, but 2.6
# and later ensure they they *don't* have /. ::shrug:: Internally, we
# continue to use pre-2.6 method.
if ti.isdir() and not ti.name.endswith("/"):
return ti.name + "/"
else:
return ti.name
def ignore_missing(fn, filename):
"""
Execute fn on filename. Ignore ENOENT errors, otherwise raise exception.
@param fn: callable
@param filename: string
"""
try:
fn(filename)
except OSError as ex:
if ex.errno == errno.ENOENT:
pass
else:
raise
def release_lockfile():
if globals.lockfile and globals.lockfile.is_locked():
log.Debug(_("Releasing lockfile %s") % globals.lockfile )
try:
globals.lockfile.release()
except UnlockError:
pass
def copyfileobj(infp, outfp, byte_count = -1):
"""Copy byte_count bytes from infp to outfp, or all if byte_count < 0
Returns the number of bytes actually written (may be less than
byte_count if find eof. Does not close either fileobj.
"""
blocksize = 64 * 1024
bytes_written = 0
if byte_count < 0:
while 1:
buf = infp.read(blocksize)
if not buf:
break
bytes_written += len(buf)
outfp.write(buf)
else:
while bytes_written + blocksize <= byte_count:
buf = infp.read(blocksize)
if not buf:
break
bytes_written += len(buf)
outfp.write(buf)
buf = infp.read(byte_count - bytes_written)
bytes_written += len(buf)
outfp.write(buf)
return bytes_written
|