« back to all changes in this revision
Viewing changes to duplicity/statistics.py
- browse files at revision 1318
- compare with another revision
- download diff
- download tarball
- view history from revision 1318
- Committer: kenneth at loafman
- Date: 2018-07-23 16:32:30 UTC
- Revision ID: kenneth@loafman.com-20180723163230-i226wdy5q2zzgfc7
* Fixed unadorned strings to unicode in duplicity/*/*
- Some fixup due to shifting indenataion not matching PEP8.
- Substituted for non-ascii char in jottlibbackend.py comment.
- Some fixup due to shifting indenataion not matching PEP8.
- Substituted for non-ascii char in jottlibbackend.py comment.
- files modified:
- CHANGELOG
added
removed
duplicity/statistics.py
19
19
# along with duplicity; if not, write to the Free Software Foundation,
20
20
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21
21
22
"""Generate and process backup statistics"""
22
u"""Generate and process backup statistics"""
23
23
24
24
from future_builtins import map
25
25
35
35
36
36
37
37
class StatsObj:
38
"""Contains various statistics, provide string conversion functions"""
38
u"""Contains various statistics, provide string conversion functions"""
39
39
# used when quoting files in get_stats_line
40
space_regex = re.compile(" ")
40
space_regex = re.compile(u" ")
41
41
42
stat_file_attrs = ('SourceFiles',
43
'SourceFileSize',
44
'NewFiles',
45
'NewFileSize',
46
'DeletedFiles',
47
'ChangedFiles',
48
'ChangedFileSize',
49
'ChangedDeltaSize',
50
'DeltaEntries',
51
'RawDeltaSize')
52
stat_misc_attrs = ('Errors',
53
'TotalDestinationSizeChange')
54
stat_time_attrs = ('StartTime',
55
'EndTime',
56
'ElapsedTime')
57
stat_attrs = (('Filename',) + stat_time_attrs +
42
stat_file_attrs = (u'SourceFiles',
43
u'SourceFileSize',
44
u'NewFiles',
45
u'NewFileSize',
46
u'DeletedFiles',
47
u'ChangedFiles',
48
u'ChangedFileSize',
49
u'ChangedDeltaSize',
50
u'DeltaEntries',
51
u'RawDeltaSize')
52
stat_misc_attrs = (u'Errors',
53
u'TotalDestinationSizeChange')
54
stat_time_attrs = (u'StartTime',
55
u'EndTime',
56
u'ElapsedTime')
57
stat_attrs = ((u'Filename',) + stat_time_attrs +
58
58
stat_misc_attrs + stat_file_attrs)
59
59
60
60
# Below, the second value in each pair is true iff the value
61
61
# indicates a number of bytes
62
stat_file_pairs = (('SourceFiles', False),
63
('SourceFileSize', True),
64
('NewFiles', False),
65
('NewFileSize', True),
66
('DeletedFiles', False),
67
('ChangedFiles', False),
68
('ChangedFileSize', True),
69
('ChangedDeltaSize', True),
70
('DeltaEntries', False),
71
('RawDeltaSize', True))
62
stat_file_pairs = ((u'SourceFiles', False),
63
(u'SourceFileSize', True),
64
(u'NewFiles', False),
65
(u'NewFileSize', True),
66
(u'DeletedFiles', False),
67
(u'ChangedFiles', False),
68
(u'ChangedFileSize', True),
69
(u'ChangedDeltaSize', True),
70
(u'DeltaEntries', False),
71
(u'RawDeltaSize', True))
72
72
73
73
# This is used in get_byte_summary_string below
74
byte_abbrev_list = ((1024 * 1024 * 1024 * 1024, "TB"),
75
(1024 * 1024 * 1024, "GB"),
76
(1024 * 1024, "MB"),
77
(1024, "KB"))
74
byte_abbrev_list = ((1024 * 1024 * 1024 * 1024, u"TB"),
75
(1024 * 1024 * 1024, u"GB"),
76
(1024 * 1024, u"MB"),
77
(1024, u"KB"))
78
78
79
79
def __init__(self):
80
"""Set attributes to None"""
80
u"""Set attributes to None"""
81
81
for attr in self.stat_attrs:
82
82
self.__dict__[attr] = None
83
83
84
84
def get_stat(self, attribute):
85
"""Get a statistic"""
85
u"""Get a statistic"""
86
86
return self.__dict__[attribute]
87
87
88
88
def set_stat(self, attr, value):
89
"""Set attribute to given value"""
89
u"""Set attribute to given value"""
90
90
self.__dict__[attr] = value
91
91
92
92
def increment_stat(self, attr):
93
"""Add 1 to value of attribute"""
93
u"""Add 1 to value of attribute"""
94
94
self.__dict__[attr] += 1
95
95
96
96
def get_total_dest_size_change(self):
97
"""Return total destination size change
97
u"""Return total destination size change
98
98
99
99
This represents the total increase in the size of the
100
100
duplicity destination directory, or None if not available.
103
103
return 0 # this needs to be re-done for duplicity
104
104
105
105
def get_stats_line(self, index, use_repr=1):
106
"""Return one line abbreviated version of full stats string"""
106
u"""Return one line abbreviated version of full stats string"""
107
107
file_attrs = [str(self.get_stat(a)) for a in self.stat_file_attrs]
108
108
if not index:
109
filename = "."
109
filename = u"."
110
110
else:
111
111
filename = os.path.join(*index)
112
112
if use_repr:
113
113
# use repr to quote newlines in relative filename, then
114
114
# take of leading and trailing quote and quote spaces.
115
filename = self.space_regex.sub("\\x20", repr(filename)[1:-1])
116
return " ".join([filename, ] + file_attrs)
115
filename = self.space_regex.sub(u"\\x20", repr(filename)[1:-1])
116
return u" ".join([filename, ] + file_attrs)
117
117
118
118
def set_stats_from_line(self, line):
119
"""Set statistics from given line"""
119
u"""Set statistics from given line"""
120
120
def error():
121
raise StatsException("Bad line '%s'" % line)
122
if line[-1] == "\n":
121
raise StatsException(u"Bad line '%s'" % line)
122
if line[-1] == u"\n":
123
123
line = line[:-1]
124
lineparts = line.split(" ")
124
lineparts = line.split(u" ")
125
125
if len(lineparts) < len(self.stat_file_attrs):
126
126
error()
127
127
for attr, val_string in zip(self.stat_file_attrs,
137
137
return self
138
138
139
139
def get_stats_string(self):
140
"""Return extended string printing out statistics"""
141
return "%s%s%s" % (self.get_timestats_string(),
142
self.get_filestats_string(),
143
self.get_miscstats_string())
140
u"""Return extended string printing out statistics"""
141
return u"%s%s%s" % (self.get_timestats_string(),
142
self.get_filestats_string(),
143
self.get_miscstats_string())
144
144
145
145
def get_timestats_string(self):
146
"""Return portion of statistics string dealing with time"""
146
u"""Return portion of statistics string dealing with time"""
147
147
timelist = []
148
148
if self.StartTime is not None:
149
timelist.append("StartTime %.2f (%s)\n" %
149
timelist.append(u"StartTime %.2f (%s)\n" %
150
150
(self.StartTime, dup_time.timetopretty(self.StartTime)))
151
151
if self.EndTime is not None:
152
timelist.append("EndTime %.2f (%s)\n" %
152
timelist.append(u"EndTime %.2f (%s)\n" %
153
153
(self.EndTime, dup_time.timetopretty(self.EndTime)))
154
154
if self.ElapsedTime or (self.StartTime is not None and
155
155
self.EndTime is not None):
156
156
if self.ElapsedTime is None:
157
157
self.ElapsedTime = self.EndTime - self.StartTime
158
timelist.append("ElapsedTime %.2f (%s)\n" %
158
timelist.append(u"ElapsedTime %.2f (%s)\n" %
159
159
(self.ElapsedTime, dup_time.inttopretty(self.ElapsedTime)))
160
return "".join(timelist)
160
return u"".join(timelist)
161
161
162
162
def get_filestats_string(self):
163
"""Return portion of statistics string about files and bytes"""
163
u"""Return portion of statistics string about files and bytes"""
164
164
def fileline(stat_file_pair):
165
"""Return zero or one line of the string"""
165
u"""Return zero or one line of the string"""
166
166
attr, in_bytes = stat_file_pair
167
167
val = self.get_stat(attr)
168
168
if val is None:
169
return ""
169
return u""
170
170
if in_bytes:
171
return "%s %s (%s)\n" % (attr, val,
172
self.get_byte_summary_string(val))
171
return u"%s %s (%s)\n" % (attr, val,
172
self.get_byte_summary_string(val))
173
173
else:
174
return "%s %s\n" % (attr, val)
174
return u"%s %s\n" % (attr, val)
175
175
176
return "".join(map(fileline, self.stat_file_pairs))
176
return u"".join(map(fileline, self.stat_file_pairs))
177
177
178
178
def get_miscstats_string(self):
179
"""Return portion of extended stat string about misc attributes"""
180
misc_string = ""
179
u"""Return portion of extended stat string about misc attributes"""
180
misc_string = u""
181
181
tdsc = self.TotalDestinationSizeChange
182
182
if tdsc is not None:
183
misc_string += ("TotalDestinationSizeChange %s (%s)\n" %
183
misc_string += (u"TotalDestinationSizeChange %s (%s)\n" %
184
184
(tdsc, self.get_byte_summary_string(tdsc)))
185
185
if self.Errors is not None:
186
misc_string += "Errors %d\n" % self.Errors
186
misc_string += u"Errors %d\n" % self.Errors
187
187
return misc_string
188
188
189
189
def get_byte_summary_string(self, byte_count):
190
"""Turn byte count into human readable string like "7.23GB" """
190
u"""Turn byte count into human readable string like "7.23GB" """
191
191
if byte_count < 0:
192
sign = "-"
192
sign = u"-"
193
193
byte_count = -byte_count
194
194
else:
195
sign = ""
195
sign = u""
196
196
197
197
for abbrev_bytes, abbrev_string in self.byte_abbrev_list:
198
198
if byte_count >= abbrev_bytes:
204
204
precision = 1
205
205
else:
206
206
precision = 2
207
return "%s%%.%df %s" % (sign, precision, abbrev_string) \
207
return u"%s%%.%df %s" % (sign, precision, abbrev_string) \
208
208
% (abbrev_count,)
209
209
byte_count = round(byte_count)
210
210
if byte_count == 1:
211
return sign + "1 byte"
211
return sign + u"1 byte"
212
212
else:
213
return "%s%d bytes" % (sign, byte_count)
213
return u"%s%d bytes" % (sign, byte_count)
214
214
215
215
def get_stats_logstring(self, title):
216
"""Like get_stats_string, but add header and footer"""
217
header = "--------------[ %s ]--------------" % title
218
footer = "-" * len(header)
219
return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)
216
u"""Like get_stats_string, but add header and footer"""
217
header = u"--------------[ %s ]--------------" % title
218
footer = u"-" * len(header)
219
return u"%s\n%s%s\n" % (header, self.get_stats_string(), footer)
220
220
221
221
def set_stats_from_string(self, s):
222
"""Initialize attributes from string, return self for convenience"""
222
u"""Initialize attributes from string, return self for convenience"""
223
223
def error(line):
224
raise StatsException("Bad line '%s'" % line)
224
raise StatsException(u"Bad line '%s'" % line)
225
225
226
for line in s.split("\n"):
226
for line in s.split(u"\n"):
227
227
if not line:
228
228
continue
229
229
line_parts = line.split()
247
247
return self
248
248
249
249
def write_stats_to_path(self, path):
250
"""Write statistics string to given path"""
251
fin = path.open("w")
250
u"""Write statistics string to given path"""
251
fin = path.open(u"w")
252
252
fin.write(self.get_stats_string())
253
253
assert not fin.close()
254
254
255
255
def read_stats_from_path(self, path):
256
"""Set statistics from path, return self for convenience"""
257
fp = path.open("r")
256
u"""Set statistics from path, return self for convenience"""
257
fp = path.open(u"r")
258
258
self.set_stats_from_string(fp.read())
259
259
assert not fp.close()
260
260
return self
261
261
262
262
def stats_equal(self, s):
263
"""Return true if s has same statistics as self"""
263
u"""Return true if s has same statistics as self"""
264
264
assert isinstance(s, StatsObj)
265
265
for attr in self.stat_file_attrs:
266
266
if self.get_stat(attr) != s.get_stat(attr):
268
268
return 1
269
269
270
270
def set_to_average(self, statobj_list):
271
"""Set self's attributes to average of those in statobj_list"""
271
u"""Set self's attributes to average of those in statobj_list"""
272
272
for attr in self.stat_attrs:
273
273
self.set_stat(attr, 0)
274
274
for statobj in statobj_list:
290
290
return self
291
291
292
292
def get_statsobj_copy(self):
293
"""Return new StatsObj object with same stats as self"""
293
u"""Return new StatsObj object with same stats as self"""
294
294
s = StatsObj()
295
295
for attr in self.stat_attrs:
296
296
s.set_stat(attr, self.get_stat(attr))
298
298
299
299
300
300
class StatsDeltaProcess(StatsObj):
301
"""Keep track of statistics during DirDelta process"""
301
u"""Keep track of statistics during DirDelta process"""
302
302
def __init__(self):
303
"""StatsDeltaProcess initializer - zero file attributes"""
303
u"""StatsDeltaProcess initializer - zero file attributes"""
304
304
StatsObj.__init__(self)
305
305
for attr in StatsObj.stat_file_attrs:
306
306
self.__dict__[attr] = 0
309
309
self.files_changed = []
310
310
311
311
def add_new_file(self, path):
312
"""Add stats of new file path to statistics"""
312
u"""Add stats of new file path to statistics"""
313
313
filesize = path.getsize()
314
314
self.SourceFiles += 1
315
315
# SourceFileSize is added-to incrementally as read
316
316
self.NewFiles += 1
317
317
self.NewFileSize += filesize
318
318
self.DeltaEntries += 1
319
self.add_delta_entries_file(path, 'new')
319
self.add_delta_entries_file(path, u'new')
320
320
321
321
def add_changed_file(self, path):
322
"""Add stats of file that has changed since last backup"""
322
u"""Add stats of file that has changed since last backup"""
323
323
filesize = path.getsize()
324
324
self.SourceFiles += 1
325
325
# SourceFileSize is added-to incrementally as read
326
326
self.ChangedFiles += 1
327
327
self.ChangedFileSize += filesize
328
328
self.DeltaEntries += 1
329
self.add_delta_entries_file(path, 'changed')
329
self.add_delta_entries_file(path, u'changed')
330
330
331
331
def add_deleted_file(self, path):
332
"""Add stats of file no longer in source directory"""
332
u"""Add stats of file no longer in source directory"""
333
333
self.DeletedFiles += 1 # can't add size since not available
334
334
self.DeltaEntries += 1
335
self.add_delta_entries_file(path, 'deleted')
335
self.add_delta_entries_file(path, u'deleted')
336
336
337
337
def add_unchanged_file(self, path):
338
"""Add stats of file that hasn't changed since last backup"""
338
u"""Add stats of file that hasn't changed since last backup"""
339
339
filesize = path.getsize()
340
340
self.SourceFiles += 1
341
341
self.SourceFileSize += filesize
342
342
343
343
def close(self):
344
"""End collection of data, set EndTime"""
344
u"""End collection of data, set EndTime"""
345
345
self.EndTime = time.time()
346
346
347
347
def add_delta_entries_file(self, path, action_type):
Loggerhead is a web-based interface for Breezy
Version: 2.0.1