« back to all changes in this revision
Viewing changes to duplicity/patchdir.py
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: bescoto
- Date: 2002-10-29 01:49:46 UTC
- Revision ID: vcs-imports@canonical.com-20021029014946-3m4rmm5plom7pl6q
Initial checkin
- files added:
- dist
- duplicity
- testing
added
removed
duplicity/patchdir.py
1
# Copyright 2002 Ben Escoto
2
#
3
# This file is part of duplicity.
4
#
5
# duplicity is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA
8
# 02139, USA; either version 2 of the License, or (at your option) any
9
# later version; incorporated herein by reference.
10
11
"""Functions for patching of directories"""
12
13
from __future__ import generators
14
import re
15
import tarfile, librsync, log, diffdir
16
from path import *
17
from lazy import *
18
19
class PatchDirException(Exception): pass
20
21
def Patch(base_path, difftar_fileobj):
22
"""Patch given base_path and file object containing delta"""
23
diff_tarfile = tarfile.TarFile("arbitrary", "r", difftar_fileobj)
24
patch_diff_tarfile(base_path, diff_tarfile)
25
assert not difftar_fileobj.close()
26
27
def Patch_from_iter(base_path, fileobj_iter, restrict_index = ()):
28
"""Patch given base_path and iterator of delta file objects"""
29
diff_tarfile = TarFile_FromFileobjs(fileobj_iter)
30
patch_diff_tarfile(base_path, diff_tarfile, restrict_index)
31
32
def patch_diff_tarfile(base_path, diff_tarfile, restrict_index = ()):
33
"""Patch given Path object using delta tarfile (as in tarfile.TarFile)
34
35
If restrict_index is set, ignore any deltas in diff_tarfile that
36
don't start with restrict_index.
37
38
"""
39
if base_path.exists(): path_iter = selection.Select(base_path).set_iter()
40
else: path_iter = empty_iter() # probably untarring full backup
41
42
diff_path_iter = difftar2path_iter(diff_tarfile)
43
if restrict_index:
44
diff_path_iter = filter_path_iter(diff_path_iter, restrict_index)
45
collated = diffdir.collate_iters(path_iter, diff_path_iter)
46
47
ITR = IterTreeReducer(PathPatcher, [base_path])
48
for basis_path, diff_ropath in collated:
49
if basis_path:
50
log.Log("Patching %s" % (basis_path.get_relative_path(),), 5)
51
ITR(basis_path.index, basis_path, diff_ropath)
52
else:
53
log.Log("Patching %s" % (diff_ropath.get_relative_path(),), 5)
54
ITR(diff_ropath.index, basis_path, diff_ropath)
55
ITR.Finish()
56
base_path.setdata()
57
58
def empty_iter():
59
if 0: yield 1 # this never happens, but fools into generator treatment
60
61
def filter_path_iter(path_iter, index):
62
"""Rewrite path elements of path_iter so they start with index
63
64
Discard any that doesn't start with index, and remove the index
65
prefix from the rest.
66
67
"""
68
assert isinstance(index, tuple) and index, index
69
l = len(index)
70
for path in path_iter:
71
if path.index[:l] == index:
72
path.index = path.index[l:]
73
yield path
74
75
def difftar2path_iter(diff_tarfile):
76
"""Turn file-like difftarobj into iterator of ROPaths"""
77
prefixes = ["snapshot/", "diff/", "deleted/"]
78
tar_iter = iter(diff_tarfile)
79
multivol_fileobj = None
80
81
# The next tar_info is stored in this one element list so
82
# Multivol_Filelike below can update it. Any StopIterations will
83
# be passed upwards.
84
tarinfo_list = [tar_iter.next()]
85
86
while 1:
87
# This section relevant when a multivol diff is last in tar
88
if not tarinfo_list[0]: raise StopIteration
89
if multivol_fileobj and not multivol_fileobj.at_end:
90
multivol_fileobj.close() # aborting in middle of multivol
91
continue
92
93
index, difftype, multivol = get_index_from_tarinfo(tarinfo_list[0])
94
ropath = ROPath(index)
95
ropath.init_from_tarinfo(tarinfo_list[0])
96
ropath.difftype = difftype
97
if ropath.isreg():
98
if multivol:
99
multivol_fileobj = Multivol_Filelike(diff_tarfile, tar_iter,
100
tarinfo_list, index)
101
ropath.setfileobj(multivol_fileobj)
102
yield ropath
103
continue # Multivol_Filelike will reset tarinfo_list
104
else: ropath.setfileobj(diff_tarfile.extractfile(tarinfo_list[0]))
105
yield ropath
106
tarinfo_list[0] = tar_iter.next()
107
108
def get_index_from_tarinfo(tarinfo):
109
"""Return (index, difftype, multivol) pair from tarinfo object"""
110
for prefix in ["snapshot/", "diff/", "deleted/",
111
"multivol_diff/", "multivol_snapshot/"]:
112
if tarinfo.name.startswith(prefix):
113
name = tarinfo.name[len(prefix):] # strip prefix
114
if prefix.startswith("multivol"):
115
if prefix == "multivol_diff/": difftype = "diff"
116
else: difftype = "snapshot"
117
multivol = 1
118
name, num_subs = \
119
re.subn("^multivol_(diff|snapshot)/(.*)/[0-9]+$",
120
"\\2", tarinfo.name)
121
if num_subs != 1:
122
raise DiffDirException("Unrecognized diff entry %s" %
123
(tarinfo.name,))
124
else:
125
difftype = prefix[:-1] # strip trailing /
126
name = tarinfo.name[len(prefix):]
127
if name.endswith("/"): name = name[:-1] # strip trailing /'s
128
multivol = 0
129
break
130
else: raise DiffDirException("Unrecognized diff entry %s" %
131
(tarinfo.name,))
132
if name == ".": index = ()
133
else: index = tuple(name.split("/"))
134
return (index, difftype, multivol)
135
136
137
class Multivol_Filelike:
138
"""Emulate a file like object from multivols
139
140
Maintains a buffer about the size of a volume. When it is read()
141
to the end, pull in more volumes as desired.
142
143
"""
144
def __init__(self, tf, tar_iter, tarinfo_list, index):
145
"""Initializer. tf is TarFile obj, tarinfo is first tarinfo"""
146
self.tf, self.tar_iter = tf, tar_iter
147
self.tarinfo_list = tarinfo_list # must store as list for write access
148
self.index = index
149
self.buffer = ""
150
self.at_end = 0
151
152
def read(self, length = -1):
153
"""Read length bytes from file"""
154
if length < 0:
155
while self.addtobuffer(): pass
156
real_len = len(self.buffer)
157
else:
158
while len(self.buffer) < length:
159
if not self.addtobuffer(): break
160
real_len = min(len(self.buffer), length)
161
162
result = self.buffer[:real_len]
163
self.buffer = self.buffer[real_len:]
164
return result
165
166
def addtobuffer(self):
167
"""Add next chunk to buffer"""
168
if self.at_end: return None
169
index, difftype, multivol = get_index_from_tarinfo(
170
self.tarinfo_list[0])
171
if not multivol or index != self.index: # we've moved on
172
# the following communicates next tarinfo to difftar2path_iter
173
self.at_end = 1
174
return None
175
176
fp = self.tf.extractfile(self.tarinfo_list[0])
177
self.buffer += fp.read()
178
fp.close()
179
180
try: self.tarinfo_list[0] = self.tar_iter.next()
181
except StopIteration:
182
self.tarinfo_list[0] = None
183
self.at_end = 1
184
return None
185
return 1
186
187
def close(self):
188
"""If not at end, read remaining data"""
189
if not self.at_end:
190
while 1:
191
self.buffer = ""
192
if not self.addtobuffer(): break
193
self.at_end = 1
194
195
196
class PathPatcher(ITRBranch):
197
"""Used by DirPatch, process the given basis and diff"""
198
def __init__(self, base_path):
199
"""Set base_path, Path of root of tree"""
200
self.base_path = base_path
201
self.dir_diff_ropath = None
202
203
def start_process(self, index, basis_path, diff_ropath):
204
"""Start processing when diff_ropath is a directory"""
205
if not (diff_ropath and diff_ropath.isdir()):
206
assert index == () # this should only happen for first elem
207
self.fast_process(index, basis_path, diff_ropath)
208
return
209
210
if not basis_path:
211
basis_path = self.base_path.new_index(index)
212
assert not basis_path.exists()
213
basis_path.mkdir() # Need place for later files to go into
214
elif not basis_path.isdir():
215
basis_path.delete()
216
basis_path.mkdir()
217
self.dir_basis_path = basis_path
218
self.dir_diff_ropath = diff_ropath
219
220
def end_process(self):
221
"""Copy directory permissions when leaving tree"""
222
if self.dir_diff_ropath:
223
self.dir_diff_ropath.copy_attribs(self.dir_basis_path)
224
225
def can_fast_process(self, index, basis_path, diff_ropath):
226
"""No need to recurse if diff_ropath isn't a directory"""
227
return not (diff_ropath and diff_ropath.isdir())
228
229
def fast_process(self, index, basis_path, diff_ropath):
230
"""For use when neither is a directory"""
231
if not diff_ropath: return # no change
232
elif not basis_path:
233
if diff_ropath.difftype == "deleted": pass # already deleted
234
else: # just copy snapshot over
235
diff_ropath.copy(self.base_path.new_index(index))
236
elif diff_ropath.difftype == "deleted":
237
if basis_path.isdir(): basis_path.deltree()
238
else: basis_path.delete()
239
elif not basis_path.isreg():
240
if basis_path.isdir(): basis_path.deltree()
241
else: basis_path.delete()
242
diff_ropath.copy(basis_path)
243
else:
244
assert diff_ropath.difftype == "diff", diff_ropath.difftype
245
basis_path.patch_with_attribs(diff_ropath)
246
247
248
class TarFile_FromFileobjs:
249
"""Like a tarfile.TarFile iterator, but read from multiple fileobjs"""
250
def __init__(self, fileobj_iter):
251
"""Make new tarinfo iterator
252
253
fileobj_iter should be an iterator of file objects opened for
254
reading. They will be closed at end of reading.
255
256
"""
257
self.fileobj_iter = fileobj_iter
258
self.tarfile, self.tar_iter = None, None
259
self.current_fp = None
260
261
def __iter__(self): return self
262
263
def set_tarfile(self):
264
"""Set tarfile from next file object, or raise StopIteration"""
265
if self.current_fp: assert not self.current_fp.close()
266
self.current_fp = self.fileobj_iter.next()
267
self.tarfile = tarfile.TarFile("arbitrary", "r", self.current_fp)
268
self.tar_iter = iter(self.tarfile)
269
270
def next(self):
271
if not self.tarfile: self.set_tarfile()
272
try: return self.tar_iter.next()
273
except StopIteration:
274
assert not self.tarfile.close()
275
self.set_tarfile()
276
return self.tar_iter.next()
277
278
def extractfile(self, tarinfo):
279
"""Return data associated with given tarinfo"""
280
return self.tarfile.extractfile(tarinfo)
281
Loggerhead is a web-based interface for Breezy
Version: 2.0.1