~duplicity-team/duplicity/0.8-series : revision 1318

37

from duplicity.path import * # @UnusedWildImport

38

from duplicity.lazy import * # @UnusedWildImport

39

40

"""Functions for patching of directories"""

40

u"""Functions for patching of directories"""

41

42

43

class PatchDirException(Exception):

45

46

47

def Patch(base_path, difftar_fileobj):

48

"""Patch given base_path and file object containing delta"""

49

diff_tarfile = tarfile.TarFile("arbitrary", "r", difftar_fileobj)

48

u"""Patch given base_path and file object containing delta"""

49

diff_tarfile = tarfile.TarFile(u"arbitrary", u"r", difftar_fileobj)

50

patch_diff_tarfile(base_path, diff_tarfile)

51

assert not difftar_fileobj.close()

52

53

54

def Patch_from_iter(base_path, fileobj_iter, restrict_index=()):

55

"""Patch given base_path and iterator of delta file objects"""

55

u"""Patch given base_path and iterator of delta file objects"""

56

diff_tarfile = TarFile_FromFileobjs(fileobj_iter)

57

patch_diff_tarfile(base_path, diff_tarfile, restrict_index)

58

59

60

def patch_diff_tarfile(base_path, diff_tarfile, restrict_index=()):

61

"""Patch given Path object using delta tarfile (as in tarfile.TarFile)

61

u"""Patch given Path object using delta tarfile (as in tarfile.TarFile)

62

63

If restrict_index is set, ignore any deltas in diff_tarfile that

64

don't start with restrict_index.

77

ITR = IterTreeReducer(PathPatcher, [base_path])

78

for basis_path, diff_ropath in collated:

79

if basis_path:

80

log.Info(_("Patching %s") % (util.fsdecode(basis_path.get_relative_path())),

80

log.Info(_(u"Patching %s") % (util.fsdecode(basis_path.get_relative_path())),

81

log.InfoCode.patch_file_patching,

82

util.escape(basis_path.get_relative_path()))

83

ITR(basis_path.index, basis_path, diff_ropath)

84

else:

85

log.Info(_("Patching %s") % (util.fsdecode(diff_ropath.get_relative_path())),

85

log.Info(_(u"Patching %s") % (util.fsdecode(diff_ropath.get_relative_path())),

86

log.InfoCode.patch_file_patching,

87

util.escape(diff_ropath.get_relative_path()))

88

ITR(diff_ropath.index, basis_path, diff_ropath)

96

97

98

def filter_path_iter(path_iter, index):

99

"""Rewrite path elements of path_iter so they start with index

99

u"""Rewrite path elements of path_iter so they start with index

100

101

Discard any that doesn't start with index, and remove the index

102

prefix from the rest.

111

112

113

def difftar2path_iter(diff_tarfile):

114

"""Turn file-like difftarobj into iterator of ROPaths"""

114

u"""Turn file-like difftarobj into iterator of ROPaths"""

115

tar_iter = iter(diff_tarfile)

116

multivol_fileobj = None

117

132

ropath = ROPath(index)

133

ropath.init_from_tarinfo(tarinfo_list[0])

134

ropath.difftype = difftype

135

if difftype == "deleted":

135

if difftype == u"deleted":

136

ropath.type = None

137

elif ropath.isreg():

138

if multivol:

148

149

150

def get_index_from_tarinfo(tarinfo):

151

"""Return (index, difftype, multivol) pair from tarinfo object"""

152

for prefix in ["snapshot/", "diff/", "deleted/",

153

"multivol_diff/", "multivol_snapshot/"]:

151

u"""Return (index, difftype, multivol) pair from tarinfo object"""

152

for prefix in [u"snapshot/", u"diff/", u"deleted/",

153

u"multivol_diff/", u"multivol_snapshot/"]:

154

tiname = util.get_tarinfo_name(tarinfo)

155

if tiname.startswith(prefix):

156

name = tiname[len(prefix):] # strip prefix

157

if prefix.startswith("multivol"):

158

if prefix == "multivol_diff/":

159

difftype = "diff"

157

if prefix.startswith(u"multivol"):

158

if prefix == u"multivol_diff/":

159

difftype = u"diff"

160

else:

161

difftype = "snapshot"

161

difftype = u"snapshot"

162

multivol = 1

163

name, num_subs = \

164

re.subn("(?s)^multivol_(diff|snapshot)/?(.*)/[0-9]+$",

165

"\\2", tiname)

164

re.subn(u"(?s)^multivol_(diff|snapshot)/?(.*)/[0-9]+$",

165

u"\\2", tiname)

166

if num_subs != 1:

167

raise PatchDirException(u"Unrecognized diff entry %s" %

168

util.fsdecode(tiname))

169

else:

170

difftype = prefix[:-1] # strip trailing /

171

name = tiname[len(prefix):]

172

if name.endswith("/"):

172

if name.endswith(u"/"):

173

name = name[:-1] # strip trailing /'s

174

multivol = 0

175

break

176

else:

177

raise PatchDirException(u"Unrecognized diff entry %s" %

178

util.fsdecode(tiname))

179

if name == "." or name == "":

179

if name == u"." or name == u"":

180

index = ()

181

else:

182

index = tuple(name.split("/"))

183

if '..' in index:

182

index = tuple(name.split(u"/"))

183

if u'..' in index:

184

raise PatchDirException(u"Tar entry %s contains '..'. Security "

185

"violation" % util.fsdecode(tiname))

185

u"violation" % util.fsdecode(tiname))

186

return (index, difftype, multivol)

187

188

189

class Multivol_Filelike:

190

"""Emulate a file like object from multivols

190

u"""Emulate a file like object from multivols

191

192

Maintains a buffer about the size of a volume. When it is read()

193

to the end, pull in more volumes as desired.

194

195

"""

196

def __init__(self, tf, tar_iter, tarinfo_list, index):

197

"""Initializer. tf is TarFile obj, tarinfo is first tarinfo"""

197

u"""Initializer. tf is TarFile obj, tarinfo is first tarinfo"""

198

self.tf, self.tar_iter = tf, tar_iter

199

self.tarinfo_list = tarinfo_list # must store as list for write access

200

self.index = index

201

self.buffer = ""

201

self.buffer = u""

202

self.at_end = 0

203

204

def read(self, length=-1):

205

"""Read length bytes from file"""

205

u"""Read length bytes from file"""

206

if length < 0:

207

while self.addtobuffer():

208

pass

218

return result

219

220

def addtobuffer(self):

221

"""Add next chunk to buffer"""

221

u"""Add next chunk to buffer"""

222

if self.at_end:

223

return None

224

index, difftype, multivol = get_index_from_tarinfo( # @UnusedVariable

242

return 1

243

244

def close(self):

245

"""If not at end, read remaining data"""

245

u"""If not at end, read remaining data"""

246

if not self.at_end:

247

while 1:

248

self.buffer = ""

248

self.buffer = u""

249

if not self.addtobuffer():

250

break

251

self.at_end = 1

252

253

254

class PathPatcher(ITRBranch):

255

"""Used by DirPatch, process the given basis and diff"""

255

u"""Used by DirPatch, process the given basis and diff"""

256

def __init__(self, base_path):

257

"""Set base_path, Path of root of tree"""

257

u"""Set base_path, Path of root of tree"""

258

self.base_path = base_path

259

self.dir_diff_ropath = None

260

261

def start_process(self, index, basis_path, diff_ropath):

262

"""Start processing when diff_ropath is a directory"""

262

u"""Start processing when diff_ropath is a directory"""

263

if not (diff_ropath and diff_ropath.isdir()):

264

assert index == (), util.uindex(index) # should only happen for first elem

265

self.fast_process(index, basis_path, diff_ropath)

276

self.dir_diff_ropath = diff_ropath

277

278

def end_process(self):

279

"""Copy directory permissions when leaving tree"""

279

u"""Copy directory permissions when leaving tree"""

280

if self.dir_diff_ropath:

281

self.dir_diff_ropath.copy_attribs(self.dir_basis_path)

282

283

def can_fast_process(self, index, basis_path, diff_ropath):

284

"""No need to recurse if diff_ropath isn't a directory"""

284

u"""No need to recurse if diff_ropath isn't a directory"""

285

return not (diff_ropath and diff_ropath.isdir())

286

287

def fast_process(self, index, basis_path, diff_ropath):

288

"""For use when neither is a directory"""

288

u"""For use when neither is a directory"""

289

if not diff_ropath:

290

return # no change

291

elif not basis_path:

292

if diff_ropath.difftype == "deleted":

292

if diff_ropath.difftype == u"deleted":

293

pass # already deleted

294

else:

295

# just copy snapshot over

296

diff_ropath.copy(self.base_path.new_index(index))

297

elif diff_ropath.difftype == "deleted":

297

elif diff_ropath.difftype == u"deleted":

298

if basis_path.isdir():

299

basis_path.deltree()

300

else:

301

basis_path.delete()

302

elif not basis_path.isreg() or (basis_path.isreg() and diff_ropath.difftype == "snapshot"):

302

elif not basis_path.isreg() or (basis_path.isreg() and diff_ropath.difftype == u"snapshot"):

303

if basis_path.isdir():

304

basis_path.deltree()

305

else:

306

basis_path.delete()

307

diff_ropath.copy(basis_path)

308

else:

309

assert diff_ropath.difftype == "diff", diff_ropath.difftype

309

assert diff_ropath.difftype == u"diff", diff_ropath.difftype

310

basis_path.patch_with_attribs(diff_ropath)

311

312

313

class TarFile_FromFileobjs:

314

"""Like a tarfile.TarFile iterator, but read from multiple fileobjs"""

314

u"""Like a tarfile.TarFile iterator, but read from multiple fileobjs"""

315

def __init__(self, fileobj_iter):

316

"""Make new tarinfo iterator

316

u"""Make new tarinfo iterator

317

318

fileobj_iter should be an iterator of file objects opened for

319

reading. They will be closed at end of reading.

327

return self

328

329

def set_tarfile(self):

330

"""Set tarfile from next file object, or raise StopIteration"""

330

u"""Set tarfile from next file object, or raise StopIteration"""

331

if self.current_fp:

332

assert not self.current_fp.close()

333

self.current_fp = next(self.fileobj_iter)

334

self.tarfile = util.make_tarfile("r", self.current_fp)

334

self.tarfile = util.make_tarfile(u"r", self.current_fp)

335

self.tar_iter = iter(self.tarfile)

336

337

def next(self):

345

return next(self.tar_iter)

346

347

def extractfile(self, tarinfo):

348

"""Return data associated with given tarinfo"""

348

u"""Return data associated with given tarinfo"""

349

return self.tarfile.extractfile(tarinfo)

350

351

352

def collate_iters(iter_list):

353

"""Collate iterators by index

353

u"""Collate iterators by index

354

355

Input is a list of n iterators each of which must iterate elements

356

with an index attribute. The elements must come out in increasing

371

elems = overflow[:]

372

373

def setrorps(overflow, elems):

374

"""Set the overflow and rorps list"""

374

u"""Set the overflow and rorps list"""

375

for i in range(iter_num):

376

if not overflow[i] and elems[i] is None:

377

try:

381

elems[i] = None

382

383

def getleastindex(elems):

384

"""Return the first index in elems, assuming elems isn't empty"""

384

u"""Return the first index in elems, assuming elems isn't empty"""

385

return min(map(lambda elem: elem.index, filter(lambda x: x, elems)))

386

387

def yield_tuples(iter_num, overflow, elems):

403

404

405

class IndexedTuple:

406

"""Like a tuple, but has .index (used previously by collate_iters)"""

406

u"""Like a tuple, but has .index (used previously by collate_iters)"""

407

def __init__(self, index, sequence):

408

self.index = index

409

self.data = tuple(sequence)

412

return len(self.data)

413

414

def __getitem__(self, key):

415

"""This only works for numerical keys (easier this way)"""

415

u"""This only works for numerical keys (easier this way)"""

416

return self.data[key]

417

418

def __lt__(self, other):

448

return None

449

450

def __str__(self):

451

return "(%s).%s" % (", ".join(map(str, self.data)), self.index)

451

return u"(%s).%s" % (u", ".join(map(str, self.data)), self.index)

452

453

454

def normalize_ps(patch_sequence):

455

"""Given an sequence of ROPath deltas, remove blank and unnecessary

455

u"""Given an sequence of ROPath deltas, remove blank and unnecessary

456

457

The sequence is assumed to be in patch order (later patches apply

458

to earlier ones). A patch is unnecessary if a later one doesn't

467

if delta is not None:

468

# skip blank entries

469

result_list.insert(0, delta)

470

if delta.difftype != "diff":

470

if delta.difftype != u"diff":

471

break

472

i -= 1

473

return result_list

474

475

476

def patch_seq2ropath(patch_seq):

477

"""Apply the patches in patch_seq, return single ropath"""

477

u"""Apply the patches in patch_seq, return single ropath"""

478

first = patch_seq[0]

479

assert first.difftype != "diff", "First patch in sequence " \

480

"%s was a diff" % patch_seq

479

assert first.difftype != u"diff", u"First patch in sequence %s was a diff" % patch_seq

481

480

if not first.isreg():

482

481

# No need to bother with data if not regular file

483

assert len(patch_seq) == 1, "Patch sequence isn't regular, but " \

484

"has %d entries" % len(patch_seq)

482

assert len(patch_seq) == 1, u"Patch sequence isn't regular, but " \

483

u"has %d entries" % len(patch_seq)

485

484

return first.get_ropath()

486

485

487

current_file = first.open("rb")

486

current_file = first.open(u"rb")

488

487

489

488

for delta_ropath in patch_seq[1:]:

490

assert delta_ropath.difftype == "diff", delta_ropath.difftype

489

assert delta_ropath.difftype == u"diff", delta_ropath.difftype

491

490

if not isinstance(current_file, file):

492

"""

491

u"""

493

492

librsync insists on a real file object, which we create manually

494

493

by using the duplicity.tempdir to tell us where.

495

494

503

502

tempfp.seek(0)

504

503

current_file = tempfp

505

504

current_file = librsync.PatchedFile(current_file,

506

delta_ropath.open("rb"))

505

delta_ropath.open(u"rb"))

507

506

result = patch_seq[-1].get_ropath()

508

507

result.setfileobj(current_file)

509

508

return result

510

509

511

510

512

511

def integrate_patch_iters(iter_list):

513

"""Combine a list of iterators of ropath patches

512

u"""Combine a list of iterators of ropath patches

514

513

515

514

The iter_list should be sorted in patch order, and the elements in

516

515

each iter_list need to be orderd by index. The output will be an

527

526

yield final_ropath

528

527

except Exception as e:

529

528

filename = normalized[-1].get_ropath().get_relative_path()

530

log.Warn(_("Error '%s' patching %s") %

529

log.Warn(_(u"Error '%s' patching %s") %

531

530

(util.uexc(e), util.fsdecode(filename)),

532

531

log.WarningCode.cannot_process,

533

532

util.escape(filename))

534

533

535

534

536

535

def tarfiles2rop_iter(tarfile_list, restrict_index=()):

537

"""Integrate tarfiles of diffs into single ROPath iter

536

u"""Integrate tarfiles of diffs into single ROPath iter

538

537

539

538

Then filter out all the diffs in that index which don't start with

540

539

the restrict_index.

548

547

549

548

550

549

def Write_ROPaths(base_path, rop_iter):

551

"""Write out ropaths in rop_iter starting at base_path

550

u"""Write out ropaths in rop_iter starting at base_path

552

551

553

552

Returns 1 if something was actually written, 0 otherwise.

554

553

564

563

565

564

566

565

class ROPath_IterWriter(ITRBranch):

567

"""Used in Write_ROPaths above

566

u"""Used in Write_ROPaths above

568

567

569

568

We need to use an ITR because we have to update the

570

569

permissions/times of directories after we write the files in them.

571

570

572

571

"""

573

572

def __init__(self, base_path):

574

"""Set base_path, Path of root of tree"""

573

u"""Set base_path, Path of root of tree"""

575

574

self.base_path = base_path

576

575

self.dir_diff_ropath = None

577

576

self.dir_new_path = None

578

577

579

578

def start_process(self, index, ropath):

580

"""Write ropath. Only handles the directory case"""

579

u"""Write ropath. Only handles the directory case"""

581

580

if not ropath.isdir():

582

581

# Base may not be a directory, but rest should

583

582

assert ropath.index == (), ropath.index

596

595

self.dir_diff_ropath = ropath

597

596

598

597

def end_process(self):

599

"""Update information of a directory when leaving it"""

598

u"""Update information of a directory when leaving it"""

600

599

if self.dir_diff_ropath:

601

600

self.dir_diff_ropath.copy_attribs(self.dir_new_path)

602

601

603

602

def can_fast_process(self, index, ropath):

604

"""Can fast process (no recursion) if ropath isn't a directory"""

605

log.Info(_("Writing %s of type %s") %

603

u"""Can fast process (no recursion) if ropath isn't a directory"""

604

log.Info(_(u"Writing %s of type %s") %

606

605

(util.fsdecode(ropath.get_relative_path()), ropath.type),

607

606

log.InfoCode.patch_file_writing,

608

"%s %s" % (util.escape(ropath.get_relative_path()), ropath.type))

607

u"%s %s" % (util.escape(ropath.get_relative_path()), ropath.type))

609

608

return not ropath.isdir()

610

609

611

610

def fast_process(self, index, ropath):

612

"""Write non-directory ropath to destination"""

611

u"""Write non-directory ropath to destination"""

613

612

if ropath.exists():

614

613

ropath.copy(self.base_path.new_index(index))