~duplicity-team/duplicity/0.8-series : revision 1318

19

# along with duplicity; if not, write to the Free Software Foundation,

20

# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

21

22

"""

22

u"""

23

Provides a common interface to all backends and certain sevices

24

intended to be used by the backends themselves.

25

"""

78

79

80

def import_backends():

81

"""

81

u"""

82

Import files in the duplicity/backends directory where

83

the filename ends in 'backend.py' and ignore the rest.

84

86

@return: void

87

"""

88

path = duplicity.backends.__path__[0]

89

assert path.endswith("duplicity/backends"), duplicity.backends.__path__

89

assert path.endswith(u"duplicity/backends"), duplicity.backends.__path__

90

91

files = os.listdir(path)

92

files.sort()

93

for fn in files:

94

if fn.endswith("backend.py"):

94

if fn.endswith(u"backend.py"):

95

fn = fn[:-3]

96

imp = "duplicity.backends.%s" % (fn,)

96

imp = u"duplicity.backends.%s" % (fn,)

97

try:

98

__import__(imp)

99

res = "Succeeded"

99

res = u"Succeeded"

100

except Exception:

101

res = "Failed: " + str(sys.exc_info()[1])

102

log.Log(_("Import of %s %s") % (imp, res), log.INFO)

101

res = u"Failed: " + str(sys.exc_info()[1])

102

log.Log(_(u"Import of %s %s") % (imp, res), log.INFO)

103

else:

104

continue

105

106

107

def register_backend(scheme, backend_factory):

108

"""

108

u"""

109

Register a given backend factory responsible for URL:s with the

110

given scheme.

111

120

"""

121

global _backends

122

123

assert callable(backend_factory), "backend factory must be callable"

123

assert callable(backend_factory), u"backend factory must be callable"

124

125

if scheme in _backends:

126

raise ConflictingScheme("the scheme %s already has a backend "

127

"associated with it"

128

"" % (scheme,))

126

raise ConflictingScheme(u"the scheme %s already has a backend "

127

u"associated with it"

128

u"" % (scheme,))

129

130

_backends[scheme] = backend_factory

131

132

133

def register_backend_prefix(scheme, backend_factory):

134

"""

134

u"""

135

Register a given backend factory responsible for URL:s with the

136

given scheme prefix.

137

146

"""

147

global _backend_prefixes

148

149

assert callable(backend_factory), "backend factory must be callable"

149

assert callable(backend_factory), u"backend factory must be callable"

150

151

if scheme in _backend_prefixes:

152

raise ConflictingScheme("the prefix %s already has a backend "

153

"associated with it"

154

"" % (scheme,))

152

raise ConflictingScheme(u"the prefix %s already has a backend "

153

u"associated with it"

154

u"" % (scheme,))

155

156

_backend_prefixes[scheme] = backend_factory

157

158

159

def strip_prefix(url_string, prefix_scheme):

160

"""

160

u"""

161

strip the prefix from a string e.g. par2+ftp://... -> ftp://...

162

"""

163

return re.sub(r'(?i)^' + re.escape(prefix_scheme) + r'\+', r'', url_string)

164

165

166

def is_backend_url(url_string):

167

"""

167

u"""

168

@return Whether the given string looks like a backend URL.

169

"""

170

pu = ParsedUrl(url_string)

177

178

179

def get_backend_object(url_string):

180

"""

180

u"""

181

Find the right backend class instance for the given URL, or return None

182

if the given string looks like a local path rather than a URL.

183

189

global _backends, _backend_prefixes

190

191

pu = ParsedUrl(url_string)

192

assert pu.scheme, "should be a backend url according to is_backend_url"

192

assert pu.scheme, u"should be a backend url according to is_backend_url"

193

194

factory = None

195

196

for prefix in _backend_prefixes:

197

if url_string.startswith(prefix + '+'):

197

if url_string.startswith(prefix + u'+'):

198

factory = _backend_prefixes[prefix]

199

pu = ParsedUrl(strip_prefix(url_string, prefix))

200

break

208

try:

209

return factory(pu)

210

except ImportError:

211

raise BackendException(_("Could not initialize backend: %s") % str(sys.exc_info()[1]))

211

raise BackendException(_(u"Could not initialize backend: %s") % str(sys.exc_info()[1]))

212

213

214

def get_backend(url_string):

215

"""

215

u"""

216

Instantiate a backend suitable for the given URL, or return None

217

if the given string looks like a local path rather than a URL.

218

219

Raise InvalidBackendURL if the URL is not a valid URL.

220

"""

221

if globals.use_gio:

222

url_string = 'gio+' + url_string

222

url_string = u'gio+' + url_string

223

obj = get_backend_object(url_string)

224

if obj:

225

obj = BackendWrapper(obj)

227

228

229

class ParsedUrl:

230

"""

230

u"""

231

Parse the given URL as a duplicity backend URL.

232

233

Returns the data of a parsed URL with the same names as that of

252

try:

253

pu = urlparse.urlparse(url_string)

254

except Exception:

255

raise InvalidBackendURL("Syntax error in: %s" % url_string)

255

raise InvalidBackendURL(u"Syntax error in: %s" % url_string)

256

257

try:

258

self.scheme = pu.scheme

259

except Exception:

260

raise InvalidBackendURL("Syntax error (scheme) in: %s" % url_string)

260

raise InvalidBackendURL(u"Syntax error (scheme) in: %s" % url_string)

261

262

try:

263

self.netloc = pu.netloc

264

except Exception:

265

raise InvalidBackendURL("Syntax error (netloc) in: %s" % url_string)

265

raise InvalidBackendURL(u"Syntax error (netloc) in: %s" % url_string)

266

267

try:

268

self.path = pu.path

269

if self.path:

270

self.path = urllib.unquote(self.path)

271

except Exception:

272

raise InvalidBackendURL("Syntax error (path) in: %s" % url_string)

272

raise InvalidBackendURL(u"Syntax error (path) in: %s" % url_string)

273

274

try:

275

self.username = pu.username

276

except Exception:

277

raise InvalidBackendURL("Syntax error (username) in: %s" % url_string)

277

raise InvalidBackendURL(u"Syntax error (username) in: %s" % url_string)

278

if self.username:

279

self.username = urllib.unquote(pu.username)

280

else:

283

try:

284

self.password = pu.password

285

except Exception:

286

raise InvalidBackendURL("Syntax error (password) in: %s" % url_string)

286

raise InvalidBackendURL(u"Syntax error (password) in: %s" % url_string)

287

if self.password:

288

self.password = urllib.unquote(self.password)

289

else:

292

try:

293

self.hostname = pu.hostname

294

except Exception:

295

raise InvalidBackendURL("Syntax error (hostname) in: %s" % url_string)

295

raise InvalidBackendURL(u"Syntax error (hostname) in: %s" % url_string)

296

297

# init to None, overwrite with actual value on success

298

self.port = None

300

self.port = pu.port

301

except Exception: # not raised in python2.7+, just returns None

302

# old style rsync://host::[/]dest, are still valid, though they contain no port

303

if not (self.scheme in ['rsync'] and re.search('::[^:]*$', self.url_string)):

304

raise InvalidBackendURL("Syntax error (port) in: %s A%s B%s C%s" %

305

(url_string, (self.scheme in ['rsync']),

306

re.search('::[^:]+$', self.netloc), self.netloc))

303

if not (self.scheme in [u'rsync'] and re.search(u'::[^:]*$', self.url_string)):

304

raise InvalidBackendURL(u"Syntax error (port) in: %s A%s B%s C%s" %

305

(url_string, (self.scheme in [u'rsync']),

306

re.search(u'::[^:]+$', self.netloc), self.netloc))

307

308

# Our URL system uses two slashes more than urlparse's does when using

309

# non-netloc URLs. And we want to make sure that if urlparse assuming

310

# a netloc where we don't want one, that we correct it.

311

if self.scheme not in uses_netloc:

312

if self.netloc:

313

self.path = '//' + self.netloc + self.path

314

self.netloc = ''

313

self.path = u'//' + self.netloc + self.path

314

self.netloc = u''

315

self.hostname = None

316

elif not self.path.startswith('//') and self.path.startswith('/'):

317

self.path = '//' + self.path

316

elif not self.path.startswith(u'//') and self.path.startswith(u'/'):

317

self.path = u'//' + self.path

318

319

# This happens for implicit local paths.

320

if not self.scheme:

322

323

# Our backends do not handle implicit hosts.

324

if self.scheme in uses_netloc and not self.hostname:

325

raise InvalidBackendURL("Missing hostname in a backend URL which "

326

"requires an explicit hostname: %s"

327

"" % (url_string))

325

raise InvalidBackendURL(u"Missing hostname in a backend URL which "

326

u"requires an explicit hostname: %s"

327

u"" % (url_string))

328

329

# Our backends do not handle implicit relative paths.

330

if self.scheme not in uses_netloc and not self.path.startswith('//'):

331

raise InvalidBackendURL("missing // - relative paths not supported "

332

"for scheme %s: %s"

333

"" % (self.scheme, url_string))

330

if self.scheme not in uses_netloc and not self.path.startswith(u'//'):

331

raise InvalidBackendURL(u"missing // - relative paths not supported "

332

u"for scheme %s: %s"

333

u"" % (self.scheme, url_string))

334

335

def geturl(self):

336

return self.url_string

337

338

339

def strip_auth_from_url(parsed_url):

340

"""Return a URL from a urlparse object without a username or password."""

340

u"""Return a URL from a urlparse object without a username or password."""

341

342

clean_url = re.sub('^([^:/]+://)(.*@)?(.*)', r'\1\3', parsed_url.geturl())

342

clean_url = re.sub(u'^([^:/]+://)(.*@)?(.*)', r'\1\3', parsed_url.geturl())

343

return clean_url

344

345

346

def _get_code_from_exception(backend, operation, e):

347

if isinstance(e, BackendException) and e.code != log.ErrorCode.backend_error:

348

return e.code

349

elif hasattr(backend, '_error_code'):

349

elif hasattr(backend, u'_error_code'):

350

return backend._error_code(operation, e) or log.ErrorCode.backend_error

351

elif hasattr(e, 'errno'):

351

elif hasattr(e, u'errno'):

352

# A few backends return such errors (local, paramiko, etc)

353

if e.errno == errno.EACCES:

354

return log.ErrorCode.backend_permission_denied

372

raise e

373

except Exception as e:

374

# retry on anything else

375

log.Debug(_("Backtrace of previous error: %s")

375

log.Debug(_(u"Backtrace of previous error: %s")

376

% exception_traceback())

377

at_end = n == globals.num_retries

378

code = _get_code_from_exception(self.backend, operation, e)

386

return util.escape(f.uc_name)

387

else:

388

return util.escape(f)

389

extra = ' '.join([operation] + [make_filename(x) for x in args if x])

390

log.FatalError(_("Giving up after %s attempts. %s: %s")

389

extra = u' '.join([operation] + [make_filename(x) for x in args if x])

390

log.FatalError(_(u"Giving up after %s attempts. %s: %s")

391

% (n, e.__class__.__name__,

392

util.uexc(e)), code=code, extra=extra)

393

else:

394

log.Warn(_("Attempt %s failed. %s: %s")

394

log.Warn(_(u"Attempt %s failed. %s: %s")

395

% (n, e.__class__.__name__, util.uexc(e)))

396

if not at_end:

397

if isinstance(e, TemporaryLoadException):

398

time.sleep(3 * globals.backend_retry_delay) # wait longer before trying again

399

else:

400

time.sleep(globals.backend_retry_delay) # wait a bit before trying again

401

if hasattr(self.backend, '_retry_cleanup'):

401

if hasattr(self.backend, u'_retry_cleanup'):

402

self.backend._retry_cleanup()

403

404

return inner_retry

406

407

408

class Backend(object):

409

"""

409

u"""

410

See README in backends directory for information on how to write a backend.

411

"""

412

def __init__(self, parsed_url):

413

self.parsed_url = parsed_url

414

415

""" use getpass by default, inherited backends may overwrite this behaviour """

415

u""" use getpass by default, inherited backends may overwrite this behaviour """

416

use_getpass = True

417

418

def get_password(self):

419

"""

419

u"""

420

Return a password for authentication purposes. The password

421

will be obtained from the backend URL, the environment, by

422

asking the user, or by some other method. When applicable, the

426

return self.parsed_url.password

427

428

try:

429

password = os.environ['FTP_PASSWORD']

429

password = os.environ[u'FTP_PASSWORD']

430

except KeyError:

431

if self.use_getpass:

432

password = getpass.getpass("Password for '%s@%s': " %

432

password = getpass.getpass(u"Password for '%s@%s': " %

433

(self.parsed_url.username, self.parsed_url.hostname))

434

os.environ['FTP_PASSWORD'] = password

434

os.environ[u'FTP_PASSWORD'] = password

435

else:

436

password = None

437

return password

438

439

def munge_password(self, commandline):

440

"""

440

u"""

441

Remove password from commandline by substituting the password

442

found in the URL, if any, with a generic place-holder.

443

451

return commandline

452

453

def __subprocess_popen(self, args):

454

"""

454

u"""

455

For internal use.

456

Execute the given command line, interpreted as a shell command.

457

Returns int Exitcode, string StdOut, string StdErr

464

465

return p.returncode, stdout, stderr

466

467

""" a dictionary for breaking exceptions, syntax is

467

u""" a dictionary for breaking exceptions, syntax is

468

{ 'command' : [ code1, code2 ], ... } see ftpbackend for an example """

469

popen_breaks = {}

470

471

def subprocess_popen(self, commandline):

472

"""

472

u"""

473

Execute the given command line with error check.

474

Returns int Exitcode, string StdOut, string StdErr

475

478

import shlex

479

480

if isinstance(commandline, (types.ListType, types.TupleType)):

481

logstr = ' '.join(commandline)

481

logstr = u' '.join(commandline)

482

args = commandline

483

else:

484

logstr = commandline

485

args = shlex.split(commandline)

486

487

logstr = self.munge_password(logstr)

488

log.Info(_("Reading results of '%s'") % logstr)

488

log.Info(_(u"Reading results of '%s'") % logstr)

489

490

result, stdout, stderr = self.__subprocess_popen(args)

491

if result != 0:

492

try:

493

ignores = self.popen_breaks[args[0]]

494

ignores.index(result)

495

""" ignore a predefined set of error codes """

496

return 0, '', ''

495

u""" ignore a predefined set of error codes """

496

return 0, u'', u''

497

except (KeyError, ValueError):

498

raise BackendException("Error running '%s': returned %d, with output:\n%s" %

499

(logstr, result, stdout + '\n' + stderr))

498

raise BackendException(u"Error running '%s': returned %d, with output:\n%s" %

499

(logstr, result, stdout + u'\n' + stderr))

500

return result, stdout, stderr

501

502

503

class BackendWrapper(object):

504

"""

504

u"""

505

Represents a generic duplicity backend, capable of storing and

506

retrieving files.

507

"""

510

self.backend = backend

511

512

def __do_put(self, source_path, remote_filename):

513

if hasattr(self.backend, '_put'):

514

log.Info(_("Writing %s") % util.fsdecode(remote_filename))

513

if hasattr(self.backend, u'_put'):

514

log.Info(_(u"Writing %s") % util.fsdecode(remote_filename))

515

self.backend._put(source_path, remote_filename)

516

else:

517

raise NotImplementedError()

518

519

@retry('put', fatal=True)

519

@retry(u'put', fatal=True)

520

def put(self, source_path, remote_filename=None):

521

"""

521

u"""

522

Transfer source_path (Path object) to remote_filename (string)

523

524

If remote_filename is None, get the filename from the last

528

remote_filename = source_path.get_filename()

529

self.__do_put(source_path, remote_filename)

530

531

@retry('move', fatal=True)

531

@retry(u'move', fatal=True)

532

def move(self, source_path, remote_filename=None):

533

"""

533

u"""

534

Move source_path (Path object) to remote_filename (string)

535

536

Same as put(), but unlinks source_path in the process. This allows the

538

"""

539

if not remote_filename:

540

remote_filename = source_path.get_filename()

541

if hasattr(self.backend, '_move'):

541

if hasattr(self.backend, u'_move'):

542

if self.backend._move(source_path, remote_filename) is not False:

543

source_path.setdata()

544

return

545

self.__do_put(source_path, remote_filename)

546

source_path.delete()

547

548

@retry('get', fatal=True)

548

@retry(u'get', fatal=True)

549

def get(self, remote_filename, local_path):

550

"""Retrieve remote_filename and place in local_path"""

551

if hasattr(self.backend, '_get'):

550

u"""Retrieve remote_filename and place in local_path"""

551

if hasattr(self.backend, u'_get'):

552

self.backend._get(remote_filename, local_path)

553

local_path.setdata()

554

if not local_path.exists():

555

raise BackendException(_("File %s not found locally after get "

556

"from backend") % local_path.uc_name)

555

raise BackendException(_(u"File %s not found locally after get "

556

u"from backend") % local_path.uc_name)

557

else:

558

raise NotImplementedError()

559

560

@retry('list', fatal=True)

560

@retry(u'list', fatal=True)

561

def list(self):

562

"""

562

u"""

563

Return list of filenames (byte strings) present in backend

564

"""

565

def tobytes(filename):

566

"Convert a (maybe unicode) filename to bytes"

566

u"Convert a (maybe unicode) filename to bytes"

567

if isinstance(filename, unicode):

568

# There shouldn't be any encoding errors for files we care

569

# about, since duplicity filenames are ascii. But user files

572

else:

573

return filename

574

575

if hasattr(self.backend, '_list'):

575

if hasattr(self.backend, u'_list'):

576

# Make sure that duplicity internals only ever see byte strings

577

# for filenames, no matter what the backend thinks it is talking.

578

return [tobytes(x) for x in self.backend._list()]

580

raise NotImplementedError()

581

582

def delete(self, filename_list):

583

"""

583

u"""

584

Delete each filename in filename_list, in order if possible.

585

"""

586

assert not isinstance(filename_list, types.StringType)

587

if hasattr(self.backend, '_delete_list'):

587

if hasattr(self.backend, u'_delete_list'):

588

self._do_delete_list(filename_list)

589

elif hasattr(self.backend, '_delete'):

589

elif hasattr(self.backend, u'_delete'):

590

for filename in filename_list:

591

self._do_delete(filename)

592

else:

593

raise NotImplementedError()

594

595

@retry('delete', fatal=False)

595

@retry(u'delete', fatal=False)

596

def _do_delete_list(self, filename_list):

597

while filename_list:

598

sublist = filename_list[:100]

599

self.backend._delete_list(sublist)

600

filename_list = filename_list[100:]

601

602

@retry('delete', fatal=False)

602

@retry(u'delete', fatal=False)

603

def _do_delete(self, filename):

604

self.backend._delete(filename)

605

614

# Returned dictionary is guaranteed to contain a metadata dictionary for

615

# each filename, and all metadata are guaranteed to be present.

616

def query_info(self, filename_list):

617

"""

617

u"""

618

Return metadata about each filename in filename_list

619

"""

620

info = {}

621

if hasattr(self.backend, '_query_list'):

621

if hasattr(self.backend, u'_query_list'):

622

info = self._do_query_list(filename_list)

623

if info is None:

624

info = {}

625

elif hasattr(self.backend, '_query'):

625

elif hasattr(self.backend, u'_query'):

626

for filename in filename_list:

627

info[filename] = self._do_query(filename)

628

631

for filename in filename_list:

632

if filename not in info or info[filename] is None:

633

info[filename] = {}

634

for metadata in ['size']:

634

for metadata in [u'size']:

635

info[filename].setdefault(metadata, None)

636

637

return info

638

639

@retry('query', fatal=False)

639

@retry(u'query', fatal=False)

640

def _do_query_list(self, filename_list):

641

info = self.backend._query_list(filename_list)

642

if info is None:

643

info = {}

644

return info

645

646

@retry('query', fatal=False)

646

@retry(u'query', fatal=False)

647

def _do_query(self, filename):

648

try:

649

return self.backend._query(filename)

650

except Exception as e:

651

code = _get_code_from_exception(self.backend, 'query', e)

651

code = _get_code_from_exception(self.backend, u'query', e)

652

if code == log.ErrorCode.backend_not_found:

653

return {'size': -1}

653

return {u'size': -1}

654

else:

655

raise e

656

657

def close(self):

658

"""

658

u"""

659

Close the backend, releasing any resources held and

660

invalidating any file objects obtained from the backend.

661

"""

662

if hasattr(self.backend, '_close'):

662

if hasattr(self.backend, u'_close'):

663

self.backend._close()

664

665

def get_fileobj_read(self, filename, parseresults=None):

666

"""

666

u"""

667

Return fileobject opened for reading of filename on backend

668

669

The file will be downloaded first into a temp file. When the

671

"""

672

if not parseresults:

673

parseresults = file_naming.parse(filename)

674

assert parseresults, "Filename not correctly parsed"

674

assert parseresults, u"Filename not correctly parsed"

675

tdp = dup_temp.new_tempduppath(parseresults)

676

self.get(filename, tdp)

677

tdp.setdata()

678

return tdp.filtered_open_with_delete("rb")

678

return tdp.filtered_open_with_delete(u"rb")

679

680

def get_data(self, filename, parseresults=None):

681

"""

681

u"""

682

Retrieve a file from backend, process it, return contents.

683

"""

684

fin = self.get_fileobj_read(filename, parseresults)