1
# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
3
# Copyright 2002 Ben Escoto <ben@emerose.org>
4
# Copyright 2007 Kenneth Loafman <kenneth@loafman.com>
6
# This file is part of duplicity.
8
# Duplicity is free software; you can redistribute it and/or modify it
9
# under the terms of the GNU General Public License as published by the
10
# Free Software Foundation; either version 2 of the License, or (at your
11
# option) any later version.
13
# Duplicity is distributed in the hope that it will be useful, but
14
# WITHOUT ANY WARRANTY; without even the implied warranty of
15
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16
# General Public License for more details.
18
# You should have received a copy of the GNU General Public License
19
# along with duplicity; if not, write to the Free Software Foundation,
20
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24
import duplicity.backend
25
from duplicity import globals
26
from duplicity import log
27
from duplicity.errors import * #@UnusedWildImport
28
from duplicity.util import exception_traceback
30
class BotoBackend(duplicity.backend.Backend):
32
Backend for Amazon's Simple Storage System, (aka Amazon S3), though
33
the use of the boto module, (http://code.google.com/p/boto/).
35
To make use of this backend you must set aws_access_key_id
36
and aws_secret_access_key in your ~/.boto or /etc/boto.cfg
37
with your Amazon Web Services key id and secret respectively.
38
Alternatively you can export the environment variables
39
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
42
def __init__(self, parsed_url):
43
duplicity.backend.Backend.__init__(self, parsed_url)
45
from boto.s3.key import Key
47
# This folds the null prefix and all null parts, which means that:
48
# //MyBucket/ and //MyBucket are equivalent.
49
# //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent.
50
self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/'))
53
self.bucket_name = self.url_parts.pop(0)
55
# Duplicity hangs if boto gets a null bucket name.
56
# HC: Caught a socket error, trying to recover
57
raise BackendException('Boto requires a bucket name.')
59
self.scheme = parsed_url.scheme
64
self.key_prefix = '%s/' % '/'.join(self.url_parts)
68
self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url)
69
self.parsed_url = parsed_url
70
self.resetConnection()
72
def resetConnection(self):
77
from boto.s3.connection import S3Connection
78
from boto.s3.key import Key
79
assert hasattr(S3Connection, 'lookup')
81
# Newer versions of boto default to using
82
# virtual hosting for buckets as a result of
83
# upstream deprecation of the old-style access
84
# method by Amazon S3. This change is not
85
# backwards compatible (in particular with
86
# respect to upper case characters in bucket
87
# names); so we default to forcing use of the
88
# old-style method unless the user has
89
# explicitly asked us to use new-style bucket
92
# Note that if the user wants to use new-style
93
# buckets, we use the subdomain calling form
94
# rather than given the option of both
95
# subdomain and vhost. The reason being that
96
# anything addressable as a vhost, is also
97
# addressable as a subdomain. Seeing as the
98
# latter is mostly a convenience method of
99
# allowing browse:able content semi-invisibly
100
# being hosted on S3, the former format makes
101
# a lot more sense for us to use - being
102
# explicit about what is happening (the fact
103
# that we are talking to S3 servers).
106
from boto.s3.connection import OrdinaryCallingFormat
107
from boto.s3.connection import SubdomainCallingFormat
109
calling_format = OrdinaryCallingFormat()
111
cfs_supported = False
112
calling_format = None
114
if globals.s3_use_new_style:
116
calling_format = SubdomainCallingFormat()
118
log.FatalError("Use of new-style (subdomain) S3 bucket addressing was"
119
"requested, but does not seem to be supported by the "
120
"boto library. Either you need to upgrade your boto "
121
"library or duplicity has failed to correctly detect "
122
"the appropriate support.",
123
log.ErrorCode.boto_old_style)
126
calling_format = OrdinaryCallingFormat()
128
calling_format = None
131
log.FatalError("This backend (s3) requires boto library, version 0.9d or later, "
132
"(http://code.google.com/p/boto/).",
133
log.ErrorCode.boto_lib_too_old)
134
if self.scheme == 's3+http':
135
# Use the default Amazon S3 host.
136
self.conn = S3Connection(is_secure=(not globals.s3_unencrypted_connection))
138
assert self.scheme == 's3'
139
self.conn = S3Connection(
140
host=self.parsed_url.hostname,
141
is_secure=(not globals.s3_unencrypted_connection))
143
if hasattr(self.conn, 'calling_format'):
144
if calling_format is None:
145
log.FatalError("It seems we previously failed to detect support for calling "
146
"formats in the boto library, yet the support is there. This is "
147
"almost certainly a duplicity bug.",
148
log.ErrorCode.boto_calling_format)
150
self.conn.calling_format = calling_format
153
# Duplicity hangs if boto gets a null bucket name.
154
# HC: Caught a socket error, trying to recover
155
raise BackendException('Boto requires a bucket name.')
157
self.bucket = self.conn.lookup(self.bucket_name)
159
def put(self, source_path, remote_filename=None):
160
from boto.s3.connection import Location
161
if globals.s3_european_buckets:
162
if not globals.s3_use_new_style:
163
log.FatalError("European bucket creation was requested, but not new-style "
164
"bucket addressing (--s3-use-new-style)",
165
log.ErrorCode.s3_bucket_not_style)
166
#Network glitch may prevent first few attempts of creating/looking up a bucket
167
for n in range(1, globals.num_retries+1):
174
self.bucket = self.conn.get_bucket(self.bucket_name, validate=True)
176
if "NoSuchBucket" in str(e):
177
if globals.s3_european_buckets:
178
self.bucket = self.conn.create_bucket(self.bucket_name,
179
location=Location.EU)
181
self.bucket = self.conn.create_bucket(self.bucket_name)
185
log.Warn("Failed to create bucket (attempt #%d) '%s' failed (reason: %s: %s)"
186
"" % (n, self.bucket_name,
187
e.__class__.__name__,
189
self.resetConnection()
191
if not remote_filename:
192
remote_filename = source_path.get_filename()
193
key = self.key_class(self.bucket)
194
key.key = self.key_prefix + remote_filename
195
for n in range(1, globals.num_retries+1):
197
# sleep before retry (new connection to a **hopeful** new host, so no need to wait so long)
200
if globals.s3_use_rrs:
201
storage_class = 'REDUCED_REDUNDANCY'
203
storage_class = 'STANDARD'
204
log.Info("Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class))
206
key.set_contents_from_filename(source_path.name, {'Content-Type': 'application/octet-stream',
207
'x-amz-storage-class': storage_class})
209
self.resetConnection()
212
log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)"
213
"" % (self.straight_url,
216
e.__class__.__name__,
218
log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
219
self.resetConnection()
220
log.Warn("Giving up trying to upload %s/%s after %d attempts" %
221
(self.straight_url, remote_filename, globals.num_retries))
222
raise BackendException("Error uploading %s/%s" % (self.straight_url, remote_filename))
224
def get(self, remote_filename, local_path):
225
key = self.key_class(self.bucket)
226
key.key = self.key_prefix + remote_filename
227
for n in range(1, globals.num_retries+1):
229
# sleep before retry (new connection to a **hopeful** new host, so no need to wait so long)
231
log.Info("Downloading %s/%s" % (self.straight_url, remote_filename))
233
key.get_contents_to_filename(local_path.name)
235
self.resetConnection()
238
log.Warn("Download %s/%s failed (attempt #%d, reason: %s: %s)"
239
"" % (self.straight_url,
242
e.__class__.__name__,
244
log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
245
self.resetConnection()
246
log.Warn("Giving up trying to download %s/%s after %d attempts" %
247
(self.straight_url, remote_filename, globals.num_retries))
248
raise BackendException("Error downloading %s/%s" % (self.straight_url, remote_filename))
254
for n in range(1, globals.num_retries+1):
258
log.Info("Listing %s" % self.straight_url)
260
return self._list_filenames_in_bucket()
262
log.Warn("List %s failed (attempt #%d, reason: %s: %s)"
263
"" % (self.straight_url,
265
e.__class__.__name__,
267
log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
268
log.Warn("Giving up trying to list %s after %d attempts" %
269
(self.straight_url, globals.num_retries))
270
raise BackendException("Error listng %s" % self.straight_url)
272
def _list_filenames_in_bucket(self):
273
# We add a 'd' to the prefix to make sure it is not null (for boto) and
274
# to optimize the listing of our filenames, which always begin with 'd'.
275
# This will cause a failure in the regression tests as below:
276
# FAIL: Test basic backend operations
278
# AssertionError: Got list: []
279
# Wanted: ['testfile']
280
# Because of the need for this optimization, it should be left as is.
281
#for k in self.bucket.list(prefix = self.key_prefix + 'd', delimiter = '/'):
283
for k in self.bucket.list(prefix = self.key_prefix, delimiter = '/'):
285
filename = k.key.replace(self.key_prefix, '', 1)
286
filename_list.append(filename)
287
log.Debug("Listed %s/%s" % (self.straight_url, filename))
288
except AttributeError:
292
def delete(self, filename_list):
293
for filename in filename_list:
294
self.bucket.delete_key(self.key_prefix + filename)
295
log.Debug("Deleted %s/%s" % (self.straight_url, filename))
297
duplicity.backend.register_backend("s3", BotoBackend)
298
duplicity.backend.register_backend("s3+http", BotoBackend)
added
removed
src/backends/botobackend.py
