1
# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
3
# Permission is hereby granted, free of charge, to any person obtaining a
4
# copy of this software and associated documentation files (the
5
# "Software"), to deal in the Software without restriction, including
6
# without limitation the rights to use, copy, modify, merge, publish, dis-
7
# tribute, sublicense, and/or sell copies of the Software, and to permit
8
# persons to whom the Software is furnished to do so, subject to the fol-
11
# The above copyright notice and this permission notice shall be included
12
# in all copies or substantial portions of the Software.
14
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
16
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
17
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
18
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
28
from boto.exception import S3ResponseError, S3DataError, BotoClientError
29
from boto.s3.user import User
30
from boto import UserAgent
33
from hashlib import md5
40
DefaultContentType = 'application/octet-stream'
44
def __init__(self, bucket=None, name=None):
48
self.content_type = self.DefaultContentType
49
self.content_encoding = None
52
self.last_modified = None
54
self.storage_class = None
61
self.version_id = None
62
self.source_version_id = None
63
self.delete_marker = False
67
return '<Key: %s,%s>' % (self.bucket.name, self.name)
69
return '<Key: None,%s>' % self.name
71
def __getattr__(self, name):
77
def __setattr__(self, name, value):
79
self.__dict__['name'] = value
81
self.__dict__[name] = value
86
def handle_version_headers(self, resp):
87
self.version_id = resp.getheader('x-amz-version-id', None)
88
self.source_version_id = resp.getheader('x-amz-copy-source-version-id', None)
89
if resp.getheader('x-amz-delete-marker', 'false') == 'true':
90
self.delete_marker = True
92
self.delete_marker = False
94
def open_read(self, headers=None, query_args=None):
96
Open this key for reading
99
:param headers: Headers to pass in the web request
101
:type query_args: string
102
:param query_args: Arguments to pass in the query string (ie, 'torrent')
104
if self.resp == None:
107
self.resp = self.bucket.connection.make_request('GET',
110
query_args=query_args)
111
if self.resp.status < 199 or self.resp.status > 299:
112
body = self.resp.read()
113
raise S3ResponseError(self.resp.status, self.resp.reason, body)
114
response_headers = self.resp.msg
115
self.metadata = boto.utils.get_aws_metadata(response_headers)
116
for name,value in response_headers.items():
117
if name.lower() == 'content-length':
118
self.size = int(value)
119
elif name.lower() == 'etag':
121
elif name.lower() == 'content-type':
122
self.content_type = value
123
elif name.lower() == 'content-encoding':
124
self.content_encoding = value
125
elif name.lower() == 'last-modified':
126
self.last_modified = value
127
self.handle_version_headers(self.resp)
129
def open_write(self, headers=None):
131
Open this key for writing.
135
:param headers: Headers to pass in the write request
137
raise BotoClientError('Not Implemented')
139
def open(self, mode='r', headers=None, query_args=None):
142
self.open_read(headers=headers, query_args=query_args)
145
self.open_write(headers=headers)
147
raise BotoClientError('Invalid mode: %s' % mode)
159
By providing a next method, the key object supports use as an iterator.
160
For example, you can now say:
163
write bytes to a file or whatever
165
All of the HTTP connection stuff is handled for you.
168
data = self.resp.read(self.BufferSize)
174
def read(self, size=0):
176
size = self.BufferSize
178
data = self.resp.read(size)
183
def copy(self, dst_bucket, dst_key, metadata=None):
185
Copy this Key to another bucket.
187
:type dst_bucket: string
188
:param dst_bucket: The name of the destination bucket
190
:type dst_key: string
191
:param dst_key: The name of the destinatino key
194
:param metadata: Metadata to be associated with new key.
195
If metadata is supplied, it will replace the
196
metadata of the source key being copied.
197
If no metadata is supplied, the source key's
198
metadata will be copied to the new key.
200
:rtype: :class:`boto.s3.key.Key` or subclass
201
:returns: An instance of the newly created key object
203
dst_bucket = self.bucket.connection.lookup(dst_bucket)
204
return dst_bucket.copy_key(dst_key, self.bucket.name, self.name, metadata)
206
def startElement(self, name, attrs, connection):
208
self.owner = User(self)
213
def endElement(self, name, value, connection):
215
self.name = value.encode('utf-8')
218
elif name == 'LastModified':
219
self.last_modified = value
221
self.size = int(value)
222
elif name == 'StorageClass':
223
self.storage_class = value
224
elif name == 'Owner':
226
elif name == 'VersionId':
227
self.version_id = value
229
setattr(self, name, value)
233
Returns True if the key exists
236
:return: Whether the key exists on S3
238
return bool(self.bucket.lookup(self.name))
242
Delete this key from S3
244
return self.bucket.delete_key(self.name)
246
def get_metadata(self, name):
247
return self.metadata.get(name)
249
def set_metadata(self, name, value):
250
self.metadata[name] = value
252
def update_metadata(self, d):
253
self.metadata.update(d)
255
# convenience methods for setting/getting ACL
256
def set_acl(self, acl_str, headers=None):
257
if self.bucket != None:
258
self.bucket.set_acl(acl_str, self.name, headers=headers)
260
def get_acl(self, headers=None):
261
if self.bucket != None:
262
return self.bucket.get_acl(self.name, headers=headers)
264
def get_xml_acl(self, headers=None):
265
if self.bucket != None:
266
return self.bucket.get_xml_acl(self.name, headers=headers)
268
def set_xml_acl(self, acl_str, headers=None):
269
if self.bucket != None:
270
return self.bucket.set_xml_acl(acl_str, self.name, headers=headers)
272
def set_canned_acl(self, acl_str, headers=None):
273
return self.bucket.set_canned_acl(acl_str, self.name, headers)
275
def make_public(self, headers=None):
276
return self.bucket.set_canned_acl('public-read', self.name, headers)
278
def generate_url(self, expires_in, method='GET', headers=None,
279
query_auth=True, force_http=False):
281
Generate a URL to access this key.
283
:type expires_in: int
284
:param expires_in: How long the url is valid for, in seconds
287
:param method: The method to use for retrieving the file (default is GET)
290
:param headers: Any headers to pass along in the request
292
:type query_auth: bool
296
:return: The URL to access the key
298
return self.bucket.connection.generate_url(expires_in, method,
299
self.bucket.name, self.name,
300
headers, query_auth, force_http)
302
def send_file(self, fp, headers=None, cb=None, num_cb=10):
304
Upload a file to a key into a bucket on S3.
307
:param fp: The file pointer to upload
310
:param headers: The headers to pass along with the PUT request
313
:param cb: a callback function that will be called to report
314
progress on the upload. The callback should accept two integer
315
parameters, the first representing the number of bytes that have
316
been successfully transmitted to S3 and the second representing
317
the total number of bytes that need to be transmitted.
320
:param num_cb: (optional) If a callback is specified with the cb parameter
321
this parameter determines the granularity of the callback by defining
322
the maximum number of times the callback will be called during the file transfer.
325
def sender(http_conn, method, path, data, headers):
326
http_conn.putrequest(method, path)
328
http_conn.putheader(key, headers[key])
329
http_conn.endheaders()
331
save_debug = self.bucket.connection.debug
332
self.bucket.connection.debug = 0
335
cb_count = self.size / self.BufferSize / (num_cb-2)
339
cb(total_bytes, self.size)
340
l = fp.read(self.BufferSize)
344
total_bytes += len(l)
347
cb(total_bytes, self.size)
349
l = fp.read(self.BufferSize)
351
cb(total_bytes, self.size)
352
response = http_conn.getresponse()
353
body = response.read()
355
self.bucket.connection.debug = save_debug
356
if response.status == 500 or response.status == 503 or \
357
response.getheader('location'):
360
elif response.status >= 200 and response.status <= 299:
361
self.etag = response.getheader('etag')
362
if self.etag != '"%s"' % self.md5:
363
raise S3DataError('ETag from S3 did not match computed MD5')
366
raise S3ResponseError(response.status, response.reason, body)
371
headers = headers.copy()
372
headers['User-Agent'] = UserAgent
373
headers['Content-MD5'] = self.base64md5
374
if headers.has_key('Content-Type'):
375
self.content_type = headers['Content-Type']
377
self.content_type = mimetypes.guess_type(self.path)[0]
378
if self.content_type == None:
379
self.content_type = self.DefaultContentType
380
headers['Content-Type'] = self.content_type
382
headers['Content-Type'] = self.content_type
383
headers['Content-Length'] = str(self.size)
384
headers['Expect'] = '100-Continue'
385
headers = boto.utils.merge_meta(headers, self.metadata)
386
resp = self.bucket.connection.make_request('PUT', self.bucket.name,
389
self.handle_version_headers(resp)
391
def compute_md5(self, fp):
394
:param fp: File pointer to the file to MD5 hash. The file pointer will be
395
reset to the beginning of the file before the method returns.
398
:return: A tuple containing the hex digest version of the MD5 hash
399
as the first element and the base64 encoded version of the
400
plain digest as the second element.
404
s = fp.read(self.BufferSize)
407
s = fp.read(self.BufferSize)
408
hex_md5 = m.hexdigest()
409
base64md5 = base64.encodestring(m.digest())
410
if base64md5[-1] == '\n':
411
base64md5 = base64md5[0:-1]
412
self.size = fp.tell()
414
return (hex_md5, base64md5)
416
def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10,
417
policy=None, md5=None):
419
Store an object in S3 using the name of the Key object as the
420
key in S3 and the contents of the file pointed to by 'fp' as the
424
:param fp: the file whose contents to upload
427
:param headers: additional HTTP headers that will be sent with the PUT request.
430
:param replace: If this parameter is False, the method
431
will first check to see if an object exists in the
432
bucket with the same key. If it does, it won't
433
overwrite it. The default value is True which will
434
overwrite the object.
437
:param cb: a callback function that will be called to report
438
progress on the upload. The callback should accept two integer
439
parameters, the first representing the number of bytes that have
440
been successfully transmitted to S3 and the second representing
441
the total number of bytes that need to be transmitted.
444
:param num_cb: (optional) If a callback is specified with the cb parameter
445
this parameter determines the granularity of the callback by defining
446
the maximum number of times the callback will be called during the file transfer.
448
:type policy: :class:`boto.s3.acl.CannedACLStrings`
449
:param policy: A canned ACL policy that will be applied to the new key in S3.
451
:type md5: A tuple containing the hexdigest version of the MD5 checksum of the
452
file as the first element and the Base64-encoded version of the plain
453
checksum as the second element. This is the same format returned by
454
the compute_md5 method.
455
:param md5: If you need to compute the MD5 for any reason prior to upload,
456
it's silly to have to do it twice so this param, if present, will be
457
used as the MD5 values of the file. Otherwise, the checksum will be computed.
461
headers['x-amz-acl'] = policy
463
headers = {'x-amz-acl' : policy}
464
if hasattr(fp, 'name'):
466
if self.bucket != None:
468
md5 = self.compute_md5(fp)
470
self.base64md5 = md5[1]
471
if self.name == None:
474
k = self.bucket.lookup(self.name)
477
self.send_file(fp, headers, cb, num_cb)
479
def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10,
480
policy=None, md5=None):
482
Store an object in S3 using the name of the Key object as the
483
key in S3 and the contents of the file named by 'filename'.
484
See set_contents_from_file method for details about the
487
:type filename: string
488
:param filename: The name of the file that you want to put onto S3
491
:param headers: Additional headers to pass along with the request to AWS.
494
:param replace: If True, replaces the contents of the file if it already exists.
497
:param cb: (optional) a callback function that will be called to report
498
progress on the download. The callback should accept two integer
499
parameters, the first representing the number of bytes that have
500
been successfully transmitted from S3 and the second representing
501
the total number of bytes that need to be transmitted.
504
:param num_cb: (optional) If a callback is specified with the cb parameter
505
this parameter determines the granularity of the callback by defining
506
the maximum number of times the callback will be called during the file transfer.
508
:type policy: :class:`boto.s3.acl.CannedACLStrings`
509
:param policy: A canned ACL policy that will be applied to the new key in S3.
511
:type md5: A tuple containing the hexdigest version of the MD5 checksum of the
512
file as the first element and the Base64-encoded version of the plain
513
checksum as the second element. This is the same format returned by
514
the compute_md5 method.
515
:param md5: If you need to compute the MD5 for any reason prior to upload,
516
it's silly to have to do it twice so this param, if present, will be
517
used as the MD5 values of the file. Otherwise, the checksum will be computed.
519
fp = open(filename, 'rb')
520
self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy)
523
def set_contents_from_string(self, s, headers=None, replace=True, cb=None, num_cb=10,
524
policy=None, md5=None):
526
Store an object in S3 using the name of the Key object as the
527
key in S3 and the string 's' as the contents.
528
See set_contents_from_file method for details about the
532
:param headers: Additional headers to pass along with the request to AWS.
535
:param replace: If True, replaces the contents of the file if it already exists.
538
:param cb: (optional) a callback function that will be called to report
539
progress on the download. The callback should accept two integer
540
parameters, the first representing the number of bytes that have
541
been successfully transmitted from S3 and the second representing
542
the total number of bytes that need to be transmitted.
545
:param num_cb: (optional) If a callback is specified with the cb parameter
546
this parameter determines the granularity of the callback by defining
547
the maximum number of times the callback will be called during the file transfer.
549
:type policy: :class:`boto.s3.acl.CannedACLStrings`
550
:param policy: A canned ACL policy that will be applied to the new key in S3.
552
:type md5: A tuple containing the hexdigest version of the MD5 checksum of the
553
file as the first element and the Base64-encoded version of the plain
554
checksum as the second element. This is the same format returned by
555
the compute_md5 method.
556
:param md5: If you need to compute the MD5 for any reason prior to upload,
557
it's silly to have to do it twice so this param, if present, will be
558
used as the MD5 values of the file. Otherwise, the checksum will be computed.
560
fp = StringIO.StringIO(s)
561
r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy)
565
def get_file(self, fp, headers=None, cb=None, num_cb=10,
566
torrent=False, version_id=None):
568
Retrieves a file from an S3 Key
571
:param fp: File pointer to put the data into
573
:type headers: string
574
:param: headers to send when retrieving the files
577
:param cb: (optional) a callback function that will be called to report
578
progress on the download. The callback should accept two integer
579
parameters, the first representing the number of bytes that have
580
been successfully transmitted from S3 and the second representing
581
the total number of bytes that need to be transmitted.
585
:param num_cb: (optional) If a callback is specified with the cb parameter
586
this parameter determines the granularity of the callback by defining
587
the maximum number of times the callback will be called during the file transfer.
590
:param torrent: Flag for whether to get a torrent for the file
594
cb_count = self.size / self.BufferSize / (num_cb-2)
598
cb(total_bytes, self.size)
599
save_debug = self.bucket.connection.debug
600
if self.bucket.connection.debug == 1:
601
self.bucket.connection.debug = 0
605
query_args = 'torrent'
607
query_args = 'versionId=%s' % version_id
608
self.open('r', headers, query_args=query_args)
612
total_bytes += len(bytes)
615
cb(total_bytes, self.size)
618
cb(total_bytes, self.size)
620
self.bucket.connection.debug = save_debug
622
def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10):
624
Get a torrent file (see to get_file)
627
:param fp: The file pointer of where to put the torrent
630
:param headers: Headers to be passed
633
:param cb: Callback function to call on retrieved data
636
:param num_cb: (optional) If a callback is specified with the cb parameter
637
this parameter determines the granularity of the callback by defining
638
the maximum number of times the callback will be called during the file transfer.
641
return self.get_file(fp, headers, cb, num_cb, torrent=True)
643
def get_contents_to_file(self, fp, headers=None,
648
Retrieve an object from S3 using the name of the Key object as the
649
key in S3. Write the contents of the object to the file pointed
652
:type fp: File -like object
656
:param headers: additional HTTP headers that will be sent with the GET request.
659
:param cb: (optional) a callback function that will be called to report
660
progress on the download. The callback should accept two integer
661
parameters, the first representing the number of bytes that have
662
been successfully transmitted from S3 and the second representing
663
the total number of bytes that need to be transmitted.
667
:param num_cb: (optional) If a callback is specified with the cb parameter
668
this parameter determines the granularity of the callback by defining
669
the maximum number of times the callback will be called during the file transfer.
672
:param torrent: If True, returns the contents of a torrent file as a string.
675
if self.bucket != None:
676
self.get_file(fp, headers, cb, num_cb, torrent=torrent,
677
version_id=version_id)
679
def get_contents_to_filename(self, filename, headers=None,
684
Retrieve an object from S3 using the name of the Key object as the
685
key in S3. Store contents of the object to a file named by 'filename'.
686
See get_contents_to_file method for details about the
689
:type filename: string
690
:param filename: The filename of where to put the file contents
693
:param headers: Any additional headers to send in the request
696
:param cb: (optional) a callback function that will be called to report
697
progress on the download. The callback should accept two integer
698
parameters, the first representing the number of bytes that have
699
been successfully transmitted from S3 and the second representing
700
the total number of bytes that need to be transmitted.
704
:param num_cb: (optional) If a callback is specified with the cb parameter
705
this parameter determines the granularity of the callback by defining
706
the maximum number of times the callback will be called during the file transfer.
709
:param torrent: If True, returns the contents of a torrent file as a string.
712
fp = open(filename, 'wb')
713
self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent,
714
version_id=version_id)
716
# if last_modified date was sent from s3, try to set file's timestamp
717
if self.last_modified != None:
719
modified_tuple = rfc822.parsedate_tz(self.last_modified)
720
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
721
os.utime(fp.name, (modified_stamp, modified_stamp))
722
except Exception: pass
724
def get_contents_as_string(self, headers=None,
729
Retrieve an object from S3 using the name of the Key object as the
730
key in S3. Return the contents of the object as a string.
731
See get_contents_to_file method for details about the
735
:param headers: Any additional headers to send in the request
738
:param cb: (optional) a callback function that will be called to report
739
progress on the download. The callback should accept two integer
740
parameters, the first representing the number of bytes that have
741
been successfully transmitted from S3 and the second representing
742
the total number of bytes that need to be transmitted.
745
:param num_cb: (optional) If a callback is specified with the cb parameter
746
this parameter determines the granularity of the callback by defining
747
the maximum number of times the callback will be called during the file transfer.
751
:param num_cb: (optional) If a callback is specified with the cb parameter
752
this parameter determines the granularity of the callback by defining
753
the maximum number of times the callback will be called during the file transfer.
756
:param torrent: If True, returns the contents of a torrent file as a string.
759
:returns: The contents of the file as a string
761
fp = StringIO.StringIO()
762
self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent,
763
version_id=version_id)
766
def add_email_grant(self, permission, email_address):
768
Convenience method that provides a quick way to add an email grant to a key.
769
This method retrieves the current ACL, creates a new grant based on the parameters
770
passed in, adds that grant to the ACL and then PUT's the new ACL back to S3.
772
:type permission: string
773
:param permission: The permission being granted. Should be one of:
774
READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL
775
See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html
776
for more details on permissions.
778
:type email_address: string
779
:param email_address: The email address associated with the AWS account your are granting
782
policy = self.get_acl()
783
policy.acl.add_email_grant(permission, email_address)
786
def add_user_grant(self, permission, user_id):
788
Convenience method that provides a quick way to add a canonical user grant to a key.
789
This method retrieves the current ACL, creates a new grant based on the parameters
790
passed in, adds that grant to the ACL and then PUT's the new ACL back to S3.
792
:type permission: string
793
:param permission: The permission being granted. Should be one of:
794
READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL
795
See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html
796
for more details on permissions.
798
:type user_id: string
799
:param user_id: The canonical user id associated with the AWS account your are granting
802
policy = self.get_acl()
803
policy.acl.add_user_grant(permission, user_id)
added
removed
vendor/boto/boto/s3/key.py
