1
#-------------------------------------------------------------------------
2
# Copyright (c) Microsoft. All rights reserved.
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
# See the License for the specific language governing permissions and
13
# limitations under the License.
14
#--------------------------------------------------------------------------
20
from datetime import datetime
21
from xml.dom import minidom
22
from azure import (WindowsAzureData,
27
_decode_base64_to_text,
28
_decode_base64_to_bytes,
31
_fill_instance_element,
34
_get_children_from_path,
35
_get_entry_properties,
36
_general_error_handler,
38
_parse_response_for_dict,
40
_ERROR_CANNOT_SERIALIZE_VALUE_TO_ENTITY,
43
# x-ms-version for storage service.
44
X_MS_VERSION = '2012-02-12'
47
class EnumResultsBase(object):
49
''' base class for EnumResults. '''
55
self.next_marker = u''
58
class ContainerEnumResults(EnumResultsBase):
60
''' Blob Container list. '''
63
EnumResultsBase.__init__(self)
64
self.containers = _list_of(Container)
67
return iter(self.containers)
70
return len(self.containers)
72
def __getitem__(self, index):
73
return self.containers[index]
76
class Container(WindowsAzureData):
78
''' Blob container class. '''
83
self.properties = Properties()
87
class Properties(WindowsAzureData):
89
''' Blob container's properties class. '''
92
self.last_modified = u''
96
class RetentionPolicy(WindowsAzureData):
98
''' RetentionPolicy in service properties. '''
102
self.__dict__['days'] = None
105
# convert days to int value
106
return int(self.__dict__['days'])
108
def set_days(self, value):
109
''' set default days if days is set to empty. '''
110
self.__dict__['days'] = value
112
days = property(fget=get_days, fset=set_days)
115
class Logging(WindowsAzureData):
117
''' Logging class in service properties. '''
120
self.version = u'1.0'
124
self.retention_policy = RetentionPolicy()
127
class Metrics(WindowsAzureData):
129
''' Metrics class in service properties. '''
132
self.version = u'1.0'
134
self.include_apis = None
135
self.retention_policy = RetentionPolicy()
138
class StorageServiceProperties(WindowsAzureData):
140
''' Storage Service Propeties class. '''
143
self.logging = Logging()
144
self.metrics = Metrics()
147
class AccessPolicy(WindowsAzureData):
149
''' Access Policy class in service properties. '''
151
def __init__(self, start=u'', expiry=u'', permission='u'):
154
self.permission = permission
157
class SignedIdentifier(WindowsAzureData):
159
''' Signed Identifier class for service properties. '''
163
self.access_policy = AccessPolicy()
166
class SignedIdentifiers(WindowsAzureData):
168
''' SignedIdentifier list. '''
171
self.signed_identifiers = _list_of(SignedIdentifier)
174
return iter(self.signed_identifiers)
177
return len(self.signed_identifiers)
179
def __getitem__(self, index):
180
return self.signed_identifiers[index]
183
class BlobEnumResults(EnumResultsBase):
188
EnumResultsBase.__init__(self)
189
self.blobs = _list_of(Blob)
190
self.prefixes = _list_of(BlobPrefix)
194
return iter(self.blobs)
197
return len(self.blobs)
199
def __getitem__(self, index):
200
return self.blobs[index]
203
class BlobResult(bytes):
205
def __new__(cls, blob, properties):
206
return bytes.__new__(cls, blob if blob else b'')
208
def __init__(self, blob, properties):
209
self.properties = properties
212
class Blob(WindowsAzureData):
220
self.properties = BlobProperties()
224
class BlobProperties(WindowsAzureData):
226
''' Blob Properties '''
229
self.last_modified = u''
231
self.content_length = 0
232
self.content_type = u''
233
self.content_encoding = u''
234
self.content_language = u''
235
self.content_md5 = u''
236
self.xms_blob_sequence_number = 0
238
self.lease_status = u''
239
self.lease_state = u''
240
self.lease_duration = u''
242
self.copy_source = u''
243
self.copy_status = u''
244
self.copy_progress = u''
245
self.copy_completion_time = u''
246
self.copy_status_description = u''
249
class BlobPrefix(WindowsAzureData):
251
''' BlobPrefix in Blob. '''
257
class BlobBlock(WindowsAzureData):
259
''' BlobBlock class '''
261
def __init__(self, id=None, size=None):
266
class BlobBlockList(WindowsAzureData):
268
''' BlobBlockList class '''
271
self.committed_blocks = []
272
self.uncommitted_blocks = []
275
class PageRange(WindowsAzureData):
277
''' Page Range for page blob. '''
284
class PageList(object):
286
''' Page list for page blob. '''
289
self.page_ranges = _list_of(PageRange)
292
return iter(self.page_ranges)
295
return len(self.page_ranges)
297
def __getitem__(self, index):
298
return self.page_ranges[index]
301
class QueueEnumResults(EnumResultsBase):
306
EnumResultsBase.__init__(self)
307
self.queues = _list_of(Queue)
310
return iter(self.queues)
313
return len(self.queues)
315
def __getitem__(self, index):
316
return self.queues[index]
319
class Queue(WindowsAzureData):
329
class QueueMessagesList(WindowsAzureData):
331
''' Queue message list. '''
334
self.queue_messages = _list_of(QueueMessage)
337
return iter(self.queue_messages)
340
return len(self.queue_messages)
342
def __getitem__(self, index):
343
return self.queue_messages[index]
346
class QueueMessage(WindowsAzureData):
348
''' Queue message class. '''
351
self.message_id = u''
352
self.insertion_time = u''
353
self.expiration_time = u''
354
self.pop_receipt = u''
355
self.time_next_visible = u''
356
self.dequeue_count = u''
357
self.message_text = u''
360
class Entity(WindowsAzureData):
362
''' Entity class. The attributes of entity will be created dynamically. '''
366
class EntityProperty(WindowsAzureData):
368
''' Entity property. contains type and value. '''
370
def __init__(self, type=None, value=None):
375
class Table(WindowsAzureData):
377
''' Only for intellicens and telling user the return type. '''
381
def _parse_blob_enum_results_list(response):
382
respbody = response.body
383
return_obj = BlobEnumResults()
384
doc = minidom.parseString(respbody)
386
for enum_results in _get_child_nodes(doc, 'EnumerationResults'):
387
for child in _get_children_from_path(enum_results, 'Blobs', 'Blob'):
388
return_obj.blobs.append(_fill_instance_element(child, Blob))
390
for child in _get_children_from_path(enum_results,
393
return_obj.prefixes.append(
394
_fill_instance_element(child, BlobPrefix))
396
for name, value in vars(return_obj).items():
397
if name == 'blobs' or name == 'prefixes':
399
value = _fill_data_minidom(enum_results, name, value)
400
if value is not None:
401
setattr(return_obj, name, value)
406
def _update_storage_header(request):
407
''' add additional headers for storage request. '''
409
assert isinstance(request.body, bytes)
411
# if it is PUT, POST, MERGE, DELETE, need to add content-lengt to header.
412
if request.method in ['PUT', 'POST', 'MERGE', 'DELETE']:
413
request.headers.append(('Content-Length', str(len(request.body))))
415
# append addtional headers base on the service
416
request.headers.append(('x-ms-version', X_MS_VERSION))
418
# append x-ms-meta name, values to header
419
for name, value in request.headers:
420
if 'x-ms-meta-name-values' in name and value:
421
for meta_name, meta_value in value.items():
422
request.headers.append(('x-ms-meta-' + meta_name, meta_value))
423
request.headers.remove((name, value))
428
def _update_storage_blob_header(request, account_name, account_key):
429
''' add additional headers for storage blob request. '''
431
request = _update_storage_header(request)
432
current_time = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
433
request.headers.append(('x-ms-date', current_time))
434
request.headers.append(
435
('Content-Type', 'application/octet-stream Charset=UTF-8'))
436
request.headers.append(('Authorization',
437
_sign_storage_blob_request(request,
441
return request.headers
444
def _update_storage_queue_header(request, account_name, account_key):
445
''' add additional headers for storage queue request. '''
446
return _update_storage_blob_header(request, account_name, account_key)
449
def _update_storage_table_header(request):
450
''' add additional headers for storage table request. '''
452
request = _update_storage_header(request)
453
for name, _ in request.headers:
454
if name.lower() == 'content-type':
457
request.headers.append(('Content-Type', 'application/atom+xml'))
458
request.headers.append(('DataServiceVersion', '2.0;NetFx'))
459
request.headers.append(('MaxDataServiceVersion', '2.0;NetFx'))
460
current_time = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
461
request.headers.append(('x-ms-date', current_time))
462
request.headers.append(('Date', current_time))
463
return request.headers
466
def _sign_storage_blob_request(request, account_name, account_key):
468
Returns the signed string for blob request which is used to set
469
Authorization header. This is also used to sign queue request.
472
uri_path = request.path.split('?')[0]
475
string_to_sign = request.method + '\n'
477
# get headers to sign
479
'content-encoding', 'content-language', 'content-length',
480
'content-md5', 'content-type', 'date', 'if-modified-since',
481
'if-match', 'if-none-match', 'if-unmodified-since', 'range']
483
request_header_dict = dict((name.lower(), value)
484
for name, value in request.headers if value)
485
string_to_sign += '\n'.join(request_header_dict.get(x, '')
486
for x in headers_to_sign) + '\n'
488
# get x-ms header to sign
490
for name, value in request.headers:
492
x_ms_headers.append((name.lower(), value))
494
for name, value in x_ms_headers:
496
string_to_sign += ''.join([name, ':', value, '\n'])
498
# get account_name and uri path to sign
499
string_to_sign += '/' + account_name + uri_path
501
# get query string to sign if it is not table service
502
query_to_sign = request.query
506
for name, value in query_to_sign:
508
if current_name != name:
509
string_to_sign += '\n' + name + ':' + value
511
string_to_sign += '\n' + ',' + value
514
auth_string = 'SharedKey ' + account_name + ':' + \
515
_sign_string(account_key, string_to_sign)
519
def _sign_storage_table_request(request, account_name, account_key):
520
uri_path = request.path.split('?')[0]
522
string_to_sign = request.method + '\n'
523
headers_to_sign = ['content-md5', 'content-type', 'date']
524
request_header_dict = dict((name.lower(), value)
525
for name, value in request.headers if value)
526
string_to_sign += '\n'.join(request_header_dict.get(x, '')
527
for x in headers_to_sign) + '\n'
529
# get account_name and uri path to sign
530
string_to_sign += ''.join(['/', account_name, uri_path])
532
for name, value in request.query:
533
if name == 'comp' and uri_path == '/':
534
string_to_sign += '?comp=' + value
538
auth_string = 'SharedKey ' + account_name + ':' + \
539
_sign_string(account_key, string_to_sign)
543
def _sign_string(account_key, string_to_sign):
544
decoded_account_key = _decode_base64_to_bytes(account_key)
545
if isinstance(string_to_sign, _unicode_type):
546
string_to_sign = string_to_sign.encode('utf-8')
547
signed_hmac_sha256 = hmac.HMAC(
548
decoded_account_key, string_to_sign, hashlib.sha256)
549
digest = signed_hmac_sha256.digest()
550
encoded_digest = _encode_base64(digest)
551
return encoded_digest
554
def _to_python_bool(value):
555
if value.lower() == 'true':
560
def _to_entity_int(data):
561
int_max = (2 << 30) - 1
562
if data > (int_max) or data < (int_max + 1) * (-1):
563
return 'Edm.Int64', str(data)
565
return 'Edm.Int32', str(data)
568
def _to_entity_bool(value):
570
return 'Edm.Boolean', 'true'
571
return 'Edm.Boolean', 'false'
574
def _to_entity_datetime(value):
575
return 'Edm.DateTime', value.strftime('%Y-%m-%dT%H:%M:%S')
578
def _to_entity_float(value):
579
return 'Edm.Double', str(value)
582
def _to_entity_property(value):
583
if value.type == 'Edm.Binary':
584
return value.type, _encode_base64(value.value)
586
return value.type, str(value.value)
589
def _to_entity_none(value):
593
def _to_entity_str(value):
594
return 'Edm.String', value
597
# Tables of conversions to and from entity types. We support specific
598
# datatypes, and beyond that the user can use an EntityProperty to get
599
# custom data type support.
601
def _from_entity_binary(value):
602
return EntityProperty('Edm.Binary', _decode_base64_to_bytes(value))
605
def _from_entity_int(value):
609
def _from_entity_datetime(value):
610
format = '%Y-%m-%dT%H:%M:%S'
612
format = format + '.%f'
613
if value.endswith('Z'):
614
format = format + 'Z'
615
return datetime.strptime(value, format)
617
_ENTITY_TO_PYTHON_CONVERSIONS = {
618
'Edm.Binary': _from_entity_binary,
619
'Edm.Int32': _from_entity_int,
620
'Edm.Int64': _from_entity_int,
622
'Edm.Boolean': _to_python_bool,
623
'Edm.DateTime': _from_entity_datetime,
626
# Conversion from Python type to a function which returns a tuple of the
627
# type string and content string.
628
_PYTHON_TO_ENTITY_CONVERSIONS = {
630
bool: _to_entity_bool,
631
datetime: _to_entity_datetime,
632
float: _to_entity_float,
633
EntityProperty: _to_entity_property,
637
if sys.version_info < (3,):
638
_PYTHON_TO_ENTITY_CONVERSIONS.update({
639
long: _to_entity_int,
640
types.NoneType: _to_entity_none,
641
unicode: _to_entity_str,
645
def _convert_entity_to_xml(source):
646
''' Converts an entity object to xml to send.
648
The entity format is:
649
<entry xmlns:d="http://schemas.microsoft.com/ado/2007/08/dataservices" xmlns:m="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata" xmlns="http://www.w3.org/2005/Atom">
651
<updated>2008-09-18T23:46:19.3857256Z</updated>
656
<content type="application/xml">
658
<d:Address>Mountain View</d:Address>
659
<d:Age m:type="Edm.Int32">23</d:Age>
660
<d:AmountDue m:type="Edm.Double">200.23</d:AmountDue>
661
<d:BinaryData m:type="Edm.Binary" m:null="true" />
662
<d:CustomerCode m:type="Edm.Guid">c9da6455-213d-42c9-9a79-3e9149a57833</d:CustomerCode>
663
<d:CustomerSince m:type="Edm.DateTime">2008-07-10T00:00:00</d:CustomerSince>
664
<d:IsActive m:type="Edm.Boolean">true</d:IsActive>
665
<d:NumOfOrders m:type="Edm.Int64">255</d:NumOfOrders>
666
<d:PartitionKey>mypartitionkey</d:PartitionKey>
667
<d:RowKey>myrowkey1</d:RowKey>
668
<d:Timestamp m:type="Edm.DateTime">0001-01-01T00:00:00</d:Timestamp>
674
# construct the entity body included in <m:properties> and </m:properties>
675
entity_body = '<m:properties xml:space="preserve">{properties}</m:properties>'
677
if isinstance(source, WindowsAzureData):
678
source = vars(source)
682
# set properties type for types we know if value has no type info.
683
# if value has type info, then set the type to value.type
684
for name, value in source.items():
686
conv = _PYTHON_TO_ENTITY_CONVERSIONS.get(type(value))
687
if conv is None and sys.version_info >= (3,) and value is None:
688
conv = _to_entity_none
690
raise WindowsAzureError(
691
_ERROR_CANNOT_SERIALIZE_VALUE_TO_ENTITY.format(
692
type(value).__name__))
694
mtype, value = conv(value)
696
# form the property node
697
properties_str += ''.join(['<d:', name])
699
properties_str += ' m:null="true" />'
702
properties_str += ''.join([' m:type="', mtype, '"'])
703
properties_str += ''.join(['>',
704
xml_escape(value), '</d:', name, '>'])
706
if sys.version_info < (3,):
707
if isinstance(properties_str, unicode):
708
properties_str = properties_str.encode(encoding='utf-8')
710
# generate the entity_body
711
entity_body = entity_body.format(properties=properties_str)
712
xmlstr = _create_entry(entity_body)
716
def _convert_table_to_xml(table_name):
718
Create xml to send for a given table name. Since xml format for table is
719
the same as entity and the only difference is that table has only one
720
property 'TableName', so we just call _convert_entity_to_xml.
722
table_name: the name of the table
724
return _convert_entity_to_xml({'TableName': table_name})
727
def _convert_block_list_to_xml(block_id_list):
729
Convert a block list to xml to send.
732
a str list containing the block ids that are used in put_block_list.
733
Only get block from latest blocks.
735
if block_id_list is None:
737
xml = '<?xml version="1.0" encoding="utf-8"?><BlockList>'
738
for value in block_id_list:
739
xml += '<Latest>{0}</Latest>'.format(_encode_base64(value))
741
return xml + '</BlockList>'
744
def _create_blob_result(response):
745
blob_properties = _parse_response_for_dict(response)
746
return BlobResult(response.body, blob_properties)
749
def _convert_response_to_block_list(response):
751
Converts xml response to block list class.
753
blob_block_list = BlobBlockList()
755
xmldoc = minidom.parseString(response.body)
756
for xml_block in _get_children_from_path(xmldoc,
760
xml_block_id = _decode_base64_to_text(
761
_get_child_nodes(xml_block, 'Name')[0].firstChild.nodeValue)
762
xml_block_size = int(
763
_get_child_nodes(xml_block, 'Size')[0].firstChild.nodeValue)
764
blob_block_list.committed_blocks.append(
765
BlobBlock(xml_block_id, xml_block_size))
767
for xml_block in _get_children_from_path(xmldoc,
771
xml_block_id = _decode_base64_to_text(
772
_get_child_nodes(xml_block, 'Name')[0].firstChild.nodeValue)
773
xml_block_size = int(
774
_get_child_nodes(xml_block, 'Size')[0].firstChild.nodeValue)
775
blob_block_list.uncommitted_blocks.append(
776
BlobBlock(xml_block_id, xml_block_size))
778
return blob_block_list
781
def _remove_prefix(name):
782
colon = name.find(':')
784
return name[colon + 1:]
788
def _convert_response_to_entity(response):
791
return _convert_xml_to_entity(response.body)
794
def _convert_xml_to_entity(xmlstr):
795
''' Convert xml response to entity.
797
The format of entity:
798
<entry xmlns:d="http://schemas.microsoft.com/ado/2007/08/dataservices" xmlns:m="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata" xmlns="http://www.w3.org/2005/Atom">
800
<updated>2008-09-18T23:46:19.3857256Z</updated>
805
<content type="application/xml">
807
<d:Address>Mountain View</d:Address>
808
<d:Age m:type="Edm.Int32">23</d:Age>
809
<d:AmountDue m:type="Edm.Double">200.23</d:AmountDue>
810
<d:BinaryData m:type="Edm.Binary" m:null="true" />
811
<d:CustomerCode m:type="Edm.Guid">c9da6455-213d-42c9-9a79-3e9149a57833</d:CustomerCode>
812
<d:CustomerSince m:type="Edm.DateTime">2008-07-10T00:00:00</d:CustomerSince>
813
<d:IsActive m:type="Edm.Boolean">true</d:IsActive>
814
<d:NumOfOrders m:type="Edm.Int64">255</d:NumOfOrders>
815
<d:PartitionKey>mypartitionkey</d:PartitionKey>
816
<d:RowKey>myrowkey1</d:RowKey>
817
<d:Timestamp m:type="Edm.DateTime">0001-01-01T00:00:00</d:Timestamp>
822
xmldoc = minidom.parseString(xmlstr)
824
xml_properties = None
825
for entry in _get_child_nodes(xmldoc, 'entry'):
826
for content in _get_child_nodes(entry, 'content'):
828
xml_properties = _get_child_nodesNS(
829
content, METADATA_NS, 'properties')
831
if not xml_properties:
835
# extract each property node and get the type from attribute and node value
836
for xml_property in xml_properties[0].childNodes:
837
name = _remove_prefix(xml_property.nodeName)
838
# exclude the Timestamp since it is auto added by azure when
839
# inserting entity. We don't want this to mix with real properties
840
if name in ['Timestamp']:
843
if xml_property.firstChild:
844
value = xml_property.firstChild.nodeValue
848
isnull = xml_property.getAttributeNS(METADATA_NS, 'null')
849
mtype = xml_property.getAttributeNS(METADATA_NS, 'type')
851
# if not isnull and no type info, then it is a string and we just
852
# need the str type to hold the property.
853
if not isnull and not mtype:
854
_set_entity_attr(entity, name, value)
855
elif isnull == 'true':
857
property = EntityProperty(mtype, None)
859
property = EntityProperty('Edm.String', None)
860
else: # need an object to hold the property
861
conv = _ENTITY_TO_PYTHON_CONVERSIONS.get(mtype)
863
property = conv(value)
865
property = EntityProperty(mtype, value)
866
_set_entity_attr(entity, name, property)
868
# extract id, updated and name value from feed entry and set them of
870
for name, value in _get_entry_properties(xmlstr, True).items():
872
_set_entity_attr(entity, name, value)
877
def _set_entity_attr(entity, name, value):
879
setattr(entity, name, value)
880
except UnicodeEncodeError:
881
# Python 2 doesn't support unicode attribute names, so we'll
882
# add them and access them directly through the dictionary
883
entity.__dict__[name] = value
886
def _convert_xml_to_table(xmlstr):
887
''' Converts the xml response to table class.
888
Simply call convert_xml_to_entity and extract the table name, and add
889
updated and author info
892
entity = _convert_xml_to_entity(xmlstr)
893
setattr(table, 'name', entity.TableName)
894
for name, value in _get_entry_properties(xmlstr, False).items():
895
setattr(table, name, value)
899
def _storage_error_handler(http_error):
900
''' Simple error handler for storage service. '''
901
return _general_error_handler(http_error)
903
# make these available just from storage.
904
from azure.storage.blobservice import BlobService
905
from azure.storage.queueservice import QueueService
906
from azure.storage.tableservice import TableService
907
from azure.storage.cloudstorageaccount import CloudStorageAccount
908
from azure.storage.sharedaccesssignature import (
909
SharedAccessSignature,
added
removed
azure-sdk-for-python-master/azure/storage/__init__.py
