1
# Copyright (c) 2010-2011 OpenStack, LLC.
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
7
# http://www.apache.org/licenses/LICENSE-2.0
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
17
from urllib import quote, unquote
18
from json import loads as json_loads
20
from swift.common.compressing_file_reader import CompressingFileReader
21
from swift.proxy.server import BaseApplication
24
class MemcacheStub(object):
26
def get(self, *a, **kw): # pragma: no cover
29
def set(self, *a, **kw): # pragma: no cover
32
def incr(self, *a, **kw): # pragma: no cover
35
def delete(self, *a, **kw): # pragma: no cover
38
def set_multi(self, *a, **kw): # pragma: no cover
41
def get_multi(self, *a, **kw): # pragma: no cover
45
def make_request_body_file(source_file, compress=True):
46
if hasattr(source_file, 'seek'):
49
source_file = open(source_file, 'rb')
51
compressed_file = CompressingFileReader(source_file)
52
return compressed_file
56
def webob_request_copy(orig_req, source_file=None, compress=True):
57
req_copy = orig_req.copy()
59
req_copy.body_file = make_request_body_file(source_file,
61
req_copy.content_length = orig_req.content_length
65
class InternalProxy(object):
67
Set up a private instance of a proxy server that allows normal requests
68
to be made without having to actually send the request to the proxy.
69
This also doesn't log the requests to the normal proxy logs.
71
:param proxy_server_conf: proxy server configuration dictionary
72
:param logger: logger to log requests to
73
:param retries: number of times to retry each request
76
def __init__(self, proxy_server_conf=None, logger=None, retries=0):
77
self.upload_app = BaseApplication(proxy_server_conf,
78
memcache=MemcacheStub(),
80
self.retries = retries
82
def _handle_request(self, req, source_file=None, compress=True):
83
req = self.upload_app.update_request(req)
84
req_copy = webob_request_copy(req, source_file=source_file,
86
resp = self.upload_app.handle_request(req_copy)
88
while (resp.status_int < 200 or resp.status_int > 299) \
89
and tries < self.retries:
90
req_copy = webob_request_copy(req, source_file=source_file,
92
resp = self.upload_app.handle_request(req_copy)
96
def upload_file(self, source_file, account, container, object_name,
97
compress=True, content_type='application/x-gzip',
100
Upload a file to cloud files.
102
:param source_file: path to or file like object to upload
103
:param account: account to upload to
104
:param container: container to upload to
105
:param object_name: name of object being uploaded
106
:param compress: if True, compresses object as it is uploaded
107
:param content_type: content-type of object
108
:param etag: etag for object to check successful upload
109
:returns: True if successful, False otherwise
111
target_name = '/v1/%s/%s/%s' % (account, container, object_name)
113
# create the container
114
if not self.create_container(account, container):
117
# upload the file to the account
118
req = webob.Request.blank(target_name, content_type=content_type,
119
environ={'REQUEST_METHOD': 'PUT'},
120
headers={'Transfer-Encoding': 'chunked'})
121
req.content_length = None # to make sure we send chunked data
123
req.headers['etag'] = etag
124
resp = self._handle_request(req, source_file=source_file,
126
if not (200 <= resp.status_int < 300):
130
def get_object(self, account, container, object_name):
134
:param account: account name object is in
135
:param container: container name object is in
136
:param object_name: name of object to get
137
:returns: iterator for object data
139
req = webob.Request.blank('/v1/%s/%s/%s' %
140
(account, container, object_name),
141
environ={'REQUEST_METHOD': 'GET'})
142
resp = self._handle_request(req)
143
return resp.status_int, resp.app_iter
145
def create_container(self, account, container):
149
:param account: account name to put the container in
150
:param container: container name to create
151
:returns: True if successful, otherwise False
153
req = webob.Request.blank('/v1/%s/%s' % (account, container),
154
environ={'REQUEST_METHOD': 'PUT'})
155
resp = self._handle_request(req)
156
return 200 <= resp.status_int < 300
158
def get_container_list(self, account, container, marker=None,
159
end_marker=None, limit=None, prefix=None,
160
delimiter=None, full_listing=True):
162
Get a listing of objects for the container.
164
:param account: account name for the container
165
:param container: container name to get a listing for
166
:param marker: marker query
167
:param end_marker: end marker query
168
:param limit: limit query
169
:param prefix: prefix query
170
:param delimeter: string to delimit the queries on
171
:param full_listing: if True, return a full listing, else returns a max
173
:returns: list of objects
177
listing = self.get_container_list(account, container, marker,
178
end_marker, limit, prefix,
179
delimiter, full_listing=False)
183
marker = listing[-1]['name']
185
marker = listing[-1].get('name', listing[-1].get('subdir'))
186
listing = self.get_container_list(account, container, marker,
187
end_marker, limit, prefix,
191
path = '/v1/%s/%s' % (account, quote(container))
194
qs += '&marker=%s' % quote(marker)
196
qs += '&end_marker=%s' % quote(end_marker)
198
qs += '&limit=%d' % limit
200
qs += '&prefix=%s' % quote(prefix)
202
qs += '&delimiter=%s' % quote(delimiter)
204
req = webob.Request.blank(path, environ={'REQUEST_METHOD': 'GET'})
205
resp = self._handle_request(req)
206
if resp.status_int < 200 or resp.status_int >= 300:
207
return [] # TODO: distinguish between 404 and empty container
208
if resp.status_int == 204:
210
return json_loads(resp.body)
added
removed
swift/common/internal_proxy.py
