169.1.1
by Natalia
- Added proper copyright notices and coding headers. |
1 |
# -*- coding: utf-8 -*-
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
2 |
#
|
144.1.1
by Rodney Dawes
Exception for use of OpenSSL |
3 |
# Copyright 2009-2012 Canonical Ltd.
|
169.2.1
by Natalia
Cleanup. |
4 |
# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
|
22.1.1
by Rodney Dawes
Add the AGPLv3 text to COPYING and in all the source header comments |
5 |
#
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
6 |
# This program is free software: you can redistribute it and/or modify it
|
22.1.1
by Rodney Dawes
Add the AGPLv3 text to COPYING and in all the source header comments |
7 |
# under the terms of the GNU Affero General Public License version 3,
|
8 |
# as published by the Free Software Foundation.
|
|
9 |
#
|
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
10 |
# This program is distributed in the hope that it will be useful, but
|
11 |
# WITHOUT ANY WARRANTY; without even the implied warranties of
|
|
12 |
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
|
|
22.1.1
by Rodney Dawes
Add the AGPLv3 text to COPYING and in all the source header comments |
13 |
# PURPOSE. See the GNU Affero General Public License for more details.
|
14 |
#
|
|
15 |
# You should have received a copy of the GNU Affero General Public License
|
|
16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
144.1.1
by Rodney Dawes
Exception for use of OpenSSL |
17 |
#
|
18 |
# In addition, as a special exception, the copyright holders give
|
|
19 |
# permission to link the code of portions of this program with the
|
|
20 |
# OpenSSL library under certain conditions as described in each
|
|
21 |
# individual source file, and distribute linked combinations
|
|
22 |
# including the two.
|
|
23 |
# You must obey the GNU General Public License in all respects
|
|
24 |
# for all of the code used other than OpenSSL. If you modify
|
|
25 |
# file(s) with this exception, you may extend this exception to your
|
|
26 |
# version of the file(s), but you are not obligated to do so. If you
|
|
27 |
# do not wish to do so, delete this exception statement from your
|
|
28 |
# version. If you delete this exception statement from all source
|
|
29 |
# files in the program, then also delete it here.
|
|
169.2.1
by Natalia
Cleanup. |
30 |
|
31 |
"""A simple client with some tests."""
|
|
32 |
||
10.1.1
by Rodney Dawes
Fix some lint issues |
33 |
import os |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
34 |
import time |
35 |
import math |
|
36 |
import random |
|
37 |
import zlib |
|
38 |
||
39 |
from twisted.internet import reactor, defer |
|
40 |
||
172.1.1
by Natalia
- Renamed python module to magicicadaprotocol, removing the need of the namespace package "ubuntuone". |
41 |
from magicicadaprotocol import request, protocol_pb2 |
42 |
from magicicadaprotocol.client import ( |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
43 |
StorageClientFactory, StorageClient) |
172.1.1
by Natalia
- Renamed python module to magicicadaprotocol, removing the need of the namespace package "ubuntuone". |
44 |
from magicicadaprotocol.dircontent_pb2 import ( |
169.2.1
by Natalia
Cleanup. |
45 |
DirectoryContent, DIRECTORY) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
46 |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
47 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
48 |
class NotDirectory(Exception): |
49 |
"""This wasnt a directory"""
|
|
50 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
51 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
52 |
def delay_time(step): |
53 |
"""generates a delay for each step"""
|
|
169.2.1
by Natalia
Cleanup. |
54 |
return ((math.exp(step) - 1) / 10) / (1 + random.random()) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
55 |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
56 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
57 |
def retry(function): |
58 |
"""This function will be retried when it raises TRY_AGAIN from the server.
|
|
59 |
"""
|
|
60 |
def inner(self, *args, **kwargs): |
|
61 |
"""decorated."""
|
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
62 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
63 |
def do_retry(failure, step): |
64 |
"""retry."""
|
|
65 |
if failure.check(request.StorageRequestError) \ |
|
66 |
and failure.value.error_message.error.type == \ |
|
67 |
protocol_pb2.Error.TRY_AGAIN: |
|
68 |
print "retry", args |
|
69 |
d = defer.Deferred() |
|
70 |
reactor.callLater(delay_time(step), d.callback, None) |
|
71 |
d.addCallback(lambda _: function(self, *args, **kwargs)) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
72 |
d.addErrback(do_retry, step + 1) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
73 |
return d |
74 |
else: |
|
75 |
return failure |
|
76 |
d = function(self, *args, **kwargs) |
|
77 |
d.addErrback(do_retry, 0) |
|
78 |
return d |
|
79 |
return inner |
|
80 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
81 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
82 |
class EasyClient(StorageClient): |
83 |
"""Simple client that calls a callback on connection."""
|
|
84 |
||
85 |
def __init__(self): |
|
86 |
"""create a client."""
|
|
87 |
StorageClient.__init__(self) |
|
88 |
self.cwd = "/" |
|
89 |
self.cwd_id = None |
|
90 |
||
91 |
def connectionMade(self): |
|
92 |
"""Setup and call callback."""
|
|
93 |
StorageClient.connectionMade(self) |
|
94 |
self.factory.clientConnectionMade(self) |
|
95 |
||
96 |
def _get_hash(self, node_id): |
|
97 |
"""Get the hash of node_id."""
|
|
98 |
def _got_query(query): |
|
99 |
"""deferred part."""
|
|
100 |
message = query[0][1].response[0] |
|
101 |
return message.hash |
|
102 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
103 |
d = self.query([(request.ROOT, node_id, request.UNKNOWN_HASH)]) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
104 |
d.addCallback(_got_query) |
105 |
return d |
|
106 |
||
107 |
def _get_content(self, node): |
|
108 |
"""get content"""
|
|
109 |
d = self._get_hash(node) |
|
110 |
d.addCallback(lambda hash: self.get_content(request.ROOT, node, hash)) |
|
111 |
d.addCallback(lambda result: zlib.decompress(result.data)) |
|
112 |
return d |
|
113 |
||
114 |
def get_cwd_id(self): |
|
115 |
"""get current working directory."""
|
|
116 |
d = defer.Deferred() |
|
117 |
if self.cwd_id is None: |
|
118 |
d.addCallback(lambda _: self.get_root()) |
|
119 |
d.addCallback(lambda result: setattr(self, "cwd_id", result)) |
|
120 |
d.callback(None) |
|
121 |
else: |
|
122 |
d.callback(self.cwd_id) |
|
123 |
return d |
|
124 |
||
125 |
def chdir(self, path): |
|
126 |
"""change cwd to path."""
|
|
127 |
full_path = os.path.join(self.cwd, path) |
|
120.1.1
by Rodney Dawes
Update pylintrc and {disable,enable}-msg comments to work with new pylint |
128 |
parts = [part for part in full_path.split("/") if part] |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
129 |
|
130 |
def is_dir_return_id(parent, name): |
|
131 |
"""check that this part of the path is a directory and return its
|
|
132 |
id.
|
|
133 |
"""
|
|
134 |
d = self._get_content(parent) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
135 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
136 |
def is_directory(content): |
137 |
"""parse dircontent to find a directory with name == name."""
|
|
138 |
unserialized_content = DirectoryContent() |
|
139 |
unserialized_content.ParseFromString(content) |
|
140 |
for entry in unserialized_content.entries: |
|
141 |
if entry.name == name and entry.node_type == DIRECTORY: |
|
142 |
print "is directory", entry |
|
143 |
return entry.node |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
144 |
raise NotDirectory("name %s is not a directory" % name) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
145 |
d.addCallback(is_directory) |
146 |
return d |
|
147 |
||
148 |
d = self.get_root() |
|
149 |
for part in parts: |
|
150 |
d.addCallback(is_dir_return_id, part) |
|
151 |
d.addCallback(lambda x: setattr(self, "cwd_id", x)) |
|
152 |
d.addCallback(lambda x: setattr(self, "cwd", full_path)) |
|
153 |
return d |
|
154 |
||
155 |
@retry
|
|
156 |
def mkfile(self, name): |
|
157 |
"""make a file named name in cwd."""
|
|
158 |
d = self.get_cwd_id() |
|
169.2.1
by Natalia
Cleanup. |
159 |
d.addCallback( |
160 |
lambda _: self.make_file(request.ROOT, self.cwd_id, name)) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
161 |
return d |
162 |
||
163 |
def mkdir(self, name): |
|
164 |
"""make a dir named name in cwd."""
|
|
165 |
d = self.get_cwd_id() |
|
169.2.1
by Natalia
Cleanup. |
166 |
d.addCallback( |
167 |
lambda _: self.make_dir(request.ROOT, self.cwd_id, name)) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
168 |
return d |
169 |
||
170 |
def put(self, name, content): |
|
171 |
"""put content into file named name."""
|
|
172 |
pass
|
|
173 |
||
174 |
def get(self, name): |
|
175 |
"""get content from file names name."""
|
|
176 |
d = self.get_id(name) |
|
177 |
d.addCallback(self._get_content) |
|
178 |
return d |
|
179 |
||
180 |
def unlink(self, name): |
|
181 |
"""unlink file named name."""
|
|
182 |
d = self.get_id(name) |
|
183 |
d.addCallback(self.unlink) |
|
184 |
return d |
|
185 |
||
186 |
def listdir(self): |
|
187 |
"""get a dircontent list for cwd."""
|
|
188 |
d = self.get_cwd() |
|
189 |
d.addCallback(self._get_content) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
190 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
191 |
def make_listdir(content): |
192 |
"""parse dircontents."""
|
|
193 |
result = [] |
|
194 |
unserialized_content = DirectoryContent() |
|
195 |
unserialized_content.ParseFromString(content) |
|
196 |
for entry in unserialized_content.entries: |
|
197 |
result.append(entry) |
|
198 |
return result |
|
199 |
d.addCallback(make_listdir) |
|
200 |
return d |
|
201 |
||
202 |
def move(self, name, path, new_name=None): |
|
203 |
"""move file."""
|
|
204 |
if new_name is None: |
|
205 |
new_name = name |
|
206 |
||
207 |
||
208 |
class EasyClientFactory(StorageClientFactory): |
|
209 |
"""A test oriented protocol factory."""
|
|
210 |
protocol = EasyClient |
|
211 |
||
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
212 |
def __init__(self, deferrer): |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
213 |
"""create a factory."""
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
214 |
self.client = None |
215 |
self.defer = deferrer |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
216 |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
217 |
def clientConnectionMade(self, client_obj): |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
218 |
"""on client connection made."""
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
219 |
self.client = client_obj |
220 |
self.defer.callback(self.client) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
221 |
|
222 |
def clientConnectionFailed(self, connector, reason): |
|
223 |
"""We failed at connecting."""
|
|
224 |
self.defer.errback(reason) |
|
225 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
226 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
227 |
def client(host, port): |
228 |
"""return a deferred that will succeed with a connected client."""
|
|
229 |
d = defer.Deferred() |
|
230 |
factory = EasyClientFactory(d) |
|
231 |
reactor.connectTCP(host, port, factory) |
|
232 |
return d |
|
233 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
234 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
235 |
def authenticated_client(host, port, token="open sesame"): |
236 |
"""return a deferred that will succeed with an authenticated client."""
|
|
237 |
d = client(host, port) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
238 |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
239 |
def auth(client_obj): |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
240 |
"""do the auth."""
|
241 |
d = client.dummy_authenticate(token) |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
242 |
d.addCallback(lambda _: client_obj) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
243 |
return d |
244 |
d.addCallback(auth) |
|
245 |
return d |
|
246 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
247 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
248 |
def skip_error(failure, error): |
249 |
"""try: except $error: pass errback"""
|
|
250 |
if failure.check(request.StorageRequestError) and \ |
|
251 |
failure.value.error_message.error.type == error: |
|
252 |
return
|
|
253 |
else: |
|
254 |
return failure |
|
255 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
256 |
|
166.1.2
by Facundo Batista
Aesthetic details. |
257 |
def skip_result(_, f, *args, **kwargs): |
258 |
"""Deferred utilities."""
|
|
259 |
return f(*args, **kwargs) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
260 |
|
261 |
||
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
262 |
def sr_result(result, f, *args, **kwargs): |
263 |
"""skip the result when calling the function and then return it."""
|
|
264 |
f(*args, **kwargs) |
|
265 |
return result |
|
266 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
267 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
268 |
def log(*args, **kwargs): |
269 |
"""print args and kwargs."""
|
|
270 |
for arg in args: |
|
271 |
print arg, |
|
272 |
print kwargs |
|
273 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
274 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
275 |
def show_error(failure): |
276 |
"""print the traceback."""
|
|
277 |
print failure.getTraceback() |
|
278 |
||
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
279 |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
280 |
if __name__ == "__main__": |
281 |
||
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
282 |
def create_dirs(client_obj, num_dirs): |
283 |
"""Create directories."""
|
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
284 |
d = defer.succeed(None) |
285 |
for i in range(0, num_dirs): |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
286 |
d.addCallback(skip_result, client_obj.mkdir, "%s" % (i)) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
287 |
d.addErrback(skip_error, protocol_pb2.Error.ALREADY_EXISTS) |
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
288 |
d.addCallback(skip_result, log, "Directory %s created." % i) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
289 |
return d |
290 |
||
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
291 |
def make_files_client(client_obj, number, num_files): |
292 |
"""Make files."""
|
|
293 |
d = client_obj.chdir("%s" % (number)) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
294 |
for i in range(0, num_files): |
295 |
d.addCallback(skip_result, log, "Client %s creating file %s." |
|
296 |
% (number, i)) |
|
121.1.1
by Rodney Dawes
Remove contrib contents and pylintrc |
297 |
d.addCallback(skip_result, client.mkfile, "%s" % (i)) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
298 |
d.addCallback(skip_result, log, "Client %s created file %s." |
299 |
% (number, i)) |
|
300 |
return d |
|
301 |
||
302 |
NUM_CLIENTS = 200 |
|
303 |
NUM_FILES = 50 |
|
304 |
||
172.1.1
by Natalia
- Renamed python module to magicicadaprotocol, removing the need of the namespace package "ubuntuone". |
305 |
port_num = int(open("tmp/magicicada-api.port").read()) |
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
306 |
deferred = authenticated_client("localhost", int(port_num)) |
307 |
deferred.addCallback(create_dirs, NUM_CLIENTS) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
308 |
|
309 |
def fire_clients(_): |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
310 |
"""Fire off the client connections."""
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
311 |
dlist = [] |
312 |
for i in range(NUM_CLIENTS): |
|
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
313 |
d = authenticated_client("localhost", int(port_num)) |
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
314 |
d.addCallback(sr_result, log, "client", i, "logged in") |
315 |
d.addCallback(make_files_client, i, NUM_FILES) |
|
316 |
d.addErrback(show_error) |
|
317 |
dlist.append(d) |
|
318 |
||
319 |
return defer.DeferredList(dlist) |
|
320 |
||
121.1.3
by Rodney Dawes
Use pyflakes instead of pylint for now due to metaclass complications |
321 |
deferred.addCallback(fire_clients) |
322 |
deferred.addCallback(lambda _: reactor.stop()) |
|
323 |
deferred.addErrback(show_error) |
|
2
by Rodney Dawes
Import the ubuntuone-storage-protocol code |
324 |
print "Starting reactor" |
325 |
start_time = time.time() |
|
326 |
reactor.run() |