← Back to branch summary

~ubuntuone-support/+junk/syncdaemon-perftest

« back to all changes in this revision

Viewing changes to syncdaemon-perftest.py

  • Committer: Roman Yepishev
  • Date: 2012-02-29 11:45:32 UTC
  • Revision ID: roman.yepishev@canonical.com-20120229114532-29e8lgtykg2q54u9
Migrated to statsd, converting shell-based dropbox check to a python one

Show diffs side-by-side

added added

removed removed

Lines of Context:
syncdaemon-perftest.py
4
4
import time
5
5
import os
6
6
import dbus.service
7
 
from dbus.mainloop.glib import DBusGMainLoop
8
 
from shutil import rmtree
9
7
import gobject
10
 
import logging
11
8
import sys
12
9
import optparse
13
 
from socket import socket
14
 
 
15
 
CARBON_SERVER = '192.168.1.20'
16
 
CARBON_PORT = 2003
17
 
 
18
 
def format_speed_value(value):
19
 
    units = ('bps', 'kbps', 'Mbps', 'Gbps')
20
 
    for unit in units:
21
 
        if value < 1000:
22
 
            break
23
 
        value /= 1000
24
 
 
25
 
    return "%d %s" % (value, unit)
26
 
 
27
 
def format_size_value(value):
28
 
    units = ('b', 'KiB', 'MiB', 'GiB', 'TiB')
29
 
    for unit in units:
30
 
        if value < 1024:
31
 
            break
32
 
        value /= 1024
33
 
 
34
 
    return "%d %s" % (value, unit)
35
 
 
36
 
def setup_logging(filename=None):
37
 
    root_formatter = logging.Formatter(
38
 
            fmt="%(asctime)s %(process)d: %(message)s")
39
 
 
40
 
    if filename:
41
 
        root_handler = logging.FileHandler(filename)
42
 
    else:
43
 
        root_handler = logging.StreamHandler(sys.stdout)
44
 
 
45
 
    root_handler.setFormatter(root_formatter)
46
 
 
47
 
    instance = logging.getLogger('testing')
48
 
    instance.propagate = False
49
 
    instance.addHandler(root_handler)
50
 
    instance.setLevel(logging.INFO)
51
 
    return instance
52
 
 
53
 
class SDPerfTest(object):
 
10
from dbus.mainloop.glib import DBusGMainLoop
 
11
from shutil import rmtree
 
12
 
 
13
from u1perftest import PerfTest
 
14
from u1perftest.util import (format_speed_value,
 
15
        format_size_value, setup_logging)
 
16
 
 
17
STATSD_SERVER = '192.168.1.20'
 
18
STATSD_PORT = 8125
 
19
TESTDIR="syncdaemon-perftest-tmp"
 
20
 
 
21
class SDPerfTest(PerfTest):
54
22
    def __init__(self):
55
23
        self.state = 'INIT'
56
24
        self.root = None
57
25
        self.error = False
58
 
        self.logger = None
59
 
        self.timeout = None
60
 
        self.file_size = None
61
 
        self.file_count = None
62
 
 
63
 
        self.bm_upload_timeout = False
64
26
 
65
27
        # "namespace" for benchmarking
66
28
        self.bm_temp_files = {}
163
125
 
164
126
        if 'error' in info and info['error']:
165
127
            self.logger.warn("Warning: Upload of %s did not complete properly", path)
166
 
            self.write_result("upload_error", 1)
167
 
            self.write_result("upload_speed", 0)
 
128
            self.metrics.increment("upload_error")
168
129
            return # for now
169
130
 
170
131
        entry = self.bm_temp_files[path]
179
140
                format_size_value(entry['size']), delta,
180
141
                format_speed_value(speed))
181
142
 
182
 
        self.write_result("upload_speed", speed)
 
143
        if self.track_speed:
 
144
            self.metrics.gauge('upload_speed', speed)
183
145
 
184
146
    def run(self, options):
185
147
        self.logger = setup_logging(options.log)
186
 
        self.result_filename = options.result
187
 
        self.graph_prefix = options.graph_prefix
188
148
        self.file_count = int(options.file_count)
189
149
        self.file_size = int(options.file_size)
 
150
        self.track_speed = options.track_speed
 
151
        self.track_queues = options.track_queues
 
152
 
 
153
        self._setup_metrics(STATSD_SERVER, STATSD_PORT, options.graph_prefix)
 
154
 
190
155
        if options.timeout:
191
156
            self.timeout = int(options.timeout)
192
157
 
194
159
        gobject.timeout_add(0, self.setup)
195
160
 
196
161
        if self.timeout:
197
 
            gobject.timeout_add(self.timeout * 1000, self.handle_TIMEOUT)
 
162
            gobject.timeout_add(self.timeout * 1000,
 
163
                                self.handle_TIMEOUT)
198
164
 
199
165
        self.loop.run()
200
166
 
221
187
 
222
188
    def handle_CONNECTED(self):
223
189
        self.state = 'PRECLEANUP'
224
 
        if os.path.exists(os.path.join(self.root, "a")):
 
190
        if os.path.exists(os.path.join(self.root, TESTDIR)):
225
191
            self.logger.info("Found old test directory. Cleaning up")
226
192
            self.cleanup_files()
227
193
        else:
230
196
 
231
197
    def handle_TIMEOUT(self):
232
198
        self.state = "ERROR"
233
 
        self.bm_upload_timeout = True
234
199
        self.logger.warn("Upload timeout, exiting")
235
 
        self.write_result("timeout_error", 1)
236
 
        self.write_result("upload", 0)
 
200
        self.metrics.increment("error", 1)
 
201
        # fake result
237
202
        self.quit(error=True)
238
203
 
239
204
    def handle_PRECLEANUP_COMPLETED(self):
240
205
        self.state = 'UPLOADING'
241
206
        self.logger.info("Creating files")
242
207
 
243
 
        path_to_file = 'a/b/c/d/e/f'
 
208
        path_to_file = TESTDIR
 
209
 
244
210
        current_path = self.root
245
211
 
246
 
 
247
212
        for node in path_to_file.split('/'):
248
213
            current_path = os.path.join(current_path, node)
249
214
            os.mkdir(current_path)
272
237
 
273
238
            fh.close()
274
239
 
 
240
        rnd.close()
 
241
 
275
242
        self.bm_files_created = time.time()
276
243
        self.logger.info("Files created")
277
244
 
278
245
    def cleanup_files(self):
279
246
        self.logger.info("Moving existing test folder to trash")
280
247
 
281
 
        #paths = self.bm_temp_files.keys()
282
 
        #paths.sort(reverse=True)
283
 
 
284
 
        #for entry in paths:
285
 
        #    logger.debug("Removing %s", entry)
286
 
        #    if os.path.isdir(entry):
287
 
        #        os.rmdir(entry)
288
 
        #    else:
289
 
        #        os.unlink(entry)
290
 
        
291
 
        # We will cheat a bit, moving the "a" folder out of Ubuntu One
292
 
        old = os.path.join(self.root, "a")
 
248
        old = os.path.join(self.root, TESTDIR)
293
249
        new = os.path.join(self.root, "..", "trash")
294
250
 
295
251
        if os.path.exists(new):
311
267
 
312
268
        # how much time did it take?
313
269
        delta = time.time() - self.bm_files_created
314
 
        self.write_result("upload", delta)
315
 
        self.logger.info("Queue completed in %d seconds", delta)
 
270
        rate = self.file_count / delta
 
271
        if self.track_queues:
 
272
            self.metrics.gauge('meta_queue', rate)
 
273
 
 
274
        self.logger.info("Queue completed in %d seconds (%d requests/s)",
 
275
                delta, rate)
316
276
 
317
277
        self.state = 'CLEANUP'
318
278
        self.bm_clenup_started = time.time()
324
284
        test_delta = time.time() - self.bm_test_started
325
285
        cleanup_delta = time.time() - self.bm_clenup_started
326
286
        self.logger.info("Cleanup took %d seconds", cleanup_delta)
327
 
        self.write_result("cleanup", cleanup_delta)
 
287
        self.metrics.gauge('cleanup', cleanup_delta)
328
288
        self.logger.info("End-to-end test took %d seconds", test_delta)
329
289
        self.quit()
330
290
 
334
294
        if error:
335
295
            app.error = True
336
296
 
337
 
    def write_result(self, operation, delta):
338
 
        try:
339
 
            sock = socket()
340
 
 
341
 
            sock.connect( (CARBON_SERVER,CARBON_PORT) )
342
 
 
343
 
            line = "%s.%s %s %d\n" % (self.graph_prefix,
344
 
                    operation, delta, int(time.time()))
345
 
            sock.sendall(line)
346
 
        except Exception, e:
347
 
            self.logger.warn("Error writing to graphite: %s", e)
348
 
 
349
 
        fh = open(self.result_filename, "a")
350
 
        fh.write("%s: %s\n" % (operation, delta))
351
 
        fh.close()
352
 
 
353
297
if __name__ == "__main__":
354
 
 
355
298
    parser = optparse.OptionParser(usage="%prog options")
356
299
    parser.add_option("--log", dest="log", default=None,
357
300
                      help="Log file to write to")
358
 
    parser.add_option("--result", dest="result", 
359
 
                     default="/tmp/result.txt",
360
 
                     help="File to write benchmark results to")
 
301
 
361
302
    parser.add_option("--graph-prefix", dest="graph_prefix",
362
303
                     default="e2eperftest",
363
304
                     help="Prefix for Graphite")
 
305
 
364
306
    parser.add_option("--file-size", dest="file_size",
365
307
                     default=200,
366
308
                     help="Size of the files created")
 
309
 
367
310
    parser.add_option("--file-count", dest="file_count",
368
311
                     default=200,
369
312
                     help="Amount of the files to create")
 
313
 
 
314
    parser.add_option("--track-speed", dest="track_speed",
 
315
                     default=False, action="store_true",
 
316
                     help="Calculate transfer speed info")
 
317
    parser.add_option("--track-queues", dest="track_queues",
 
318
                     default=False, action="store_true",
 
319
                     help="Calculate queue processing speed")
 
320
 
370
321
    parser.add_option("--timeout", dest="timeout",
371
322
                     default=None,
372
323
                     help="Test times out after this number of seconds.")