~ipitech/jett/benchmark-framework

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
#!/usr/bin/env python

"""
Main benchmark runner.
Try `python runner.py --help`
"""

import os
import sys
from optparse import OptionParser
from time import time
try:
    import cPickle as pickle
except ImportError:
    import pickle
from loader import BenchmarkLoader


def handle_args(args):
    parser = OptionParser(usage='%prog [options] module1 [module2, ..., moduleN]')
    (options, args) = parser.parse_args()
    
    if not args:
        parser.error('At least one benchmark name argument required.')
    
    return options, args

STATE_NORMAL = 0
STATE_DONE = 1
STATE_ERROR = 2

def run_benchmark_sandboxed(benchmark):
    host_pipe, fork_pipe = os.pipe()
    pid = os.fork()
    
    if pid: 
        # we are in parent process (pid != 0)
        pid, status = os.waitpid(pid, 0)
        length = int(os.read(host_pipe, 32))
        datastr = os.read(host_pipe, length)
        data = pickle.loads(datastr)
        if data['state'] == STATE_DONE:
            print '    Min: %0.6fs, Max: %0.6fs' % (min(data['times']), max(data['times']))
    else:
        # we are in fork (pid == 0)
        state = STATE_NORMAL
        times = []
        
        for i in xrange(10):
            st = time()
            try:
                benchmark()
            except:
                import traceback
                print '\n   ', '\n    '.join(traceback.format_exc().split('\n'))
                state = STATE_ERROR
                break
            else:
                times.append(time()-st)
        
        if state == STATE_NORMAL:
            state = STATE_DONE
        
        data = pickle.dumps({'state': state, 'times': times})
        os.write(fork_pipe, '%-32d' % len(data))
        os.write(fork_pipe, data)
        sys.exit(0)


def main(args):
    (options, args) = handle_args(args)
    
    loader = BenchmarkLoader()
    
    suites = []
    for arg in args:
        suites.extend(loader.load_from_name(arg))
    
    for suite in suites:
        suite_instance = suite()
        print 'Running suite %s' % suite.__name__
        for benchmark in suite_instance.benchmarks:
            print '  %s:' % benchmark.__name__
            run_benchmark_sandboxed(benchmark)
    
    return 0


if __name__ == '__main__':
    sys.exit(main(sys.argv))