« back to all changes in this revision
Viewing changes to parse.py
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Natalia Bidart
- Date: 2014-08-20 15:51:09 UTC
- Revision ID: natalia.bidart@ubuntu.com-20140820155109-khs5tt8uchggwrhe
Utilities to parse sca test suite memusage.
- files added:
added
removed
parse.py
1
# -*- coding: utf-8 -*-
2
3
import os
4
import re
5
import sys
6
7
from collections import OrderedDict, defaultdict, namedtuple
8
from matplotlib import pyplot
9
10
PAGE_SIZE_BYTES = 4096
11
12
TEST_NAME = r'(\w+ \([\w\.]+\))'
13
TEST_DOCSTRING = r'[/#,=\(\)\[\]\+\:\.\-\'\w\s]+'
14
TEST_SEP = r' \.\.\. '
15
TEST_RESULT = r'(?:ok|ERROR|FAIL)'
16
MEM_USAGE = r'(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+\.\d+)'
17
18
TEST_RESULT_ONLY = re.compile(TEST_RESULT)
19
TEST_PLUS_MEM = re.compile(TEST_NAME + TEST_SEP + MEM_USAGE)
20
TEST_DOCSTRING_PLUS_MEM = re.compile(TEST_DOCSTRING + TEST_SEP + MEM_USAGE)
21
TEST_ONLY = re.compile(
22
r'%s|%s' % (TEST_NAME + TEST_SEP + TEST_RESULT, TEST_NAME))
23
DOCSTRING_PLUS_RESULT = re.compile(TEST_DOCSTRING + TEST_SEP + TEST_RESULT)
24
MEM_ONLY = re.compile(MEM_USAGE)
25
26
27
Stats = namedtuple('Stats', ['usage', 'percentage', 'res_megs'])
28
29
30
def mem_stats(groups):
31
assert len(groups) == 5
32
percentage = float(groups.pop(-1))
33
34
usage = map(int, groups)
35
assert len(usage) == 4
36
37
res_megs = ((usage[1] * PAGE_SIZE_BYTES) / 1024.) / 1024.
38
return Stats(usage, percentage, res_megs)
39
40
41
def parse_individual(filename):
42
with open(filename) as f:
43
lines = f.readlines()
44
45
result = []
46
for line in lines:
47
match = MEM_ONLY.match(line)
48
if match:
49
stats = mem_stats(list(match.groups()))
50
result.append(stats.res_megs)
51
else:
52
print 'no match!', line
53
54
return result
55
56
def parse_test_id(test_id):
57
test_name, test_path = test_id.split(' ')
58
test_path = test_path.strip('(').strip(')')
59
return '%s.%s' % (test_path, test_name)
60
61
62
def parse_mem_stats(groups, stats, test_id):
63
result = mem_stats(groups)
64
65
app = test_id.split('.')[0]
66
stats.setdefault(app, [])
67
stats[app].append(result.res_megs)
68
69
return result
70
71
72
def parse(filename):
73
with open(filename) as f:
74
lines = f.readlines()
75
76
memory_stats = OrderedDict()
77
last_test_id = 'init.init'
78
combined_stats = OrderedDict()
79
combined_stats[last_test_id] = []
80
for line in lines:
81
line = line.strip()
82
if (not line or TEST_RESULT_ONLY.match(line) or
83
DOCSTRING_PLUS_RESULT.match(line)):
84
continue
85
86
match = TEST_PLUS_MEM.match(line)
87
if match:
88
groups = list(match.groups())
89
last_test_id = parse_test_id(groups.pop(0))
90
stats = parse_mem_stats(groups, memory_stats, last_test_id)
91
# print 'TEST_PLUS_MEM', last_test_id, stats
92
assert last_test_id not in combined_stats
93
combined_stats[last_test_id] = [stats]
94
continue
95
96
match = TEST_ONLY.match(line)
97
if match:
98
groups = filter(None, list(match.groups()))
99
last_test_id = parse_test_id(groups.pop(0))
100
# print 'TEST_ONLY', last_test_id
101
combined_stats[last_test_id] = []
102
continue
103
104
match = TEST_DOCSTRING_PLUS_MEM.match(line)
105
if match:
106
groups = list(match.groups())
107
stats = parse_mem_stats(groups, memory_stats, last_test_id)
108
# print 'TEST_DOCSTRING_PLUS_MEM', stats
109
combined_stats[last_test_id].append(stats)
110
continue
111
112
match = MEM_ONLY.match(line)
113
if match:
114
groups = list(match.groups())
115
stats = parse_mem_stats(groups, memory_stats, last_test_id)
116
# print 'MEM_ONLY', stats
117
combined_stats[last_test_id].append(stats)
118
continue
119
120
###print '!!! not matched:', repr(line)
121
return memory_stats, combined_stats
122
123
124
def annotate(legend, x, y, color='white', alpha=0.5):
125
pyplot.annotate(
126
legend, (x, y), xytext=(-10, 10),
127
textcoords='offset points', ha='right', va='bottom',
128
bbox=dict(boxstyle='round,pad=0.5', fc=color, alpha=alpha),
129
arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0')
130
)
131
132
133
def regular_vs_dummy():
134
filenames = ('fullrun-regular.txt', 'fullrun-dummy.txt')
135
for filename in filenames:
136
memory_stats, _ = parse(filename)
137
all_stats = []
138
i = 0
139
for app, stats in memory_stats.iteritems():
140
all_stats.extend(stats)
141
if stats:
142
annotate(app, i, stats[0], alpha=1.0)
143
i += len(stats)
144
145
pyplot.plot(all_stats, label=filename.split('.')[0])
146
147
pyplot.ylabel('Resident memory (MB)')
148
pyplot.xlabel('ticks')
149
pyplot.legend(loc='upper left', shadow=True)
150
pyplot.show()
151
152
153
def dummy_annotated():
154
memory_stats , combined_stats = parse('fullrun-dummy.txt')
155
all_stats = []
156
for stat in memory_stats.itervalues():
157
all_stats.extend(stat)
158
pyplot.plot(all_stats, '.-')
159
160
def _annotate(test, stats, color, extra=''):
161
legend = '%s\n%s' % tuple(test.split('.')[-2:])
162
y = stats[0].res_megs
163
x = all_stats.index(y)
164
annotate(legend + extra, x, y, color)
165
166
last_stats = None
167
last_test = None
168
for test, stats in combined_stats.iteritems():
169
ticks = len(stats)
170
if not ticks:
171
continue
172
###if last_stats and stats[0][1] < last_stats[0][1]:
173
### # there was a drop in mem usage, let's anotate the test
174
### annotate(last_test, last_stats, color='yellow')
175
if ticks > 5:
176
_annotate(test, stats, color='grey', extra=' (%s ticks)' % ticks)
177
178
last_test = test
179
last_stats = stats
180
181
pyplot.ylabel('Resident memory (MB)')
182
pyplot.xlabel('ticks')
183
pyplot.show()
184
185
186
def app_individuals():
187
filenames = [f for f in os.listdir('.') if f.startswith('memusage-tests')]
188
for filename in filenames:
189
stats = parse_individual(filename)
190
pyplot.plot(stats, label='-'.join(filename.split('-')[2:4]))
191
192
pyplot.ylabel('Resident memory (MB)')
193
pyplot.xlabel('ticks')
194
pyplot.legend(loc='upper left')
195
pyplot.show()
196
197
198
if __name__ == '__main__':
199
# regular_vs_dummy()
200
# dummy_annotated()
201
app_individuals()
Loggerhead is a web-based interface for Breezy
Version: 2.0.1