1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
#!/usr/bin/python3
"""This script takes memevent-test result json files and uploads the
interesting data contained within to a nfss backend.
It will process any memory result files found in 'source_file_path matching the
naming scheme: "memory_usage_*.json"
"""
import datetime
import json
import os
import re
import sys
import subprocess
from collections import defaultdict
from glob import glob
source_file_path = ""
upload_script = ""
nfss_config = ""
def _get_run_details(app_name):
return dict(
image_arch='',
ran_at=datetime.datetime.utcnow().isoformat(),
package_version=_get_package_version(app_name),
application_name=app_name,
)
def _get_package_version(appname):
"""Return the package version for application *appname*.
Return empty string if appname details are not found.
"""
try:
dpkg_output = subprocess.check_output(
['adb', 'shell', 'dpkg', '-s', appname],
universal_newlines=True
)
version_line = [
l for l in dpkg_output.split('\n') if l.startswith('Version:')
]
return version_line[0].split()[1]
except (subprocess.CalledProcessError, IndexError):
return "Unknown"
def upload_json_details(run_details, app_details):
app_run_details = run_details.copy()
app_run_details["events"] = app_details
_upload_data(run_details['application_name'], app_run_details)
def _upload_data(test_name, run_json):
try:
run_json_string = json.dumps(run_json)
except ValueError as e:
print("Error: Data does not appear to be valid json: %s" % e)
sys.exit(3)
print("Uploading data for :memevent:-", test_name)
global upload_script
global nfss_config
try:
upload_process = subprocess.Popen(
[upload_script, nfss_config, 'memevent', test_name],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = upload_process.communicate(
input=run_json_string.encode()
)
print("stdout: {}\n\nstderr: {}".format(stdout, stderr))
except Exception as e:
print("Something went terribly wrong: ", e)
if upload_process.returncode != 0:
raise subprocess.CalledProcessError('Failed to upload to nfss')
def _get_files_app_name_and_test(filename):
"""return tuple containing (appname, testname)."""
app_name_search = re.search(
'memory_usage_([a-zA-Z-]*).(.*)\.json',
filename
)
return (app_name_search.group(1), app_name_search.group(2))
def get_application_readings(json_data):
app_results = []
pids = json_data["pids"]
for reading in json_data["readings"]:
results = dict(
event=reading["event"],
start_time=reading["start_time"],
stop_time=reading["stop_time"],
)
# find the data results for this event (based on pid).
for pid in pids:
if str(pid) in reading["data"].keys():
results["data"] = reading["data"][str(pid)]
results["pid"] = pid
break
app_results.append(results)
return app_results
def get_application_results(json_filepath):
with open(json_filepath, "r") as f:
json_data = json.load(f)
return get_application_readings(json_data)
def get_application_details(application_files):
"""For every file this application has grab out the details and return a
list dictionaries containing reading details.
"""
app_result = []
for json_file in application_files:
app_result.extend(get_application_results(json_file))
return app_result
def map_files_to_applications():
"""For any memory result files that exist, return a dictionary whos keys
are the applications name mapped to the file.
We can then produce a single json result for each application regardless of
there being many tests / results for it.
"""
global source_file_path
json_results_filename_pattern = os.path.join(
source_file_path,
"memory_usage_*.json"
)
file_map = defaultdict(list)
for json_result_file in glob(json_results_filename_pattern):
app_name, test_name = _get_files_app_name_and_test(json_result_file)
file_map[app_name].append(json_result_file)
return file_map
def usage():
print("{} <source file path> <nfss upload script> <nfss config file>"
.format(sys.argv[0]))
def main():
if len(sys.argv) != 4:
usage()
exit(1)
global source_file_path
source_file_path = sys.argv[1]
global upload_script
upload_script = sys.argv[2]
global nfss_config
nfss_config = sys.argv[3]
app_details = dict()
file_map = map_files_to_applications()
for app_name in file_map.keys():
app_details[app_name] = get_application_details(file_map[app_name])
for app_name in app_details.keys():
run_details = _get_run_details(app_name)
upload_json_details(run_details, app_details[app_name])
if __name__ == '__main__':
main()
|