3
"""This script takes memevent-test result json files and uploads the
4
interesting data contained within to a nfss backend.
6
It will process any memory result files found in 'source_file_path matching the
7
naming scheme: "memory_usage_*.json"
18
from collections import defaultdict
26
def _get_run_details(app_name):
29
ran_at=datetime.datetime.utcnow().isoformat(),
30
package_version=_get_package_version(app_name),
31
application_name=app_name,
35
def _get_package_version(appname):
36
"""Return the package version for application *appname*.
38
Return empty string if appname details are not found.
43
dpkg_output = subprocess.check_output(
44
['adb', 'shell', 'dpkg', '-s', appname],
45
universal_newlines=True
48
l for l in dpkg_output.split('\n') if l.startswith('Version:')
50
return version_line[0].split()[1]
51
except (subprocess.CalledProcessError, IndexError):
55
def upload_json_details(run_details, app_details):
56
app_run_details = run_details.copy()
57
app_run_details["events"] = app_details
59
_upload_data(run_details['application_name'], app_run_details)
62
def _upload_data(test_name, run_json):
64
run_json_string = json.dumps(run_json)
65
except ValueError as e:
66
print("Error: Data does not appear to be valid json: %s" % e)
69
print("Uploading data for :memevent:-", test_name)
73
upload_process = subprocess.Popen(
74
[upload_script, nfss_config, 'memevent', test_name],
75
stdin=subprocess.PIPE,
76
stdout=subprocess.PIPE,
77
stderr=subprocess.PIPE,
79
stdout, stderr = upload_process.communicate(
80
input=run_json_string.encode()
82
print("stdout: {}\n\nstderr: {}".format(stdout, stderr))
83
except Exception as e:
84
print("Something went terribly wrong: ", e)
85
if upload_process.returncode != 0:
86
raise subprocess.CalledProcessError('Failed to upload to nfss')
89
def _get_files_app_name_and_test(filename):
90
"""return tuple containing (appname, testname)."""
91
app_name_search = re.search(
92
'memory_usage_([a-zA-Z-]*).(.*)\.json',
95
return (app_name_search.group(1), app_name_search.group(2))
98
def get_application_readings(json_data):
100
pids = json_data["pids"]
101
for reading in json_data["readings"]:
103
event=reading["event"],
104
start_time=reading["start_time"],
105
stop_time=reading["stop_time"],
107
# find the data results for this event (based on pid).
109
if str(pid) in reading["data"].keys():
110
results["data"] = reading["data"][str(pid)]
113
app_results.append(results)
117
def get_application_results(json_filepath):
118
with open(json_filepath, "r") as f:
119
json_data = json.load(f)
120
return get_application_readings(json_data)
123
def get_application_details(application_files):
124
"""For every file this application has grab out the details and return a
125
list dictionaries containing reading details.
129
for json_file in application_files:
130
app_result.extend(get_application_results(json_file))
134
def map_files_to_applications():
135
"""For any memory result files that exist, return a dictionary whos keys
136
are the applications name mapped to the file.
138
We can then produce a single json result for each application regardless of
139
there being many tests / results for it.
143
global source_file_path
144
json_results_filename_pattern = os.path.join(
146
"memory_usage_*.json"
148
file_map = defaultdict(list)
149
for json_result_file in glob(json_results_filename_pattern):
150
app_name, test_name = _get_files_app_name_and_test(json_result_file)
151
file_map[app_name].append(json_result_file)
156
print("{} <source file path> <nfss upload script> <nfss config file>"
157
.format(sys.argv[0]))
161
if len(sys.argv) != 4:
165
global source_file_path
166
source_file_path = sys.argv[1]
169
upload_script = sys.argv[2]
172
nfss_config = sys.argv[3]
175
file_map = map_files_to_applications()
176
for app_name in file_map.keys():
177
app_details[app_name] = get_application_details(file_map[app_name])
179
for app_name in app_details.keys():
180
run_details = _get_run_details(app_name)
181
upload_json_details(run_details, app_details[app_name])
184
if __name__ == '__main__':
added
removed
tests/memevent/nfss_upload_results.py
