2
"""Simple performance and scale tests."""
4
from __future__ import print_function
7
from collections import defaultdict, namedtuple
8
from datetime import datetime
17
# rddtool requires the cairo/pango libs that are difficult to install
20
from jinja2 import Template
22
from deploy_stack import (
25
from logbreakdown import (
27
breakdown_log_by_timeframes,
31
add_basic_testing_arguments,
41
strf_format = '%F %H:%M:%S'
43
# Log breakdown uses the start/end too. Perhaps have a property for string
44
# rep and a ds for the datetime.
45
def __init__(self, start, end):
46
self.start = start.strftime(self.strf_format)
47
self.end = end.strftime(self.strf_format)
48
self.seconds = int((end - start).total_seconds())
50
DeployDetails = namedtuple(
51
'DeployDetails', ['name', 'applications', 'timings'])
54
SETUP_SCRIPT_PATH = 'perf_static/setup-perf-monitoring.sh'
55
COLLECTD_CONFIG_PATH = 'perf_static/collectd.conf'
58
log = logging.getLogger("assess_perf_test_simple")
61
def assess_perf_test_simple(bs_manager, upload_tools):
63
# Find the actual cause for this!! (Something to do with the template and
67
sys.setdefaultencoding('utf-8')
70
bs_start = datetime.utcnow()
71
with bs_manager.booted_context(upload_tools):
72
client = bs_manager.client
73
admin_client = client.get_controller_client()
74
admin_client.wait_for_started()
75
bs_end = datetime.utcnow()
77
apply_any_workarounds(client)
78
bootstrap_timing = TimingData(bs_start, bs_end)
80
setup_system_monitoring(admin_client)
82
deploy_details = assess_deployment_perf(client)
84
results_dir = os.path.join(
85
os.path.abspath(bs_manager.log_dir), 'performance_results/')
86
os.makedirs(results_dir)
89
('--', '-r', '0:/var/lib/collectd/rrd/localhost/*',
95
'scp', ('0:/tmp/mongodb-stats.log', results_dir)
97
except subprocess.CalledProcessError as e:
98
log.error('Failed to copy mongodb stats: {}'.format(e))
99
cleanup_start = datetime.utcnow()
100
# Cleanup happens when we move out of context
101
cleanup_end = datetime.utcnow()
102
cleanup_timing = TimingData(cleanup_start, cleanup_end)
104
bootstrap=bootstrap_timing,
105
deploys=[deploy_details],
106
cleanup=cleanup_timing,
109
# Could be smarter about this.
110
controller_log_file = os.path.join(
116
generate_reports(controller_log_file, results_dir, deployments)
119
def apply_any_workarounds(client):
120
# Work around mysql charm wanting 80% memory.
121
if client.env.get_cloud() == 'lxd':
126
'juju-{}'.format(client.env.environment),
130
subprocess.check_output(constraint_cmd)
133
def generate_reports(controller_log, results_dir, deployments):
134
"""Generate reports and graphs from run results."""
135
cpu_image = generate_cpu_graph_image(results_dir)
136
memory_image = generate_memory_graph_image(results_dir)
137
network_image = generate_network_graph_image(results_dir)
139
destination_dir = os.path.join(results_dir, 'mongodb')
140
os.mkdir(destination_dir)
142
perf_graphing.create_mongodb_rrd_files(results_dir, destination_dir)
143
except perf_graphing.SourceFileNotFound:
145
'Failed to create the MongoDB RRD file. Source file not found.'
148
# Sometimes mongostats fails to startup and start logging. Unsure yet
149
# why this is. For now generate the report without the mongodb details,
150
# the rest of the report is still useful.
151
mongo_query_image = None
152
mongo_memory_image = None
154
mongo_query_image = generate_mongo_query_graph_image(results_dir)
155
mongo_memory_image = generate_mongo_memory_graph_image(results_dir)
157
log_message_chunks = get_log_message_in_timed_chunks(
158
controller_log, deployments)
162
memory_graph=memory_image,
163
network_graph=network_image,
164
mongo_graph=mongo_query_image,
165
mongo_memory_graph=mongo_memory_image,
166
deployments=deployments,
167
log_message_chunks=log_message_chunks
170
create_html_report(results_dir, details)
173
def get_log_message_in_timed_chunks(log_file, deployments):
174
"""Breakdown log into timechunks based on event timeranges in 'deployments'
177
deploy_timings = [d.timings for d in deployments['deploys']]
179
bootstrap = deployments['bootstrap']
180
cleanup = deployments['cleanup']
181
all_event_timings = [bootstrap] + deploy_timings + [cleanup]
183
raw_details = breakdown_log_by_timeframes(log_file, all_event_timings)
185
bs_name = _render_ds_string(bootstrap.start, bootstrap.end)
186
cleanup_name = _render_ds_string(cleanup.start, cleanup.end)
189
bs_name: 'Bootstrap',
190
cleanup_name: 'Kill-Controller',
192
for dep in deployments['deploys']:
193
name_range = _render_ds_string(
194
dep.timings.start, dep.timings.end)
195
name_lookup[name_range] = dep.name
197
event_details = defaultdict(defaultdict)
198
# Outer-layer (i.e. event)
199
for event_range in raw_details.keys():
200
event_details[event_range]['name'] = name_lookup[event_range]
201
event_details[event_range]['logs'] = []
203
for log_range in raw_details[event_range].keys():
204
timeframe = log_range
205
message = '<br/>'.join(raw_details[event_range][log_range])
206
event_details[event_range]['logs'].append(
214
def create_html_report(results_dir, details):
215
# render the html file to the results dir
216
with open('./perf_report_template.html', 'rt') as f:
217
template = Template(f.read())
219
results_output = os.path.join(results_dir, 'report.html')
220
with open(results_output, 'wt') as f:
221
f.write(template.render(details))
224
def generate_graph_image(base_dir, results_dir, name, generator):
225
metric_files_dir = os.path.join(os.path.abspath(base_dir), results_dir)
226
return create_report_graph(metric_files_dir, base_dir, name, generator)
229
def create_report_graph(rrd_dir, output_dir, name, generator):
230
any_file = os.listdir(rrd_dir)[0]
231
start, end = get_duration_points(os.path.join(rrd_dir, any_file))
232
output_file = os.path.join(
233
os.path.abspath(output_dir), '{}.png'.format(name))
234
generator(start, end, rrd_dir, output_file)
235
print('Created: {}'.format(output_file))
239
def generate_cpu_graph_image(results_dir):
240
return generate_graph_image(
241
results_dir, 'aggregation-cpu-average', 'cpu', perf_graphing.cpu_graph)
244
def generate_memory_graph_image(results_dir):
245
return generate_graph_image(
246
results_dir, 'memory', 'memory', perf_graphing.memory_graph)
249
def generate_network_graph_image(results_dir):
250
return generate_graph_image(
251
results_dir, 'interface-eth0', 'network', perf_graphing.network_graph)
254
def generate_mongo_query_graph_image(results_dir):
255
return generate_graph_image(
256
results_dir, 'mongodb', 'mongodb', perf_graphing.mongodb_graph)
259
def generate_mongo_memory_graph_image(results_dir):
260
return generate_graph_image(
264
perf_graphing.mongodb_memory_graph)
267
def get_duration_points(rrd_file):
268
start = rrdtool.first(rrd_file)
269
end = rrdtool.last(rrd_file)
271
# Start gives us the start timestamp in the data but it might be null/empty
272
# (i.e no data entered for that time.)
273
# Find the timestamp in which data was first entered.
275
'rrdtool', 'fetch', rrd_file, 'AVERAGE',
276
'--start', str(start), '--end', str(end)]
277
output = subprocess.check_output(command)
279
actual_start = find_actual_start(output)
281
return actual_start, end
284
def find_actual_start(fetch_output):
285
# Gets a start timestamp this isn't a NaN.
286
for line in fetch_output.splitlines():
288
timestamp, value = line.split(':', 1)
289
if not value.startswith(' -nan'):
295
def assess_deployment_perf(client, bundle_name='cs:ubuntu'):
296
# This is where multiple services are started either at the same time
297
# or one after the other etc.
298
deploy_start = datetime.utcnow()
300
# We possibly want 2 timing details here, one for started (i.e. agents
301
# ready) and the other for the workloads to be complete.
302
client.deploy(bundle_name)
303
client.wait_for_started()
304
client.wait_for_workloads()
306
deploy_end = datetime.utcnow()
307
deploy_timing = TimingData(deploy_start, deploy_end)
309
client_details = get_client_details(client)
311
return DeployDetails(bundle_name, client_details, deploy_timing)
314
def get_client_details(client):
315
status = client.get_status()
317
for name in status.get_applications().keys():
318
units[name] = status.get_service_unit_count(name)
322
def setup_system_monitoring(admin_client):
323
# Using ssh get into the machine-0 (or all api/state servers)
324
# Install the required packages and start up logging of systems collections
325
# and mongodb details.
327
installer_script_path = _get_static_script_path(SETUP_SCRIPT_PATH)
328
collectd_config_path = _get_static_script_path(COLLECTD_CONFIG_PATH)
329
installer_script_dest_path = '/tmp/installer.sh'
330
runner_script_dest_path = '/tmp/runner.sh'
331
collectd_config_dest_file = '/tmp/collectd.config'
335
(collectd_config_path, '0:{}'.format(collectd_config_dest_file)))
339
(installer_script_path, '0:{}'.format(installer_script_dest_path)))
340
admin_client.juju('ssh', ('0', 'chmod +x {}'.format(
341
installer_script_dest_path)))
344
('0', '{installer} {config_file} {output_file}'.format(
345
installer=installer_script_dest_path,
346
config_file=collectd_config_dest_file,
347
output_file=runner_script_dest_path)))
350
# Respawn incase the initial execution fails for whatever reason.
351
admin_client.juju('ssh', ('0', '--', 'daemon --respawn {}'.format(
352
runner_script_dest_path)))
355
def _get_static_script_path(script_path):
356
full_path = os.path.abspath(__file__)
357
current_dir = os.path.dirname(full_path)
358
return os.path.join(current_dir, script_path)
361
def parse_args(argv):
362
"""Parse all arguments."""
363
parser = argparse.ArgumentParser(description="Simple perf/scale testing")
364
add_basic_testing_arguments(parser)
365
return parser.parse_args(argv)
369
args = parse_args(argv)
370
configure_logging(args.verbose)
371
bs_manager = BootstrapManager.from_args(args)
372
assess_perf_test_simple(bs_manager, args.upload_tools)
377
if __name__ == '__main__':
added
removed
generate_perfscale_results.py
