1
"""Generate graphs for system statistics."""
5
from fixtures import EnvironmentVariable
14
log = logging.getLogger("perf_graphing")
20
font = 'DEFAULT:0:Bitstream Vera Sans'
24
"""Map field order of mongostat (csv/tsv) output to readable name."""
25
# Timestamp is last item in the output
38
"""Wrapper class for a line of data in mongostats output.
40
Handles converting raw data from the output to data that we can use to
41
populate an rrd database etc.
44
def __init__(self, timestamp, insert, query, update, delete, vsize, res):
46
self.timestamp = timestamp
47
self.insert = int(insert.replace('*', ''))
48
self.query = int(query.replace('*', ''))
49
self.update = int(update.replace('*', ''))
50
self.delete = int(delete.replace('*', ''))
52
self.vsize = value_to_bytes(vsize)
56
self.res = value_to_bytes(res)
61
class SourceFileNotFound(Exception):
62
"""Indicate when an expected metrics data file does not exist."""
65
def value_to_bytes(amount):
66
"""Using SI Prefix rules."""
68
# Initially Convert to float due to mongostats output having values like:
71
if not amount[-1].isalpha():
72
return int(float(amount))
74
SIUnits = dict(K=1e3, M=1e6, G=1e9)
76
return int(float(amount[:-1]) * SIUnits[amount[-1]])
78
err_str = 'Unable to convert: {}'.format(amount)
80
raise ValueError(err_str)
83
def network_graph(start, end, rrd_path, output_file):
84
with EnvironmentVariable('TZ', 'UTC'):
87
'--start', str(start),
90
'-w', GraphDefaults.width,
91
'-h', GraphDefaults.height,
92
'-n', GraphDefaults.font,
94
'--alt-autoscale-max',
95
'-t', 'Network Usage',
96
'DEF:rx_avg={}/if_packets.rrd:rx:AVERAGE'.format(rrd_path),
97
'DEF:tx_avg={}/if_packets.rrd:tx:AVERAGE'.format(rrd_path),
98
'CDEF:rx_nnl=rx_avg,UN,0,rx_avg,IF',
99
'CDEF:tx_nnl=tx_avg,UN,0,tx_avg,IF',
100
'CDEF:rx_stk=rx_nnl',
101
'CDEF:tx_stk=tx_nnl',
102
'LINE2:rx_stk#bff7bf: RX',
103
'LINE2:tx_stk#ffb000: TX')
106
def memory_graph(start, end, rrd_path, output_file):
107
with EnvironmentVariable('TZ', 'UTC'):
110
'--start', str(start),
113
'-w', GraphDefaults.width,
114
'-h', GraphDefaults.height,
115
'-n', GraphDefaults.font,
117
'--alt-autoscale-max',
118
'-t', 'Memory Usage',
119
'DEF:free_avg={}/memory-free.rrd:value:AVERAGE'.format(rrd_path),
120
'CDEF:free_nnl=free_avg,UN,0,free_avg,IF',
121
'DEF:used_avg={}/memory-used.rrd:value:AVERAGE'.format(rrd_path),
122
'CDEF:used_nnl=used_avg,UN,0,used_avg,IF',
123
'DEF:buffered_avg={}/memory-buffered.rrd:value:AVERAGE'.format(
125
'CDEF:buffered_nnl=buffered_avg,UN,0,buffered_avg,IF',
126
'DEF:cached_avg={}/memory-cached.rrd:value:AVERAGE'.format(
128
'CDEF:cached_nnl=cached_avg,UN,0,cached_avg,IF',
129
'CDEF:free_stk=free_nnl',
130
'CDEF:used_stk=used_nnl',
131
'AREA:free_stk#ffffff',
132
'LINE1:free_stk#ffffff: free',
133
'AREA:used_stk#ffebbf',
134
'LINE1:used_stk#ffb000: used')
137
def mongodb_graph(start, end, rrd_path, output_file):
138
"""Create a graph image for mongo db actions details."""
139
with EnvironmentVariable('TZ', 'UTC'):
142
'--start', str(start),
147
'-n', 'DEFAULT:0:Bitstream Vera Sans',
149
'--alt-autoscale-max',
150
'-t', 'MongoDB Actions',
151
'DEF:insert_max={rrd}:insert:MAX'.format(
152
rrd=os.path.join(rrd_path, 'mongodb.rrd')),
153
'DEF:query_max={rrd}:query:MAX'.format(
154
rrd=os.path.join(rrd_path, 'mongodb.rrd')),
155
'DEF:update_max={rrd}:update:MAX'.format(
156
rrd=os.path.join(rrd_path, 'mongodb.rrd')),
157
'DEF:delete_max={rrd}:delete:MAX'.format(
158
rrd=os.path.join(rrd_path, 'mongodb.rrd')),
159
'CDEF:insert_nnl=insert_max,UN,0,insert_max,IF',
160
'CDEF:query_nnl=query_max,UN,0,query_max,IF',
161
'CDEF:update_nnl=update_max,UN,0,update_max,IF',
162
'CDEF:delete_nnl=delete_max,UN,0,delete_max,IF',
163
'CDEF:delete_stk=delete_nnl',
164
'CDEF:update_stk=update_nnl',
165
'CDEF:query_stk=query_nnl',
166
'CDEF:insert_stk=insert_nnl',
167
'AREA:insert_stk#bff7bf80',
168
'LINE1:insert_stk#00E000: insert',
169
'AREA:query_stk#bfbfff80',
170
'LINE1:query_stk#0000FF: query',
171
'AREA:update_stk#ffebbf80',
172
'LINE1:update_stk#FFB000: update',
173
'AREA:delete_stk#ffbfbf80',
174
'LINE1:delete_stk#FF0000: delete')
177
def mongodb_memory_graph(start, end, rrd_path, output_file):
178
"""Create a graph image for mongo memory usage."""
179
rrd_file = os.path.join(rrd_path, 'mongodb_memory.rrd')
180
with EnvironmentVariable('TZ', 'UTC'):
183
'--start', str(start),
188
'-n', 'DEFAULT:0:Bitstream Vera Sans',
190
'--alt-autoscale-max',
191
'-t', 'MongoDB Memory Usage',
192
'DEF:vsize_avg={}:vsize:AVERAGE'.format(rrd_file),
193
'CDEF:vsize_nnl=vsize_avg,UN,0,vsize_avg,IF',
194
'DEF:res_avg={}:res:AVERAGE'.format(rrd_file),
195
'CDEF:res_nnl=res_avg,UN,0,res_avg,IF',
196
'CDEF:vsize_stk=vsize_nnl',
197
'CDEF:res_stk=res_nnl,vsize_stk,+',
198
'AREA:vsize_stk#bff7bf80',
199
'LINE1:vsize_stk#00E000: vsize',
200
'AREA:res_stk#bfbfff80',
201
'LINE1:res_stk#0000FF: res')
204
def cpu_graph(start, end, rrd_path, output_file):
205
with EnvironmentVariable('TZ', 'UTC'):
208
'--start', str(start),
211
'-w', GraphDefaults.width,
212
'-h', GraphDefaults.height,
213
'-n', GraphDefaults.font,
215
'--alt-autoscale-max',
219
'DEF:idle_avg={}/cpu-idle.rrd:value:AVERAGE'.format(rrd_path),
220
'CDEF:idle_nnl=idle_avg,UN,0,idle_avg,IF',
221
'DEF:wait_avg={}/cpu-wait.rrd:value:AVERAGE'.format(rrd_path),
222
'CDEF:wait_nnl=wait_avg,UN,0,wait_avg,IF',
223
'DEF:nice_avg={}/cpu-nice.rrd:value:AVERAGE'.format(rrd_path),
224
'CDEF:nice_nnl=nice_avg,UN,0,nice_avg,IF',
225
'DEF:user_avg={}/cpu-user.rrd:value:AVERAGE'.format(rrd_path),
226
'CDEF:user_nnl=user_avg,UN,0,user_avg,IF',
227
'DEF:system_avg={}/cpu-system.rrd:value:AVERAGE'.format(rrd_path),
228
'CDEF:system_nnl=system_avg,UN,0,system_avg,IF',
229
'DEF:softirq_avg={}/cpu-softirq.rrd:value:AVERAGE'.format(
231
'CDEF:softirq_nnl=softirq_avg,UN,0,softirq_avg,IF',
232
'DEF:interrupt_avg={}/cpu-interrupt.rrd:value:AVERAGE'.format(
234
'CDEF:interrupt_nnl=interrupt_avg,UN,0,interrupt_avg,IF',
235
'DEF:steal_avg={}/cpu-steal.rrd:value:AVERAGE'.format(rrd_path),
236
'CDEF:steal_nnl=steal_avg,UN,0,steal_avg,IF',
237
'CDEF:steal_stk=steal_nnl',
238
'CDEF:interrupt_stk=interrupt_nnl,steal_stk,+',
239
'CDEF:softirq_stk=softirq_nnl,interrupt_stk,+',
240
'CDEF:system_stk=system_nnl,softirq_stk,+',
241
'CDEF:user_stk=user_nnl,system_stk,+',
242
'CDEF:nice_stk=nice_nnl,user_stk,+',
243
'CDEF:wait_stk=wait_nnl,nice_stk,+',
244
'CDEF:idle_stk=idle_nnl,wait_stk,+',
245
'AREA:idle_stk#ffffff',
246
'LINE1:idle_stk#ffffff: Idle',
247
'AREA:wait_stk#ffebbf',
248
'LINE1:wait_stk#ffb000: Wait',
249
'AREA:nice_stk#bff7bf',
250
'LINE1:nice_stk#00e000: Nice',
251
'AREA:user_stk#bfbfff',
252
'LINE1:user_stk#0000ff: User',
253
'AREA:system_stk#ffbfbf',
254
'LINE1:system_stk#ff0000: system',
255
'AREA:softirq_stk#ffbfff',
256
'LINE1:softirq_stk#ff00ff: Softirq',
257
'AREA:interrupt_stk#e7bfe7',
258
'LINE1:interrupt_stk#a000a0: Interrupt',
259
'AREA:steal_stk#bfbfbf',
260
'LINE1:steal_stk#000000: Steal')
263
def get_mongodb_stat_data(stats_file):
264
"""Parse raw mongostats log file output for use in creating rrd values.
266
:param stats_file: File-like object from which to extract the data from.
267
:return: List of MongoStatsData objects
270
for line in stats_file:
271
details = line.split()
272
raw_time = details[MongoStats.timestamp]
275
time.strptime(raw_time, '%Y-%m-%dT%H:%M:%SZ')))
279
details[MongoStats.inserts],
280
details[MongoStats.query],
281
details[MongoStats.update],
282
details[MongoStats.delete],
283
details[MongoStats.vsize],
284
details[MongoStats.res],
286
first_timestamp = data_lines[0].timestamp
287
final_timestamp = data_lines[-1].timestamp
288
return first_timestamp, final_timestamp, data_lines
291
def create_mongodb_rrd_files(results_dir, destination_dir):
292
"""Convert mongostats log output into populated rrd files.
294
Creates 2 rrd files, one for db actions and another for db memory usage.
296
source_file = os.path.join(results_dir, 'mongodb-stats.log')
298
stats_file = open(source_file, 'rt')
300
if e.errno == errno.ENOENT:
301
raise SourceFileNotFound(
302
'mongodb stats file not found ({})'.format(source_file))
306
first_ts, last_ts, all_data = get_mongodb_stat_data(stats_file)
308
query_detail_file = os.path.join(destination_dir, 'mongodb.rrd')
309
memory_detail_file = os.path.join(destination_dir, 'mongodb_memory.rrd')
311
_create_mongodb_memory_database(memory_detail_file, first_ts, all_data)
312
_create_mongodb_query_database(query_detail_file, first_ts, all_data)
315
def _create_mongodb_memory_database(memory_detail_file, first_ts, all_data):
316
"""Create a rrd file and populate it with memory usage from mongostats."""
317
_create_mongodb_memory_file(memory_detail_file, first_ts)
318
_populate_mongodb_memory_database(memory_detail_file, all_data)
321
def _create_mongodb_query_database(query_detail_file, first_ts, all_data):
322
"""Create a rrd file and populate it with db action data from mongostats.
324
_create_mongodb_actions_file(query_detail_file, first_ts)
325
_populate_mongodb_query_database(query_detail_file, all_data)
328
def _populate_mongodb_query_database(query_detail_file, all_data):
329
"""Populate a rrd file with db action details."""
330
for entry in all_data:
331
query_update_details = '{time}:{i}:{q}:{u}:{d}'.format(
332
time=entry.timestamp,
338
rrdtool.update(query_detail_file, query_update_details)
341
def _populate_mongodb_memory_database(memory_detail_file, all_data):
342
"""Populate a rrd file with db memory usage details."""
343
for entry in all_data:
344
memory_update_details = '{time}:{vsize}:{res}'.format(
345
time=entry.timestamp,
349
rrdtool.update(memory_detail_file, memory_update_details)
352
def _create_mongodb_actions_file(destination_file, first_ts):
353
"""Create a rrd file to store mongodb action statistics.
355
Mimics the file settings used by collectd for it's rrd file creation.
359
'--start', '{}-10'.format(first_ts),
361
'DS:insert:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
362
'DS:query:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
363
'DS:update:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
364
'DS:delete:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
365
'RRA:AVERAGE:0.1:1:1200',
366
'RRA:MIN:0.1:1:1200',
367
'RRA:MAX:0.1:1:1200',
368
'RRA:AVERAGE:0.1:14:1234',
369
'RRA:MIN:0.1:14:1234',
370
'RRA:MAX:0.1:14:1234',
371
'RRA:AVERAGE:0.1:100:1209',
372
'RRA:MIN:0.1:100:1209',
373
'RRA:MAX:0.1:100:1209',
374
'RRA:AVERAGE:0.1:446:1201',
375
'RRA:MIN:0.1:446:1201',
376
'RRA:MAX:0.1:446:1201',
377
'RRA:AVERAGE:0.1:5270:1200',
378
'RRA:MIN:0.1:5270:1200',
379
'RRA:MAX:0.1:5270:1200',
383
def _create_mongodb_memory_file(destination_file, first_ts):
384
"""Create a rrd file to store mongodb memory usage statistics.
386
Mimics the file settings used by collectd for it's rrd file creation.
390
'--start', '{}-10'.format(first_ts),
392
'DS:vsize:GAUGE:600:{min}:{max}'.format(min=0, max=281474976710000),
393
'DS:res:GAUGE:600:{min}:{max}'.format(min=0, max=281474976710000),
394
'RRA:AVERAGE:0.1:1:1200',
395
'RRA:MIN:0.1:1:1200',
396
'RRA:MAX:0.1:1:1200',
397
'RRA:AVERAGE:0.1:14:1234',
398
'RRA:MIN:0.1:14:1234',
399
'RRA:MAX:0.1:14:1234',
400
'RRA:AVERAGE:0.1:100:1209',
401
'RRA:MIN:0.1:100:1209',
402
'RRA:MAX:0.1:100:1209',
403
'RRA:AVERAGE:0.1:446:1201',
404
'RRA:MIN:0.1:446:1201',
405
'RRA:MAX:0.1:446:1201',
406
'RRA:AVERAGE:0.1:5270:1200',
407
'RRA:MIN:0.1:5270:1200',
408
'RRA:MAX:0.1:5270:1200',