← Back to branch summary

~andrewjbeach/juju-ci-tools/make-local-patcher

« back to all changes in this revision

Viewing changes to perf_graphing.py

  • Committer: Aaron Bentley
  • Date: 2015-06-15 19:04:10 UTC
  • mfrom: (976.2.4 fix-log-rotation)
  • Revision ID: aaron.bentley@canonical.com-20150615190410-vvhtl7yxn0xbtbiy
Fix error handling in assess_log_rotation.

Show diffs side-by-side

added added

removed removed

Lines of Context:
perf_graphing.py
1
 
"""Generate graphs for system statistics."""
2
 
 
3
 
import calendar
4
 
import errno
5
 
from fixtures import EnvironmentVariable
6
 
import os
7
 
import logging
8
 
import time
9
 
try:
10
 
    import rrdtool
11
 
except ImportError:
12
 
    rrdtool = object()
13
 
 
14
 
log = logging.getLogger("perf_graphing")
15
 
 
16
 
 
17
 
class GraphDefaults:
18
 
    height = '600'
19
 
    width = '800'
20
 
    font = 'DEFAULT:0:Bitstream Vera Sans'
21
 
 
22
 
 
23
 
class MongoStats:
24
 
    """Map field order of mongostat (csv/tsv) output to readable name."""
25
 
    # Timestamp is last item in the output
26
 
    timestamp = -1
27
 
    # Query stats
28
 
    inserts = 0
29
 
    query = 1
30
 
    update = 2
31
 
    delete = 3
32
 
    # Memory stats
33
 
    vsize = 9
34
 
    res = 10
35
 
 
36
 
 
37
 
class MongoStatsData:
38
 
    """Wrapper class for a line of data in mongostats output.
39
 
 
40
 
    Handles converting raw data from the output to data that we can use to
41
 
    populate an rrd database etc.
42
 
    """
43
 
 
44
 
    def __init__(self, timestamp, insert, query, update, delete, vsize, res):
45
 
 
46
 
        self.timestamp = timestamp
47
 
        self.insert = int(insert.replace('*', ''))
48
 
        self.query = int(query.replace('*', ''))
49
 
        self.update = int(update.replace('*', ''))
50
 
        self.delete = int(delete.replace('*', ''))
51
 
        try:
52
 
            self.vsize = value_to_bytes(vsize)
53
 
        except ValueError:
54
 
            self.vsize = 'U'
55
 
        try:
56
 
            self.res = value_to_bytes(res)
57
 
        except ValueError:
58
 
            self.res = 'U'
59
 
 
60
 
 
61
 
class SourceFileNotFound(Exception):
62
 
    """Indicate when an expected metrics data file does not exist."""
63
 
 
64
 
 
65
 
def value_to_bytes(amount):
66
 
    """Using SI Prefix rules."""
67
 
 
68
 
    # Initially Convert to float due to mongostats output having values like:
69
 
    # 96.0M.
70
 
 
71
 
    if not amount[-1].isalpha():
72
 
        return int(float(amount))
73
 
 
74
 
    SIUnits = dict(K=1e3, M=1e6, G=1e9)
75
 
    try:
76
 
        return int(float(amount[:-1]) * SIUnits[amount[-1]])
77
 
    except KeyError:
78
 
        err_str = 'Unable to convert: {}'.format(amount)
79
 
        log.error(err_str)
80
 
        raise ValueError(err_str)
81
 
 
82
 
 
83
 
def network_graph(start, end, rrd_path, output_file):
84
 
    with EnvironmentVariable('TZ', 'UTC'):
85
 
        rrdtool.graph(
86
 
            output_file,
87
 
            '--start', str(start),
88
 
            '--end', str(end),
89
 
            '--full-size-mode',
90
 
            '-w', GraphDefaults.width,
91
 
            '-h', GraphDefaults.height,
92
 
            '-n', GraphDefaults.font,
93
 
            '-v', 'Bytes',
94
 
            '--alt-autoscale-max',
95
 
            '-t', 'Network Usage',
96
 
            'DEF:rx_avg={}/if_packets.rrd:rx:AVERAGE'.format(rrd_path),
97
 
            'DEF:tx_avg={}/if_packets.rrd:tx:AVERAGE'.format(rrd_path),
98
 
            'CDEF:rx_nnl=rx_avg,UN,0,rx_avg,IF',
99
 
            'CDEF:tx_nnl=tx_avg,UN,0,tx_avg,IF',
100
 
            'CDEF:rx_stk=rx_nnl',
101
 
            'CDEF:tx_stk=tx_nnl',
102
 
            'LINE2:rx_stk#bff7bf: RX',
103
 
            'LINE2:tx_stk#ffb000: TX')
104
 
 
105
 
 
106
 
def memory_graph(start, end, rrd_path, output_file):
107
 
    with EnvironmentVariable('TZ', 'UTC'):
108
 
        rrdtool.graph(
109
 
            output_file,
110
 
            '--start', str(start),
111
 
            '--end', str(end),
112
 
            '--full-size-mode',
113
 
            '-w', GraphDefaults.width,
114
 
            '-h', GraphDefaults.height,
115
 
            '-n', GraphDefaults.font,
116
 
            '-v', 'Memory',
117
 
            '--alt-autoscale-max',
118
 
            '-t', 'Memory Usage',
119
 
            'DEF:free_avg={}/memory-free.rrd:value:AVERAGE'.format(rrd_path),
120
 
            'CDEF:free_nnl=free_avg,UN,0,free_avg,IF',
121
 
            'DEF:used_avg={}/memory-used.rrd:value:AVERAGE'.format(rrd_path),
122
 
            'CDEF:used_nnl=used_avg,UN,0,used_avg,IF',
123
 
            'DEF:buffered_avg={}/memory-buffered.rrd:value:AVERAGE'.format(
124
 
                rrd_path),
125
 
            'CDEF:buffered_nnl=buffered_avg,UN,0,buffered_avg,IF',
126
 
            'DEF:cached_avg={}/memory-cached.rrd:value:AVERAGE'.format(
127
 
                rrd_path),
128
 
            'CDEF:cached_nnl=cached_avg,UN,0,cached_avg,IF',
129
 
            'CDEF:free_stk=free_nnl',
130
 
            'CDEF:used_stk=used_nnl',
131
 
            'AREA:free_stk#ffffff',
132
 
            'LINE1:free_stk#ffffff: free',
133
 
            'AREA:used_stk#ffebbf',
134
 
            'LINE1:used_stk#ffb000: used')
135
 
 
136
 
 
137
 
def mongodb_graph(start, end, rrd_path, output_file):
138
 
    """Create a graph image for mongo db actions details."""
139
 
    with EnvironmentVariable('TZ', 'UTC'):
140
 
        rrdtool.graph(
141
 
            output_file,
142
 
            '--start', str(start),
143
 
            '--end', str(end),
144
 
            '--full-size-mode',
145
 
            '-w', '800',
146
 
            '-h', '600',
147
 
            '-n', 'DEFAULT:0:Bitstream Vera Sans',
148
 
            '-v', 'Queries',
149
 
            '--alt-autoscale-max',
150
 
            '-t', 'MongoDB Actions',
151
 
            'DEF:insert_max={rrd}:insert:MAX'.format(
152
 
                rrd=os.path.join(rrd_path, 'mongodb.rrd')),
153
 
            'DEF:query_max={rrd}:query:MAX'.format(
154
 
                rrd=os.path.join(rrd_path, 'mongodb.rrd')),
155
 
            'DEF:update_max={rrd}:update:MAX'.format(
156
 
                rrd=os.path.join(rrd_path, 'mongodb.rrd')),
157
 
            'DEF:delete_max={rrd}:delete:MAX'.format(
158
 
                rrd=os.path.join(rrd_path, 'mongodb.rrd')),
159
 
            'CDEF:insert_nnl=insert_max,UN,0,insert_max,IF',
160
 
            'CDEF:query_nnl=query_max,UN,0,query_max,IF',
161
 
            'CDEF:update_nnl=update_max,UN,0,update_max,IF',
162
 
            'CDEF:delete_nnl=delete_max,UN,0,delete_max,IF',
163
 
            'CDEF:delete_stk=delete_nnl',
164
 
            'CDEF:update_stk=update_nnl',
165
 
            'CDEF:query_stk=query_nnl',
166
 
            'CDEF:insert_stk=insert_nnl',
167
 
            'AREA:insert_stk#bff7bf80',
168
 
            'LINE1:insert_stk#00E000: insert',
169
 
            'AREA:query_stk#bfbfff80',
170
 
            'LINE1:query_stk#0000FF: query',
171
 
            'AREA:update_stk#ffebbf80',
172
 
            'LINE1:update_stk#FFB000: update',
173
 
            'AREA:delete_stk#ffbfbf80',
174
 
            'LINE1:delete_stk#FF0000: delete')
175
 
 
176
 
 
177
 
def mongodb_memory_graph(start, end, rrd_path, output_file):
178
 
    """Create a graph image for mongo memory usage."""
179
 
    rrd_file = os.path.join(rrd_path, 'mongodb_memory.rrd')
180
 
    with EnvironmentVariable('TZ', 'UTC'):
181
 
        rrdtool.graph(
182
 
            output_file,
183
 
            '--start', str(start),
184
 
            '--end', str(end),
185
 
            '--full-size-mode',
186
 
            '-w', '800',
187
 
            '-h', '600',
188
 
            '-n', 'DEFAULT:0:Bitstream Vera Sans',
189
 
            '-v', 'Queries',
190
 
            '--alt-autoscale-max',
191
 
            '-t', 'MongoDB Memory Usage',
192
 
            'DEF:vsize_avg={}:vsize:AVERAGE'.format(rrd_file),
193
 
            'CDEF:vsize_nnl=vsize_avg,UN,0,vsize_avg,IF',
194
 
            'DEF:res_avg={}:res:AVERAGE'.format(rrd_file),
195
 
            'CDEF:res_nnl=res_avg,UN,0,res_avg,IF',
196
 
            'CDEF:vsize_stk=vsize_nnl',
197
 
            'CDEF:res_stk=res_nnl,vsize_stk,+',
198
 
            'AREA:vsize_stk#bff7bf80',
199
 
            'LINE1:vsize_stk#00E000: vsize',
200
 
            'AREA:res_stk#bfbfff80',
201
 
            'LINE1:res_stk#0000FF: res')
202
 
 
203
 
 
204
 
def cpu_graph(start, end, rrd_path, output_file):
205
 
    with EnvironmentVariable('TZ', 'UTC'):
206
 
        rrdtool.graph(
207
 
            output_file,
208
 
            '--start', str(start),
209
 
            '--end', str(end),
210
 
            '--full-size-mode',
211
 
            '-w', GraphDefaults.width,
212
 
            '-h', GraphDefaults.height,
213
 
            '-n', GraphDefaults.font,
214
 
            '-v', 'Jiffies',
215
 
            '--alt-autoscale-max',
216
 
            '-t', 'CPU Average',
217
 
            '-u', '100',
218
 
            '-r',
219
 
            'DEF:idle_avg={}/cpu-idle.rrd:value:AVERAGE'.format(rrd_path),
220
 
            'CDEF:idle_nnl=idle_avg,UN,0,idle_avg,IF',
221
 
            'DEF:wait_avg={}/cpu-wait.rrd:value:AVERAGE'.format(rrd_path),
222
 
            'CDEF:wait_nnl=wait_avg,UN,0,wait_avg,IF',
223
 
            'DEF:nice_avg={}/cpu-nice.rrd:value:AVERAGE'.format(rrd_path),
224
 
            'CDEF:nice_nnl=nice_avg,UN,0,nice_avg,IF',
225
 
            'DEF:user_avg={}/cpu-user.rrd:value:AVERAGE'.format(rrd_path),
226
 
            'CDEF:user_nnl=user_avg,UN,0,user_avg,IF',
227
 
            'DEF:system_avg={}/cpu-system.rrd:value:AVERAGE'.format(rrd_path),
228
 
            'CDEF:system_nnl=system_avg,UN,0,system_avg,IF',
229
 
            'DEF:softirq_avg={}/cpu-softirq.rrd:value:AVERAGE'.format(
230
 
                rrd_path),
231
 
            'CDEF:softirq_nnl=softirq_avg,UN,0,softirq_avg,IF',
232
 
            'DEF:interrupt_avg={}/cpu-interrupt.rrd:value:AVERAGE'.format(
233
 
                rrd_path),
234
 
            'CDEF:interrupt_nnl=interrupt_avg,UN,0,interrupt_avg,IF',
235
 
            'DEF:steal_avg={}/cpu-steal.rrd:value:AVERAGE'.format(rrd_path),
236
 
            'CDEF:steal_nnl=steal_avg,UN,0,steal_avg,IF',
237
 
            'CDEF:steal_stk=steal_nnl',
238
 
            'CDEF:interrupt_stk=interrupt_nnl,steal_stk,+',
239
 
            'CDEF:softirq_stk=softirq_nnl,interrupt_stk,+',
240
 
            'CDEF:system_stk=system_nnl,softirq_stk,+',
241
 
            'CDEF:user_stk=user_nnl,system_stk,+',
242
 
            'CDEF:nice_stk=nice_nnl,user_stk,+',
243
 
            'CDEF:wait_stk=wait_nnl,nice_stk,+',
244
 
            'CDEF:idle_stk=idle_nnl,wait_stk,+',
245
 
            'AREA:idle_stk#ffffff',
246
 
            'LINE1:idle_stk#ffffff: Idle',
247
 
            'AREA:wait_stk#ffebbf',
248
 
            'LINE1:wait_stk#ffb000: Wait',
249
 
            'AREA:nice_stk#bff7bf',
250
 
            'LINE1:nice_stk#00e000: Nice',
251
 
            'AREA:user_stk#bfbfff',
252
 
            'LINE1:user_stk#0000ff: User',
253
 
            'AREA:system_stk#ffbfbf',
254
 
            'LINE1:system_stk#ff0000: system',
255
 
            'AREA:softirq_stk#ffbfff',
256
 
            'LINE1:softirq_stk#ff00ff: Softirq',
257
 
            'AREA:interrupt_stk#e7bfe7',
258
 
            'LINE1:interrupt_stk#a000a0: Interrupt',
259
 
            'AREA:steal_stk#bfbfbf',
260
 
            'LINE1:steal_stk#000000: Steal')
261
 
 
262
 
 
263
 
def get_mongodb_stat_data(stats_file):
264
 
    """Parse raw mongostats log file output for use in creating rrd values.
265
 
 
266
 
    :param stats_file: File-like object from which to extract the data from.
267
 
    :return: List of MongoStatsData objects
268
 
    """
269
 
    data_lines = []
270
 
    for line in stats_file:
271
 
        details = line.split()
272
 
        raw_time = details[MongoStats.timestamp]
273
 
        timestamp = int(
274
 
            calendar.timegm(
275
 
                time.strptime(raw_time, '%Y-%m-%dT%H:%M:%SZ')))
276
 
        data_lines.append(
277
 
            MongoStatsData(
278
 
                timestamp,
279
 
                details[MongoStats.inserts],
280
 
                details[MongoStats.query],
281
 
                details[MongoStats.update],
282
 
                details[MongoStats.delete],
283
 
                details[MongoStats.vsize],
284
 
                details[MongoStats.res],
285
 
            ))
286
 
    first_timestamp = data_lines[0].timestamp
287
 
    final_timestamp = data_lines[-1].timestamp
288
 
    return first_timestamp, final_timestamp, data_lines
289
 
 
290
 
 
291
 
def create_mongodb_rrd_files(results_dir, destination_dir):
292
 
    """Convert mongostats log output into populated rrd files.
293
 
 
294
 
    Creates 2 rrd files, one for db actions and another for db memory usage.
295
 
    """
296
 
    source_file = os.path.join(results_dir, 'mongodb-stats.log')
297
 
    try:
298
 
        stats_file = open(source_file, 'rt')
299
 
    except IOError as e:
300
 
        if e.errno == errno.ENOENT:
301
 
            raise SourceFileNotFound(
302
 
                'mongodb stats file not found ({})'.format(source_file))
303
 
        raise
304
 
 
305
 
    with stats_file:
306
 
        first_ts, last_ts, all_data = get_mongodb_stat_data(stats_file)
307
 
 
308
 
    query_detail_file = os.path.join(destination_dir, 'mongodb.rrd')
309
 
    memory_detail_file = os.path.join(destination_dir, 'mongodb_memory.rrd')
310
 
 
311
 
    _create_mongodb_memory_database(memory_detail_file, first_ts, all_data)
312
 
    _create_mongodb_query_database(query_detail_file, first_ts, all_data)
313
 
 
314
 
 
315
 
def _create_mongodb_memory_database(memory_detail_file, first_ts, all_data):
316
 
    """Create a rrd file and populate it with memory usage from mongostats."""
317
 
    _create_mongodb_memory_file(memory_detail_file, first_ts)
318
 
    _populate_mongodb_memory_database(memory_detail_file, all_data)
319
 
 
320
 
 
321
 
def _create_mongodb_query_database(query_detail_file, first_ts, all_data):
322
 
    """Create a rrd file and populate it with db action data from mongostats.
323
 
    """
324
 
    _create_mongodb_actions_file(query_detail_file, first_ts)
325
 
    _populate_mongodb_query_database(query_detail_file, all_data)
326
 
 
327
 
 
328
 
def _populate_mongodb_query_database(query_detail_file, all_data):
329
 
    """Populate a rrd file with db action details."""
330
 
    for entry in all_data:
331
 
        query_update_details = '{time}:{i}:{q}:{u}:{d}'.format(
332
 
            time=entry.timestamp,
333
 
            i=entry.insert,
334
 
            q=entry.query,
335
 
            u=entry.update,
336
 
            d=entry.delete,
337
 
        )
338
 
        rrdtool.update(query_detail_file, query_update_details)
339
 
 
340
 
 
341
 
def _populate_mongodb_memory_database(memory_detail_file, all_data):
342
 
    """Populate a rrd file with db memory usage details."""
343
 
    for entry in all_data:
344
 
        memory_update_details = '{time}:{vsize}:{res}'.format(
345
 
            time=entry.timestamp,
346
 
            vsize=entry.vsize,
347
 
            res=entry.res,
348
 
        )
349
 
        rrdtool.update(memory_detail_file, memory_update_details)
350
 
 
351
 
 
352
 
def _create_mongodb_actions_file(destination_file, first_ts):
353
 
    """Create a rrd file to store mongodb action statistics.
354
 
 
355
 
    Mimics the file settings used by collectd for it's rrd file creation.
356
 
    """
357
 
    rrdtool.create(
358
 
        destination_file,
359
 
        '--start', '{}-10'.format(first_ts),
360
 
        '--step', '5',
361
 
        'DS:insert:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
362
 
        'DS:query:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
363
 
        'DS:update:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
364
 
        'DS:delete:GAUGE:10:{min}:{max}'.format(min=0, max=281474976710000),
365
 
        'RRA:AVERAGE:0.1:1:1200',
366
 
        'RRA:MIN:0.1:1:1200',
367
 
        'RRA:MAX:0.1:1:1200',
368
 
        'RRA:AVERAGE:0.1:14:1234',
369
 
        'RRA:MIN:0.1:14:1234',
370
 
        'RRA:MAX:0.1:14:1234',
371
 
        'RRA:AVERAGE:0.1:100:1209',
372
 
        'RRA:MIN:0.1:100:1209',
373
 
        'RRA:MAX:0.1:100:1209',
374
 
        'RRA:AVERAGE:0.1:446:1201',
375
 
        'RRA:MIN:0.1:446:1201',
376
 
        'RRA:MAX:0.1:446:1201',
377
 
        'RRA:AVERAGE:0.1:5270:1200',
378
 
        'RRA:MIN:0.1:5270:1200',
379
 
        'RRA:MAX:0.1:5270:1200',
380
 
    )
381
 
 
382
 
 
383
 
def _create_mongodb_memory_file(destination_file, first_ts):
384
 
    """Create a rrd file to store mongodb memory usage statistics.
385
 
 
386
 
    Mimics the file settings used by collectd for it's rrd file creation.
387
 
    """
388
 
    rrdtool.create(
389
 
        destination_file,
390
 
        '--start', '{}-10'.format(first_ts),
391
 
        '--step', '5',
392
 
        'DS:vsize:GAUGE:600:{min}:{max}'.format(min=0, max=281474976710000),
393
 
        'DS:res:GAUGE:600:{min}:{max}'.format(min=0, max=281474976710000),
394
 
        'RRA:AVERAGE:0.1:1:1200',
395
 
        'RRA:MIN:0.1:1:1200',
396
 
        'RRA:MAX:0.1:1:1200',
397
 
        'RRA:AVERAGE:0.1:14:1234',
398
 
        'RRA:MIN:0.1:14:1234',
399
 
        'RRA:MAX:0.1:14:1234',
400
 
        'RRA:AVERAGE:0.1:100:1209',
401
 
        'RRA:MIN:0.1:100:1209',
402
 
        'RRA:MAX:0.1:100:1209',
403
 
        'RRA:AVERAGE:0.1:446:1201',
404
 
        'RRA:MIN:0.1:446:1201',
405
 
        'RRA:MAX:0.1:446:1201',
406
 
        'RRA:AVERAGE:0.1:5270:1200',
407
 
        'RRA:MIN:0.1:5270:1200',
408
 
        'RRA:MAX:0.1:5270:1200',
409
 
    )