« back to all changes in this revision
Viewing changes to src/oopstools/oops/dbsummaries.py
- browse files at revision 1
- compare with another revision
- compare with revision 6
- download diff
- download tarball
- view history from revision 1
- Committer: Robert Collins
- Date: 2011-10-13 20:18:51 UTC
- Revision ID: robertc@robertcollins.net-20111013201851-ym8jmdhoeol3p83s
Export of cruft-deleted tree.
- files added:
- _bootstrap
- apache
- bootstrap.py
- buildout-templates
- buildout-templates/apache
- buildout-templates/bin
- buildout-templates/src
- buildout-templates/src/oopstools
- etc
- src
- src/oopstools
- src/oopstools/oops
- src/oopstools/oops/migrations
- src/oopstools/oops/static
- src/oopstools/oops/static/summaries
- src/oopstools/oops/templates
- src/oopstools/oops/templatetags
- src/oopstools/oops/test
- src/oopstools/oops/test/files
- src/oopstools/oops/test/files/lazr-configs
- src/oopstools/oops/test/files/lazr-configs/edge
- src/oopstools/oops/test/files/lazr-configs/edge1
- src/oopstools/oops/test/files/lazr-configs/lpnet2
- src/oopstools/oops/test/files/lazr-configs/production
- src/oopstools/oops/test/files/oops-sample
- src/oopstools/oops/test/files/oops-sample/dir1
- src/oopstools/oops/test/files/oops-sample/dir1/2007-11-20
- src/oopstools/oops/test/files/oops-sample/dir1/2008-03-18
- src/oopstools/oops/test/files/oops-sample/dir2
- src/oopstools/oops/test/files/oops-sample/dir2/2007-11-20
- src/oopstools/oops/test/files/oops-sample/dir2/2009-07-31
- src/oopstools/oops/test/files/oops-sample/dir2/2011-04-09
- src/oopstools/scripts
added
removed
src/oopstools/oops/dbsummaries.py
1
# Copyright 2005-2011 Canonical Ltd. All rights reserved.
2
#
3
# This program is free software: you can redistribute it and/or modify
4
# it under the terms of the GNU Affero General Public License as published by
5
# the Free Software Foundation, either version 3 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU Affero General Public License for more details.
12
#
13
# You should have received a copy of the GNU Affero General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
16
import cgi
17
import datetime
18
19
from zope.cachedescriptors.property import Lazy
20
from django.conf import settings
21
from django.db import connection
22
from django.db.models import Count, Max
23
24
from oopstools.oops.models import (
25
Oops, Infestation, TIMEOUT_EXCEPTIONS)
26
from oopstools.oops.templatetags.oops_extras import get_absolute_url
27
28
__metaclass__ = type
29
30
TRACKED_HTTP_METHODS = ('GET', 'POST')
31
32
#############################################################################
33
# Groups
34
35
def _format_http_method_count(data):
36
tmp = []
37
for method in TRACKED_HTTP_METHODS + ('Other',):
38
count = data.get(method)
39
if count:
40
tmp.append("%s: %s" % (method, count))
41
return ' '.join(tmp)
42
43
def _escape(value):
44
if value is not None:
45
value = cgi.escape(value)
46
return value
47
48
# Add GroupConcat to aggregates.
49
from django.db.models.aggregates import Aggregate
50
from django.db.models.sql import aggregates
51
from django.db.models import DecimalField
52
53
class GroupConcat(aggregates.Aggregate):
54
sql_function = 'GROUP_CONCAT'
55
def __init__(self, col, source=None, **extra):
56
# stolen from
57
# http://www.mail-archive.com/django-users@googlegroups.com/msg74611.html
58
aggregates.Aggregate.__init__(self, col, source=DecimalField(), **extra)
59
aggregates.GroupConcat = GroupConcat
60
61
62
class GroupConcat(Aggregate):
63
name = 'GroupConcat'
64
# end GroupConcat
65
66
class SumBool(aggregates.Aggregate):
67
sql_function = 'SUM'
68
sql_template = (
69
"%(function)s(CASE %(field)s WHEN True THEN 1 ELSE 0 END)")
70
def __init__(self, col, source=None, **extra):
71
aggregates.Aggregate.__init__(self, col, source=source, **extra)
72
aggregates.SumBool = SumBool
73
74
class SumBool(Aggregate):
75
name = 'SumBool'
76
77
78
class AbstractGroup:
79
80
max_urls = 5
81
max_url_errors = 10
82
83
# Set these in concrete __init__.
84
etype = evalue = bug = errors = None
85
86
def __init__(self, count, bot_count, local_referrer_count, url_count):
87
self.count = count
88
self.bot_count = int(bot_count)
89
self.local_referrer_count = int(local_referrer_count)
90
self.url_count = url_count
91
92
def formatted_http_method_count(self):
93
return _format_http_method_count(self.http_method_count)
94
95
@Lazy
96
def top_local_referrer(self):
97
if self.local_referrer_count:
98
local_referrers = (
99
self.errors.filter(
100
is_local_referrer__exact=True).values(
101
'referrer').annotate(
102
count=Count('oopsid')).order_by(
103
'count', 'referrer').reverse())
104
#XXX matsubara: hack?
105
if local_referrers:
106
return local_referrers[0]['referrer']
107
else:
108
return None
109
# else: return None
110
111
@Lazy
112
def escaped_top_local_referrer(self):
113
return _escape(self.top_local_referrer)
114
115
@Lazy
116
def top_urls(self):
117
# This uses GroupConcat, a custom function supported natively by
118
# sqlite and mysql, but one that would need emulation in postgres.
119
# This is an optimization that lets us make one query instead of
120
# many.
121
res = (
122
self.errors.values(
123
'url', 'pageid').annotate(
124
count=Count('oopsid'), errors=GroupConcat('oopsid')).order_by(
125
'-count', 'url'))[:self.max_urls]
126
for data in res:
127
if data['url'].startswith(data['pageid']):
128
data['pageid'] = 'Unknown'
129
data['escaped_url'] = _escape(data['url'])
130
data['errors'] = data['errors'].split(',')
131
data['errors'].sort()
132
return res
133
134
@Lazy
135
def http_method_count(self):
136
method_data = (
137
self.errors.values(
138
'http_method').annotate(
139
count=Count('oopsid')))
140
res = {}
141
for d in method_data:
142
if d['http_method'] in TRACKED_HTTP_METHODS:
143
res[d['http_method']] = d['count']
144
else:
145
res['Other'] = res.get('Other', 0) + d['count']
146
return res
147
148
def renderTXT(self, fp):
149
"""Render this group in plain text."""
150
fp.write('%4d %s: %s\n' % (self.count, self.etype, self.evalue))
151
if self.bug:
152
fp.write(' Bug: https://launchpad.net/bugs/%s\n' % self.bug)
153
http_methods = self.formatted_http_method_count()
154
fp.write(' %s Robots: %d Local: %d'
155
% (http_methods, self.bot_count, self.local_referrer_count))
156
if self.etype == 'NotFound' and self.local_referrer_count:
157
fp.write(' Most Common Referrer: %s'
158
% (self.top_local_referrer))
159
fp.write('\n')
160
max_urls = 3
161
assert max_urls <= self.max_urls
162
max_url_errors = 5
163
for data in self.top_urls[:max_urls]:
164
fp.write(' %(count)4d %(url)s (%(pageid)s)\n' % data)
165
fp.write(' %s\n' %
166
', '.join(data['errors'][:max_url_errors]))
167
if self.url_count > max_urls:
168
fp.write(' [%s other URLs]\n'
169
% (self.url_count - max_urls))
170
fp.write('\n')
171
172
def renderHTML(self, fp):
173
"""Render this group in HTML."""
174
fp.write('<div class="exc">%d <b>%s</b>: %s</div>\n'
175
% (self.count, _escape(self.etype), _escape(self.evalue)))
176
if self.bug:
177
bug_link = "https://launchpad.net/bugs/%s" % self.bug
178
fp.write('Bug: <a href="%s">%s</a>\n' % (bug_link, self.bug))
179
http_methods = self.formatted_http_method_count()
180
fp.write('<div class="pct">%s Robots: %d Local: %d'
181
% (http_methods, self.bot_count, self.local_referrer_count))
182
if self.etype == 'NotFound' and self.local_referrer_count:
183
referrer = self.escaped_top_local_referrer
184
fp.write(' Most Common Referrer: <a href="%s">%s</a>'
185
% (referrer, referrer))
186
fp.write('</div>\n')
187
# print the top URLs
188
fp.write('<ul>\n')
189
for data in self.top_urls:
190
fp.write(
191
'<li>%(count)d '
192
'<a class="errurl" href="%(escaped_url)s">'
193
'%(escaped_url)s</a> '
194
'(%(pageid)s)\n' % data)
195
fp.write('<ul class="oops"><li>')
196
fp.write(', '.join(
197
'<a href="%s">%s</a>' % (get_absolute_url(oops), oops)
198
for oops in data['errors'][:self.max_url_errors]))
199
fp.write('</li></ul>\n')
200
fp.write('</li>\n')
201
if self.url_count > self.max_urls:
202
fp.write('<li>[%d more]</li>\n'
203
% (self.url_count - self.max_urls))
204
fp.write('</ul>\n\n')
205
206
207
class InfestationGroup(AbstractGroup):
208
# Assumes it will be passed an oopsinfestation id.
209
210
def __init__(self, oopsinfestation, errors,
211
count, bot_count, local_referrer_count, url_count):
212
super(InfestationGroup, self).__init__(
213
count, bot_count, local_referrer_count, url_count)
214
self.errors = errors.filter(oopsinfestation__exact=oopsinfestation)
215
oopsinfestation = Infestation.objects.get(id=oopsinfestation)
216
self.etype = oopsinfestation.exception_type
217
self.evalue = oopsinfestation.exception_value
218
self.bug = oopsinfestation.bug
219
220
221
class MostExpensiveStatementGroup(AbstractGroup):
222
# Assumes it will be passed a most_expensive_statement.
223
224
def __init__(self, most_expensive_statement, errors,
225
count, bot_count, local_referrer_count, url_count):
226
super(MostExpensiveStatementGroup, self).__init__(
227
count, bot_count, local_referrer_count, url_count)
228
self.errors = errors.filter(
229
most_expensive_statement__exact=most_expensive_statement)
230
self.etype = most_expensive_statement
231
self.evalue = '' # XXX None
232
233
#############################################################################
234
# Sections
235
236
237
class AbstractSection:
238
239
max_count = None
240
241
def __init__(self, title, errors, max_count=None):
242
self.errors = errors
243
self.title = title
244
self.section_id = title.lower().replace(" ", "-")
245
if max_count is not None:
246
self.max_count = max_count
247
248
249
class AbstractGroupSection(AbstractSection):
250
251
# Set a class (or define a method) in concrete class.
252
group_factory = None
253
# Subclass __init__ should define group_count and groups.
254
255
def __init__(self, title, errors, max_count=None, _all_groups=None):
256
super(AbstractGroupSection, self).__init__(title, errors, max_count)
257
self.error_count = errors.count()
258
if _all_groups is not None:
259
# This is a convenience for subclasses. `_all_groups` is not part
260
# of the API.
261
self.group_count = _all_groups.count()
262
self.groups = []
263
for group_data in _all_groups[:self.max_count]:
264
group_data.update(errors=self.errors)
265
self.groups.append(self.group_factory(**group_data))
266
267
def renderHeadlineTXT(self, fp):
268
"""Render this section stats header in plain text."""
269
fp.write(' * %d %s\n' % (self.error_count, self.title))
270
271
def renderHeadlineHTML(self, fp):
272
"""Render this section stats header in HTML."""
273
fp.write('<li><a href="#%s">%d %s</a></li>\n' %
274
(self.section_id, self.error_count, self.title))
275
276
def _renderGroups(self, fp, html=False):
277
for group in self.groups:
278
if html:
279
group.renderHTML(fp)
280
else:
281
group.renderTXT(fp)
282
283
def _limited(self):
284
return (self.max_count is not None and
285
self.max_count >= 0 and self.group_count > self.max_count)
286
287
def renderTXT(self, fp):
288
"""Render this section in plain text."""
289
if self._limited():
290
fp.write('=== Top %d %s (total of %s unique items) ===\n\n' % (
291
self.max_count, self.title, self.group_count))
292
else:
293
fp.write('=== All %s ===\n\n' % self.title)
294
self._renderGroups(fp)
295
fp.write('\n')
296
297
def renderHTML(self, fp):
298
"""Render this section in HTML."""
299
fp.write('<div id="%s">' % self.section_id)
300
if self._limited():
301
fp.write('<h2>Top %d %s (total of %s unique items)</h2>\n' % (
302
self.max_count, self.title, self.group_count))
303
else:
304
fp.write('<h2>All %s</h2>\n' % self.title)
305
self._renderGroups(fp, html=True)
306
fp.write('</div>')
307
308
309
class ErrorSection(AbstractGroupSection):
310
311
group_factory = InfestationGroup
312
313
def __init__(self, title, errors, max_count=None):
314
all_groups = (
315
errors.values(
316
'oopsinfestation').annotate(
317
count=Count('oopsid'),
318
url_count=Count('url', distinct=True),
319
bot_count=SumBool('is_bot'),
320
local_referrer_count=SumBool('is_local_referrer')).order_by(
321
'-count', 'oopsinfestation__exception_type',
322
'oopsinfestation__exception_value')
323
)
324
super(ErrorSection, self).__init__(
325
title, errors, max_count, _all_groups = all_groups)
326
327
328
class NotFoundSection(ErrorSection):
329
"""Pages Not Found section in the error summary."""
330
331
def __init__(self, title, errors, max_count=None):
332
errors = errors.filter(is_local_referrer=True)
333
super(NotFoundSection, self).__init__(
334
title, errors, max_count)
335
336
337
class TimeOutSection(AbstractGroupSection):
338
"""Timeout section in the error summary."""
339
340
group_factory = MostExpensiveStatementGroup
341
342
def __init__(self, title, errors, max_count=None):
343
all_groups = (
344
errors.filter(
345
most_expensive_statement__isnull=False).values(
346
'most_expensive_statement').annotate(
347
count=Count('oopsid'),
348
url_count=Count('url', distinct=True),
349
bot_count=SumBool('is_bot'),
350
local_referrer_count=SumBool('is_local_referrer')).order_by(
351
'-count', 'most_expensive_statement')
352
)
353
super(TimeOutSection, self).__init__(
354
title, errors, max_count, _all_groups = all_groups)
355
356
# We perform the top value queries outside of the ORM for performance.
357
#
358
# 'FROM "oops_oops"' ->
359
top_value_inner_sql = '''
360
FROM (
361
"oops_oops"
362
INNER JOIN (
363
%(inner_query)s LIMIT %(max_count)d)
364
AS "inner_max"
365
ON ("oops_oops".%(field_name)s = "inner_max"."value" AND
366
"oops_oops"."pageid" = "inner_max"."pageid"))
367
'''
368
369
class AbstractTopValueSection(AbstractSection):
370
371
max_count = 10
372
# Set these.
373
field_title = field_name = field_format = None
374
375
def __init__(self, title, errors, max_count=None):
376
super(AbstractTopValueSection, self).__init__(
377
title, errors, max_count)
378
379
def as_sql(query):
380
"""XXX Hack to make QuerySet.query.as_sql() work more or less the same.
381
382
Expected behaviour doesn't work on 1.3. See
383
http://groups.google.com/group/django-users/browse_thread/thread/a71a7e764f7622a0?pli=1
384
"""
385
return query.get_compiler('default').as_sql()
386
387
# The only way to do this without breaking into SQL means you have to
388
# divide up the queries; this way, we do it all in one query, as an
389
# optimization.
390
# inner_query, inner_args are the necessary SQL to return all the
391
# pageids with their top values, ordered by the highest value to the
392
# lowest.
393
inner_query, inner_args = as_sql(
394
self.errors.values_list(
395
'pageid').annotate(
396
value=Max(self.field_name)).order_by(
397
'value', 'pageid').reverse().query)
398
# outer_query, outer_args are the necessary SQL to return all the
399
# oopsids for the inner_query pageids. ordered by the highest value
400
# to the lowest. PostgreSQL seems to ignore the ordering issued by
401
# the inner_query done above.
402
outer_query, outer_args = as_sql(
403
self.errors.values_list(
404
self.field_name, 'oopsid', 'pageid').order_by(
405
self.field_name, 'oopsid').reverse().query)
406
join_sql = top_value_inner_sql % dict(
407
inner_query=inner_query, max_count=self.max_count,
408
field_name=connection.ops.quote_name(self.field_name))
409
original = ' FROM "oops_oops" '
410
assert original in outer_query
411
cursor = connection.cursor()
412
cursor.execute(
413
outer_query.replace(original, join_sql), inner_args + outer_args)
414
self.top_errors = []
415
for value, oopsid, pageid in cursor.fetchall():
416
# If we have multiple oopsids with the same pageid and top
417
# value (e.g. same group of OOPSes) then the top value section
418
# might end up with multiple instances of the same top value
419
# and pageid. Here we store all those OOPSes in a list but
420
# self.render* methods display only the first one.
421
if self.top_errors and self.top_errors[-1][2] == pageid:
422
assert self.top_errors[-1][0] == value, 'programmer error'
423
self.top_errors[-1][1].append(oopsid)
424
else:
425
self.top_errors.append((value, [oopsid], pageid))
426
self.top_errors.sort(
427
key=lambda error: (error[0], error[2]), reverse=True)
428
429
def renderHeadlineTXT(self, fp):
430
return
431
432
def renderHeadlineHTML(self, fp):
433
fp.write(
434
'<li><a href="#%s">%s</a></li>\n' % (self.section_id, self.title))
435
436
def renderHTML(self, fp):
437
fp.write('<div id="%s">' % self.section_id)
438
fp.write('<h2>%s</h2>\n' % self.title)
439
fp.write('<table class="top-value-table">\n')
440
fp.write('<tr>\n')
441
fp.write('<th>%s</th>\n' % self.field_title)
442
fp.write('<th>Oops ID</th>\n')
443
fp.write('<th>Page</th>\n')
444
fp.write('</tr>\n')
445
for value, oopsids, pageid in self.top_errors:
446
fp.write('<tr>\n')
447
fp.write('<td>%s</td>\n<td><a href="%s">%s</a></td>'
448
'\n<td>%s</td>\n' % (
449
self.field_format % value,
450
get_absolute_url(oopsids[0]), oopsids[0], pageid))
451
fp.write('</tr>\n')
452
fp.write('</table>')
453
fp.write('</div>')
454
455
def renderTXT(self, fp):
456
fp.write('=== Top %d %s ===\n\n' % (self.max_count, self.title))
457
for value, oopsids, pageid in self.top_errors:
458
formatted_value = self.field_format % value
459
fp.write('%s %-14s %s\n' % (formatted_value, oopsids[0], pageid))
460
fp.write('\n\n')
461
462
463
class TopDurationSection(AbstractTopValueSection):
464
"""The top page IDs by duration."""
465
title = "Durations"
466
field_title = "Duration"
467
field_name = 'duration'
468
field_format = '%9.2fs'
469
470
471
class StatementCountSection(AbstractTopValueSection):
472
"""The top statement counts section."""
473
title = "Statement Counts"
474
field_title = "Count"
475
field_name = 'statements_count'
476
field_format = '%9d'
477
478
479
class TimeOutCountSection(AbstractSection):
480
"""The timeout counts by page id section."""
481
482
def renderHeadlineTXT(self, fp):
483
pass
484
485
def renderHeadlineHTML(self, fp):
486
fp.write(
487
'<li><a href="#%s">%s</a></li>\n' % (self.section_id, self.title))
488
489
def renderTXT(self, fp):
490
fp.write('=== %s ===\n\n' % (self.title,))
491
fp.write(' Hard / Soft Page ID\n')
492
for info in self.time_out_counts:
493
fp.write('%(hard_timeouts_count)9s / %(soft_timeouts_count)4s '
494
'%(pageid)s\n' % info)
495
fp.write('\n\n')
496
497
def renderHTML(self, fp):
498
fp.write('<div id="%s">' % self.section_id)
499
fp.write('<h2>%s</h2>\n' % self.title)
500
fp.write('<table class="top-value-table">\n')
501
fp.write('<tr>\n')
502
fp.write('<th>Hard</th>\n')
503
fp.write('<th>Soft</th>\n')
504
fp.write('<th>Page ID</th>\n')
505
fp.write('</tr>\n')
506
for info in self.time_out_counts:
507
fp.write('<tr>\n')
508
fp.write('<td>%(hard_timeouts_count)s</td>\n'
509
'<td>%(soft_timeouts_count)s</td>\n'
510
'<td>%(pageid)s</td>\n' % info)
511
fp.write('</tr>\n')
512
fp.write('</table>')
513
fp.write('</div>')
514
515
@Lazy
516
def time_out_counts(self):
517
res = self.errors.filter(
518
classification__title__exact='Time Outs').values(
519
'pageid').annotate(
520
hard_timeouts_count=Count('oopsid')).order_by(
521
'hard_timeouts_count').reverse()
522
soft_timeouts = dict(
523
(d['pageid'], d['soft_timeouts_count']) for d
524
in self.errors.filter(
525
pageid__in=[d['pageid'] for d in res]).filter(
526
classification__title__exact='Soft Time Outs').values(
527
'pageid').annotate(
528
soft_timeouts_count=Count('oopsid')))
529
for info in res:
530
info['soft_timeouts_count'] = soft_timeouts.get(info['pageid'], 0)
531
# We need to sort by (hard_timeouts_count, soft_timeouts_count),
532
# but I don't see how to get both of those at once in the SQL.
533
# Therefore, we sort in Python here at the end, assuming that
534
# the max_count will keep this group small enough to be cheap.
535
# (gary)
536
res = sorted(res, key=lambda error: (
537
error['hard_timeouts_count'], error['soft_timeouts_count'],
538
error['pageid']), reverse=True)
539
return res
540
541
#############################################################################
542
# Summaries
543
544
_marker = object()
545
546
class ErrorSummary:
547
"""Summary of Oops that happened in a given date [range]. A summary
548
is composed by the stats and then the individual error sections.
549
"""
550
#TODO matsubara: maybe stats could be another type of section?
551
552
def __init__(self, start, end, prefixes):
553
for date in (start, end):
554
if not isinstance(date, datetime.datetime):
555
raise TypeError('Dates must be datetime.datetime')
556
assert start <= end, 'Bad dates.'
557
self.start = start
558
self.end = end
559
self.period = (end - start).days + 1
560
self.prefixes = prefixes
561
# When end.time() is 0, we want end's full day worth of data.
562
if not end.time():
563
end = end + datetime.timedelta(days=1)
564
self.errors = Oops.objects.filter(
565
date__gte=start, date__lt=end, prefix__value__in=prefixes)
566
self.sections = []
567
568
@property
569
def count(self):
570
"""Return the sum of all errors of all sections.
571
572
Count is calculated every time because some sections filter
573
out errors, instead of using self.errors count calculated in __init__.
574
Top Summary classes (statistics ones) aren't considered as well.
575
"""
576
total_errors = []
577
for section in self.sections:
578
if not isinstance(section, (AbstractTopValueSection,
579
TimeOutCountSection)):
580
total_errors.append(section.errors.count())
581
return sum(total_errors)
582
583
def get_section_by_id(self, section_id):
584
for section in self.sections:
585
if section.section_id == section_id:
586
return section
587
588
def _get_section_info(self, cls, args, filter=_marker):
589
if isinstance(args, basestring):
590
args = dict(title=args)
591
if filter is _marker:
592
filter = dict(classification__title__exact=args['title'])
593
return cls, args, filter
594
595
def makeSection(self, cls, args=(), kwargs=None):
596
if kwargs is None:
597
kwargs = {}
598
section = cls(*args, **kwargs)
599
return section
600
601
def addSections(self, *data):
602
"""Add sections to the report applying given filters on errors.
603
604
For each section, represented by a tuple (class, data, filters),
605
given filters are applied to the errors, and those are used to build
606
the report section.
607
"""
608
for info in data:
609
cls, args, filter = self._get_section_info(*info)
610
if filter is None:
611
errors = self.errors
612
else:
613
errors = self.errors.filter(**filter)
614
args['errors'] = errors
615
self.sections.append(self.makeSection(cls, kwargs=args))
616
617
def addExclusiveSection(self, cls, args, data):
618
"""Add a section to the report excluding all errors of given sections.
619
620
For all the given other sections (each one represented by (class, data,
621
filters)), errors are filtered out. At the end, only errors not
622
belonging to any other sections remain, and the desired section is
623
built with this data.
624
"""
625
cls, args, ignored = self._get_section_info(cls, args)
626
errors = self.errors
627
for info in data:
628
data_cls, data_title, data_filter = self._get_section_info(*info)
629
if data_filter is None:
630
raise ValueError('exclusive section would be empty')
631
errors = errors.exclude(**data_filter)
632
args['errors'] = errors
633
self.sections.append(self.makeSection(cls, kwargs=args))
634
635
def renderTXT(self, fp):
636
fp.write("=== Statistics ===\n\n")
637
fp.write(" * Log starts: %s\n" % self.start)
638
fp.write(" * Analyzed period: %d days\n" % self.period)
639
fp.write(" * Total OOPSes: %d\n" % self.count)
640
# Do not print Average OOPS if we have less than a day worth of oops.
641
if self.period > 1:
642
fp.write(" * Average OOPSes per day: %.2f\n" %
643
(self.count / self.period))
644
fp.write("\n")
645
for section in self.sections:
646
section.renderHeadlineTXT(fp)
647
fp.write("\n")
648
for section in self.sections:
649
section.renderTXT(fp)
650
651
def renderHTML(self, fp):
652
fp.write('<html>\n'
653
'<head>\n'
654
'<title>Oops Report Summary</title>\n'
655
'<link rel="stylesheet" type="text/css" href="%s/oops/static/oops.css" />\n'
656
'</head>\n'
657
'<body>\n'
658
'<div id=summary>\n'
659
'<h1>Oops Report Summary</h1>\n\n' % settings.ROOT_URL)
660
661
fp.write('<ul id="period">\n')
662
fp.write('<li>Log starts: %s</li>\n' % self.start)
663
fp.write('<li>Analyzed period: %d days</li>\n' % self.period)
664
fp.write('<li>Total exceptions: %d</li>\n' % self.count)
665
# Do not print Average OOPS if we have less than a day worth of oops.
666
if self.period > 1:
667
fp.write('<li>Average exceptions per day: %.2f</li>\n' %
668
(self.count / self.period))
669
fp.write('</ul>\n\n')
670
fp.write('<ul id="stats">\n')
671
for section in self.sections:
672
section.renderHeadlineHTML(fp)
673
fp.write('</ul>\n\n')
674
675
for section in self.sections:
676
fp.write('<a name="%s"></a>' % section.section_id)
677
section.renderHTML(fp)
678
679
fp.write('</div>\n')
680
fp.write('</body>\n')
681
fp.write('</html>\n')
682
683
684
class WebAppErrorSummary(ErrorSummary):
685
"""Summarize web app error reports"""
686
687
def __init__(self, startdate, enddate, prefixes):
688
super(WebAppErrorSummary, self).__init__(startdate, enddate, prefixes)
689
self.addSections(
690
(TopDurationSection, "Durations", None),
691
(StatementCountSection, "Statement Counts", None),
692
(TimeOutCountSection, "Time Out Counts by Page ID",
693
dict(oopsinfestation__exception_type__in=TIMEOUT_EXCEPTIONS)),
694
)
695
section_set = (
696
(TimeOutSection, 'Time Outs'),
697
(TimeOutSection, 'Soft Time Outs'),
698
(ErrorSection, 'Informational Only Errors'),
699
(ErrorSection, 'User Generated Errors'),
700
(ErrorSection, 'Unauthorized Errors'),
701
(NotFoundSection, 'Pages Not Found'),
702
)
703
self.addExclusiveSection(
704
ErrorSection, dict(title='Exceptions', max_count=50), section_set)
705
self.addSections(*section_set)
706
707
708
class GenericErrorSummary(ErrorSummary):
709
"""An error summary with only the exception section."""
710
711
def __init__(self, startdate, enddate, prefixes):
712
super(GenericErrorSummary, self).__init__(startdate, enddate, prefixes)
713
self.addExclusiveSection(
714
ErrorSection, dict(title='Exceptions', max_count=50), ())
715
716
717
class CheckwatchesErrorSummary(ErrorSummary):
718
"""Summarize checkwatches error reports."""
719
720
def __init__(self, startdate, enddate, prefixes):
721
super(CheckwatchesErrorSummary, self).__init__(
722
startdate, enddate, prefixes)
723
section_set = ((ErrorSection, 'Remote Checkwatches Warnings'),)
724
self.addExclusiveSection(
725
ErrorSection, dict(title='Exceptions', max_count=50), section_set)
726
self.addSections(*section_set)
727
728
729
class CodeHostingSummary(ErrorSummary):
730
"""Summarize errors reports for the code hosting system."""
731
732
def __init__(self, startdate, enddate, prefixes):
733
super(CodeHostingSummary, self).__init__(startdate, enddate, prefixes)
734
section_set = ((ErrorSection, 'Remote Errors'),)
735
self.addExclusiveSection(
736
ErrorSection, dict(title='Exceptions', max_count=50), section_set)
737
self.addSections(*section_set)
738
739
740
class UbuntuOneErrorSummary(ErrorSummary):
741
"""Summarize errors reports for Ubuntu One."""
742
743
def __init__(self, startdate, enddate, prefixes):
744
super(UbuntuOneErrorSummary, self).__init__(
745
startdate, enddate, prefixes)
746
section_set = (
747
(StatementCountSection, "Statement Counts"),
748
(TimeOutSection, 'Time Outs'),
749
(ErrorSection, 'Application Errors'),
750
(ErrorSection, 'Producer Errors'),
751
(ErrorSection, 'Assertion Errors'),
752
(ErrorSection, 'Value Errors'),
753
(ErrorSection, 'Unknown Errors'),
754
)
755
self.addExclusiveSection(
756
ErrorSection, dict(title='Exceptions', max_count=50), section_set)
757
self.addSections(*section_set)
758
759
class ISDErrorSummary(GenericErrorSummary):
760
"""Summarize ISD error reports (placeholder)"""
761
762
def __init__(self, startdate, enddate, prefixes):
763
super(ISDErrorSummary, self).__init__(startdate, enddate, prefixes)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1