~canonical-platform-qa/snappy-ecosystem-tests/fixing_ci
» Revision
4.3.9
: snappy_ecosystem_tests/plugins/result.py
« back to all changes in this revision
Viewing changes to snappy_ecosystem_tests/plugins/result.py
- browse files at revision 4.3.9
- compare with another revision
- download diff
- download tarball
- view history from revision 4.3.9
- Committer: Heber Parrucci
- Date: 2017-02-13 16:00:04 UTC
- mto: This revision was merged to the branch mainline in revision 14.
- Revision ID: heber.parrucci@canonical.com-20170213160004-oe9fnfkd6foqllae
Changing runner for pytest to aviod the plugins issues in nose2 using testtools.
Making pylint more strict: fail with Warnings messages.
Refactoring global variables to use Singletons instead.
Updating README.rst with the new runner options
Making pylint more strict: fail with Warnings messages.
Refactoring global variables to use Singletons instead.
Updating README.rst with the new runner options
- files added:
- snappy_ecosystem_tests/configs/pytest.cfg
- files removed:
- snappy_ecosystem_tests/configs/unittest.cfg
- snappy_ecosystem_tests/plugins
- files modified:
- README.rst
added
removed
snappy_ecosystem_tests/plugins/result.py
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
2
3
#
4
# Snappy Ecosystem Tests
5
# Copyright (C) 2017 Canonical
6
#
7
# This program is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU General Public License as published by
9
# the Free Software Foundation, either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# This program is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with this program. If not, see <http://www.gnu.org/licenses/>.
19
#
20
21
"""
22
Collect and report test results.
23
24
This plugin implements the primary user interface for nose2. It
25
collects test outcomes and reports on them to the console, as well as
26
firing several hooks for other plugins to do their own reporting.
27
28
To see this report, nose2 MUST be run with the :option:`verbose` flag::
29
30
nose2 --verbose
31
32
This plugin extends standard unittest console reporting slightly by
33
allowing custom report categories. To put events into a custom
34
reporting category, change the event.outcome to whatever you
35
want. Note, however, that customer categories are *not* treated as
36
errors or failures for the purposes of determining whether a test run
37
has succeeded.
38
39
Don't disable this plugin, unless you (a) have another one doing the
40
same job, or (b) really don't want any test results (and want all test
41
runs to ``exit(1)``).
42
"""
43
# This module contains some code copied from unittest2/runner.py and other
44
# code developed in reference to that module and others within unittest2.
45
# unittest2 is Copyright (c) 2001-2010 Python Software Foundation; All
46
# Rights Reserved. See: http://docs.python.org/license.html
47
48
import sys
49
import unittest
50
51
from nose2 import events, result, util
52
from snappy_ecosystem_tests.plugins.utils import exc_info_to_string
53
54
__unittest = True
55
56
57
class CustomResultReporter(events.Plugin):
58
59
"""Result plugin that implements standard unittest console reporting"""
60
alwaysOn = True
61
configSection = 'custom-test-result'
62
separator1 = '=' * 70
63
separator2 = '-' * 70
64
65
def __init__(self):
66
self.testsRun = 0
67
self.reportCategories = {'failures': [],
68
'errors': [],
69
'skipped': [],
70
'expectedFailures': [],
71
'unexpectedSuccesses': []}
72
self.dontReport = set(['errors', 'failures', 'skipped', 'passed',
73
'expectedFailures', 'unexpectedSuccesses'])
74
75
self.stream = util._WritelnDecorator(sys.stderr)
76
self.descriptions = self.config.as_bool('descriptions', True)
77
78
def startTest(self, event):
79
"""Handle startTest hook
80
81
- prints test description if verbosity > 1
82
"""
83
self.testsRun += 1
84
self._reportStartTest(event)
85
86
def testOutcome(self, event):
87
"""Handle testOutcome hook
88
89
- records test outcome in reportCategories
90
- prints test outcome label
91
- fires reporting hooks (:func:`reportSuccess`, :func:`reportFailure`,
92
etc)
93
94
"""
95
if event.outcome == result.ERROR:
96
self.reportCategories['errors'].append(event)
97
self._reportError(event)
98
elif event.outcome == result.FAIL:
99
if not event.expected:
100
self.reportCategories['failures'].append(event)
101
self._reportFailure(event)
102
else:
103
self.reportCategories['expectedFailures'].append(event)
104
self._reportExpectedFailure(event)
105
elif event.outcome == result.SKIP:
106
self.reportCategories['skipped'].append(event)
107
self._reportSkip(event)
108
elif event.outcome == result.PASS:
109
if event.expected:
110
self._reportSuccess(event)
111
else:
112
self.reportCategories['unexpectedSuccesses'].append(event)
113
self._reportUnexpectedSuccess(event)
114
else:
115
# generic outcome handling
116
self.reportCategories.setdefault(event.outcome, []).append(event)
117
self._reportOtherOutcome(event)
118
119
def afterTestRun(self, event):
120
"""Handle afterTestRun hook
121
122
- prints error lists
123
- prints summary
124
- fires summary reporting hooks (:func:`beforeErrorList`,
125
:func:`beforeSummaryReport`, etc)
126
127
"""
128
self._reportSummary(event)
129
130
def wasSuccessful(self, event):
131
event.success = True
132
for name, events in self.reportCategories.items():
133
for e in events:
134
if (e.outcome == result.ERROR or
135
(e.outcome == result.FAIL and not e.expected)):
136
event.success = False
137
break
138
139
def _reportStartTest(self, event):
140
evt = events.ReportTestEvent(event, self.stream)
141
self.session.hooks.reportStartTest(evt)
142
if evt.handled:
143
return
144
if self.session.verbosity > 1:
145
# allow other plugins to override/spy on stream
146
evt.stream.write(self._getDescription(event.test, errorList=False))
147
evt.stream.write(' ... ')
148
evt.stream.flush()
149
150
def _reportError(self, event):
151
self._report(event, 'reportError', 'E', 'ERROR')
152
153
def _reportFailure(self, event):
154
self._report(event, 'reportFailure', 'F', 'FAIL')
155
156
def _reportSkip(self, event):
157
self._report(event, 'reportSkip', 's', 'skipped %s' % event.reason)
158
159
def _reportExpectedFailure(self, event):
160
self._report(event, 'reportExpectedFailure', 'x', 'expected failure')
161
162
def _reportUnexpectedSuccess(self, event):
163
self._report(
164
event, 'reportUnexpectedSuccess', 'u', 'unexpected success')
165
166
def _reportOtherOutcome(self, event):
167
self._report(event, 'reportOtherOutcome', '?', 'unknown outcome')
168
169
def _reportSuccess(self, event):
170
self._report(event, 'reportSuccess', '.', 'ok')
171
172
def _reportSummary(self, event):
173
# let others print something
174
evt = events.ReportSummaryEvent(
175
event, self.stream, self.reportCategories)
176
self.session.hooks.beforeErrorList(evt)
177
# allows other plugins to mess with report categories
178
cats = evt.reportCategories
179
errors = cats.get('errors', [])
180
failures = cats.get('failures', [])
181
# use evt.stream so plugins can replace/wrap/spy it
182
evt.stream.writeln('')
183
self._printErrorList('ERROR', errors, evt.stream)
184
self._printErrorList('FAIL', failures, evt.stream)
185
186
for flavour, events_ in cats.items():
187
if flavour in self.dontReport:
188
continue
189
self._printErrorList(flavour.upper(), events_, evt.stream)
190
self._printSummary(evt)
191
192
def _printErrorList(self, flavour, events_, stream):
193
for event in events_:
194
desc = self._getDescription(event.test, errorList=True)
195
err = self._getOutcomeDetail(event)
196
stream.writeln(self.separator1)
197
stream.writeln("%s: %s" % (flavour, desc))
198
stream.writeln(self.separator2)
199
stream.writeln("%s" % err)
200
201
def _printSummary(self, reportEvent):
202
self.session.hooks.beforeSummaryReport(reportEvent)
203
204
stream = reportEvent.stream
205
stream.writeln(self.separator2)
206
run = self.testsRun
207
msg = (
208
"Ran %d test%s in %.3fs\n" %
209
(run, run != 1 and "s" or "", reportEvent.stopTestEvent.timeTaken))
210
stream.writeln(msg)
211
212
infos = []
213
extraInfos = []
214
if reportEvent.stopTestEvent.result.wasSuccessful():
215
stream.write("OK")
216
else:
217
stream.write("FAILED")
218
219
failed = len(reportEvent.reportCategories.get('failures', []))
220
errored = len(reportEvent.reportCategories.get('errors', []))
221
skipped = len(reportEvent.reportCategories.get('skipped', []))
222
expectedFails = len(
223
reportEvent.reportCategories.get('expectedFailures', []))
224
unexpectedSuccesses = len(
225
reportEvent.reportCategories.get('unexpectedSuccesses', []))
226
227
for flavour, results in reportEvent.reportCategories.items():
228
if flavour in self.dontReport:
229
continue
230
count = len(results)
231
if count:
232
extraInfos.append("%s=%d" % (flavour, count))
233
234
if failed:
235
infos.append("failures=%d" % failed)
236
if errored:
237
infos.append("errors=%d" % errored)
238
if skipped:
239
infos.append("skipped=%d" % skipped)
240
if expectedFails:
241
infos.append("expected failures=%d" % expectedFails)
242
if unexpectedSuccesses:
243
infos.append("unexpected successes=%d" % unexpectedSuccesses)
244
infos.extend(extraInfos)
245
if infos:
246
reportEvent.stream.writeln(" (%s)" % (", ".join(infos),))
247
else:
248
reportEvent.stream.writeln('')
249
250
self.session.hooks.afterSummaryReport(reportEvent)
251
252
def _getDescription(self, test, errorList):
253
if not isinstance(test, unittest.TestCase):
254
return test.__class__.__name__
255
doc_first_line = test.shortDescription()
256
if self.descriptions and doc_first_line:
257
desc = '\n'.join((str(test), doc_first_line))
258
else:
259
desc = str(test)
260
event = events.DescribeTestEvent(
261
test, description=desc, errorList=errorList)
262
self.session.hooks.describeTest(event)
263
return event.description
264
265
def _getOutcomeDetail(self, event):
266
evt = events.OutcomeDetailEvent(event)
267
result = self.session.hooks.outcomeDetail(evt)
268
if evt.handled:
269
return result
270
exc_info = getattr(event, 'exc_info', None)
271
test = getattr(event, 'test', None)
272
if exc_info:
273
detail = [exc_info_to_string(exc_info, test)]
274
else:
275
detail = []
276
if evt.extraDetail:
277
detail.extend(evt.extraDetail)
278
try:
279
return "\n".join(detail)
280
except UnicodeDecodeError:
281
return "\n".join(util.safe_decode(d) for d in detail)
282
283
def _report(self, event, hook, shortLabel, longLabel):
284
evt = events.ReportTestEvent(event, self.stream)
285
getattr(self.session.hooks, hook)(evt)
286
if evt.handled:
287
return
288
if self.session.verbosity > 1:
289
# event I fired has stream, event I received has labels
290
evt.stream.writeln(getattr(event, 'longLabel', None) or longLabel)
291
elif self.session.verbosity:
292
evt.stream.write(getattr(event, 'shortLabel', None) or shortLabel)
293
evt.stream.flush()
Loggerhead is a web-based interface for Breezy
Version: 2.0.1