1
# Copyright (c) 2008 Jonathan M. Lange. See LICENSE for details.
3
"""Test results and related things."""
7
'ExtendedToOriginalDecorator',
10
'ThreadsafeForwardingResult',
17
from testtools.compat import all, _format_exc_info, str_is_unicode, _u
19
# From http://docs.python.org/library/datetime.html
20
_ZERO = datetime.timedelta(0)
24
class UTC(datetime.tzinfo):
27
def utcoffset(self, dt):
39
class TestResult(unittest.TestResult):
40
"""Subclass of unittest.TestResult extending the protocol for flexability.
42
This test result supports an experimental protocol for providing additional
43
data to in test outcomes. All the outcome methods take an optional dict
44
'details'. If supplied any other detail parameters like 'err' or 'reason'
45
should not be provided. The details dict is a mapping from names to
46
MIME content objects (see testtools.content). This permits attaching
47
tracebacks, log files, or even large objects like databases that were
48
part of the test fixture. Until this API is accepted into upstream
49
Python it is considered experimental: it may be replaced at any point
50
by a newer version more in line with upstream Python. Compatibility would
51
be aimed for in this case, but may not be possible.
53
:ivar skip_reasons: A dict of skip-reasons -> list of tests. See addSkip.
57
# startTestRun resets all attributes, and older clients don't know to
58
# call startTestRun, so it is called once here.
59
# Because subclasses may reasonably not expect this, we call the
60
# specific version we want to run.
61
TestResult.startTestRun(self)
63
def addExpectedFailure(self, test, err=None, details=None):
64
"""Called when a test has failed in an expected manner.
66
Like with addSuccess and addError, testStopped should still be called.
68
:param test: The test that has been skipped.
69
:param err: The exc_info of the error that was raised.
72
# This is the python 2.7 implementation
73
self.expectedFailures.append(
74
(test, self._err_details_to_string(test, err, details)))
76
def addError(self, test, err=None, details=None):
77
"""Called when an error has occurred. 'err' is a tuple of values as
78
returned by sys.exc_info().
80
:param details: Alternative way to supply details about the outcome.
81
see the class docstring for more information.
83
self.errors.append((test,
84
self._err_details_to_string(test, err, details)))
86
def addFailure(self, test, err=None, details=None):
87
"""Called when an error has occurred. 'err' is a tuple of values as
88
returned by sys.exc_info().
90
:param details: Alternative way to supply details about the outcome.
91
see the class docstring for more information.
93
self.failures.append((test,
94
self._err_details_to_string(test, err, details)))
96
def addSkip(self, test, reason=None, details=None):
97
"""Called when a test has been skipped rather than running.
99
Like with addSuccess and addError, testStopped should still be called.
101
This must be called by the TestCase. 'addError' and 'addFailure' will
102
not call addSkip, since they have no assumptions about the kind of
103
errors that a test can raise.
105
:param test: The test that has been skipped.
106
:param reason: The reason for the test being skipped. For instance,
107
u"pyGL is not available".
108
:param details: Alternative way to supply details about the outcome.
109
see the class docstring for more information.
113
reason = details.get('reason')
115
reason = 'No reason given'
117
reason = ''.join(reason.iter_text())
118
skip_list = self.skip_reasons.setdefault(reason, [])
119
skip_list.append(test)
121
def addSuccess(self, test, details=None):
122
"""Called when a test succeeded."""
124
def addUnexpectedSuccess(self, test, details=None):
125
"""Called when a test was expected to fail, but succeed."""
126
self.unexpectedSuccesses.append(test)
128
def wasSuccessful(self):
129
"""Has this result been successful so far?
131
If there have been any errors, failures or unexpected successes,
132
return False. Otherwise, return True.
134
Note: This differs from standard unittest in that we consider
135
unexpected successes to be equivalent to failures, rather than
138
return not (self.errors or self.failures or self.unexpectedSuccesses)
141
# Python 3 and IronPython strings are unicode, use parent class method
142
_exc_info_to_unicode = unittest.TestResult._exc_info_to_string
144
# For Python 2, need to decode components of traceback according to
145
# their source, so can't use traceback.format_exception
146
# Here follows a little deep magic to copy the existing method and
147
# replace the formatter with one that returns unicode instead
148
from types import FunctionType as __F, ModuleType as __M
149
__f = unittest.TestResult._exc_info_to_string.im_func
150
__g = dict(__f.func_globals)
151
__m = __M("__fake_traceback")
152
__m.format_exception = _format_exc_info
153
__g["traceback"] = __m
154
_exc_info_to_unicode = __F(__f.func_code, __g, "_exc_info_to_unicode")
155
del __F, __M, __f, __g, __m
157
def _err_details_to_string(self, test, err=None, details=None):
158
"""Convert an error in exc_info form or a contents dict to a string."""
160
return self._exc_info_to_unicode(err, test)
161
return _details_to_str(details)
164
"""Return the current 'test time'.
166
If the time() method has not been called, this is equivalent to
167
datetime.now(), otherwise its the last supplied datestamp given to the
170
if self.__now is None:
171
return datetime.datetime.now(utc)
175
def startTestRun(self):
176
"""Called before a test run starts.
178
New in python 2.7. The testtools version resets the result to a
179
pristine condition ready for use in another test run.
181
super(TestResult, self).__init__()
182
self.skip_reasons = {}
184
# -- Start: As per python 2.7 --
185
self.expectedFailures = []
186
self.unexpectedSuccesses = []
187
# -- End: As per python 2.7 --
189
def stopTestRun(self):
190
"""Called after a test run completes
195
def time(self, a_datetime):
196
"""Provide a timestamp to represent the current time.
198
This is useful when test activity is time delayed, or happening
199
concurrently and getting the system time between API calls will not
200
accurately represent the duration of tests (or the whole run).
202
Calling time() sets the datetime used by the TestResult object.
203
Time is permitted to go backwards when using this call.
205
:param a_datetime: A datetime.datetime object with TZ information or
206
None to reset the TestResult to gathering time from the system.
208
self.__now = a_datetime
211
"""Called when the test runner is done.
213
deprecated in favour of stopTestRun.
217
class MultiTestResult(TestResult):
218
"""A test result that dispatches to many test results."""
220
def __init__(self, *results):
221
TestResult.__init__(self)
222
self._results = list(map(ExtendedToOriginalDecorator, results))
224
def _dispatch(self, message, *args, **kwargs):
226
getattr(result, message)(*args, **kwargs)
227
for result in self._results)
229
def startTest(self, test):
230
return self._dispatch('startTest', test)
232
def stopTest(self, test):
233
return self._dispatch('stopTest', test)
235
def addError(self, test, error=None, details=None):
236
return self._dispatch('addError', test, error, details=details)
238
def addExpectedFailure(self, test, err=None, details=None):
239
return self._dispatch(
240
'addExpectedFailure', test, err, details=details)
242
def addFailure(self, test, err=None, details=None):
243
return self._dispatch('addFailure', test, err, details=details)
245
def addSkip(self, test, reason=None, details=None):
246
return self._dispatch('addSkip', test, reason, details=details)
248
def addSuccess(self, test, details=None):
249
return self._dispatch('addSuccess', test, details=details)
251
def addUnexpectedSuccess(self, test, details=None):
252
return self._dispatch('addUnexpectedSuccess', test, details=details)
254
def startTestRun(self):
255
return self._dispatch('startTestRun')
257
def stopTestRun(self):
258
return self._dispatch('stopTestRun')
260
def time(self, a_datetime):
261
return self._dispatch('time', a_datetime)
264
return self._dispatch('done')
266
def wasSuccessful(self):
267
"""Was this result successful?
269
Only returns True if every constituent result was successful.
271
return all(self._dispatch('wasSuccessful'))
274
class TextTestResult(TestResult):
275
"""A TestResult which outputs activity to a text stream."""
277
def __init__(self, stream):
278
"""Construct a TextTestResult writing to stream."""
279
super(TextTestResult, self).__init__()
281
self.sep1 = '=' * 70 + '\n'
282
self.sep2 = '-' * 70 + '\n'
284
def _delta_to_float(self, a_timedelta):
285
return (a_timedelta.days * 86400.0 + a_timedelta.seconds +
286
a_timedelta.microseconds / 1000000.0)
288
def _show_list(self, label, error_list):
289
for test, output in error_list:
290
self.stream.write(self.sep1)
291
self.stream.write("%s: %s\n" % (label, test.id()))
292
self.stream.write(self.sep2)
293
self.stream.write(output)
295
def startTestRun(self):
296
super(TextTestResult, self).startTestRun()
297
self.__start = self._now()
298
self.stream.write("Tests running...\n")
300
def stopTestRun(self):
301
if self.testsRun != 1:
306
self._show_list('ERROR', self.errors)
307
self._show_list('FAIL', self.failures)
308
for test in self.unexpectedSuccesses:
310
"%sUNEXPECTED SUCCESS: %s\n%s" % (
311
self.sep1, test.id(), self.sep2))
312
self.stream.write("Ran %d test%s in %.3fs\n\n" %
313
(self.testsRun, plural,
314
self._delta_to_float(stop - self.__start)))
315
if self.wasSuccessful():
316
self.stream.write("OK\n")
318
self.stream.write("FAILED (")
320
details.append("failures=%d" % (
322
self.failures, self.errors, self.unexpectedSuccesses)))))
323
self.stream.write(", ".join(details))
324
self.stream.write(")\n")
325
super(TextTestResult, self).stopTestRun()
328
class ThreadsafeForwardingResult(TestResult):
329
"""A TestResult which ensures the target does not receive mixed up calls.
331
This is used when receiving test results from multiple sources, and batches
332
up all the activity for a single test into a thread-safe batch where all
333
other ThreadsafeForwardingResult objects sharing the same semaphore will be
336
Typical use of ThreadsafeForwardingResult involves creating one
337
ThreadsafeForwardingResult per thread in a ConcurrentTestSuite. These
338
forward to the TestResult that the ConcurrentTestSuite run method was
341
target.done() is called once for each ThreadsafeForwardingResult that
342
forwards to the same target. If the target's done() takes special action,
343
care should be taken to accommodate this.
346
def __init__(self, target, semaphore):
347
"""Create a ThreadsafeForwardingResult forwarding to target.
349
:param target: A TestResult.
350
:param semaphore: A threading.Semaphore with limit 1.
352
TestResult.__init__(self)
353
self.result = ExtendedToOriginalDecorator(target)
354
self.semaphore = semaphore
356
def _add_result_with_semaphore(self, method, test, *args, **kwargs):
357
self.semaphore.acquire()
359
self.result.time(self._test_start)
360
self.result.startTest(test)
361
self.result.time(self._now())
363
method(test, *args, **kwargs)
365
self.result.stopTest(test)
367
self.semaphore.release()
369
def addError(self, test, err=None, details=None):
370
self._add_result_with_semaphore(self.result.addError,
371
test, err, details=details)
373
def addExpectedFailure(self, test, err=None, details=None):
374
self._add_result_with_semaphore(self.result.addExpectedFailure,
375
test, err, details=details)
377
def addFailure(self, test, err=None, details=None):
378
self._add_result_with_semaphore(self.result.addFailure,
379
test, err, details=details)
381
def addSkip(self, test, reason=None, details=None):
382
self._add_result_with_semaphore(self.result.addSkip,
383
test, reason, details=details)
385
def addSuccess(self, test, details=None):
386
self._add_result_with_semaphore(self.result.addSuccess,
387
test, details=details)
389
def addUnexpectedSuccess(self, test, details=None):
390
self._add_result_with_semaphore(self.result.addUnexpectedSuccess,
391
test, details=details)
393
def startTestRun(self):
394
self.semaphore.acquire()
396
self.result.startTestRun()
398
self.semaphore.release()
400
def stopTestRun(self):
401
self.semaphore.acquire()
403
self.result.stopTestRun()
405
self.semaphore.release()
408
self.semaphore.acquire()
412
self.semaphore.release()
414
def startTest(self, test):
415
self._test_start = self._now()
416
super(ThreadsafeForwardingResult, self).startTest(test)
418
def wasSuccessful(self):
419
return self.result.wasSuccessful()
422
class ExtendedToOriginalDecorator(object):
423
"""Permit new TestResult API code to degrade gracefully with old results.
425
This decorates an existing TestResult and converts missing outcomes
426
such as addSkip to older outcomes such as addSuccess. It also supports
427
the extended details protocol. In all cases the most recent protocol
428
is attempted first, and fallbacks only occur when the decorated result
429
does not support the newer style of calling.
432
def __init__(self, decorated):
433
self.decorated = decorated
435
def __getattr__(self, name):
436
return getattr(self.decorated, name)
438
def addError(self, test, err=None, details=None):
439
self._check_args(err, details)
440
if details is not None:
442
return self.decorated.addError(test, details=details)
445
err = self._details_to_exc_info(details)
446
return self.decorated.addError(test, err)
448
def addExpectedFailure(self, test, err=None, details=None):
449
self._check_args(err, details)
450
addExpectedFailure = getattr(
451
self.decorated, 'addExpectedFailure', None)
452
if addExpectedFailure is None:
453
return self.addSuccess(test)
454
if details is not None:
456
return addExpectedFailure(test, details=details)
459
err = self._details_to_exc_info(details)
460
return addExpectedFailure(test, err)
462
def addFailure(self, test, err=None, details=None):
463
self._check_args(err, details)
464
if details is not None:
466
return self.decorated.addFailure(test, details=details)
469
err = self._details_to_exc_info(details)
470
return self.decorated.addFailure(test, err)
472
def addSkip(self, test, reason=None, details=None):
473
self._check_args(reason, details)
474
addSkip = getattr(self.decorated, 'addSkip', None)
476
return self.decorated.addSuccess(test)
477
if details is not None:
479
return addSkip(test, details=details)
481
# extract the reason if it's available
483
reason = ''.join(details['reason'].iter_text())
485
reason = _details_to_str(details)
486
return addSkip(test, reason)
488
def addUnexpectedSuccess(self, test, details=None):
489
outcome = getattr(self.decorated, 'addUnexpectedSuccess', None)
493
except test.failureException:
494
return self.addFailure(test, sys.exc_info())
495
if details is not None:
497
return outcome(test, details=details)
502
def addSuccess(self, test, details=None):
503
if details is not None:
505
return self.decorated.addSuccess(test, details=details)
508
return self.decorated.addSuccess(test)
510
def _check_args(self, err, details):
514
if details is not None:
517
raise ValueError("Must pass only one of err '%s' and details '%s"
520
def _details_to_exc_info(self, details):
521
"""Convert a details dict to an exc_info tuple."""
522
return (_StringException,
523
_StringException(_details_to_str(details)), None)
527
return self.decorated.done()
528
except AttributeError:
531
def progress(self, offset, whence):
532
method = getattr(self.decorated, 'progress', None)
535
return method(offset, whence)
538
def shouldStop(self):
539
return self.decorated.shouldStop
541
def startTest(self, test):
542
return self.decorated.startTest(test)
544
def startTestRun(self):
546
return self.decorated.startTestRun()
547
except AttributeError:
551
return self.decorated.stop()
553
def stopTest(self, test):
554
return self.decorated.stopTest(test)
556
def stopTestRun(self):
558
return self.decorated.stopTestRun()
559
except AttributeError:
562
def tags(self, new_tags, gone_tags):
563
method = getattr(self.decorated, 'tags', None)
566
return method(new_tags, gone_tags)
568
def time(self, a_datetime):
569
method = getattr(self.decorated, 'time', None)
572
return method(a_datetime)
574
def wasSuccessful(self):
575
return self.decorated.wasSuccessful()
578
class _StringException(Exception):
579
"""An exception made from an arbitrary string."""
581
if not str_is_unicode:
582
def __init__(self, string):
583
if type(string) is not unicode:
584
raise TypeError("_StringException expects unicode, got %r" %
586
Exception.__init__(self, string)
589
return self.args[0].encode("utf-8")
591
def __unicode__(self):
593
# For 3.0 and above the default __str__ is fine, so we don't define one.
598
def __eq__(self, other):
600
return self.args == other.args
601
except AttributeError:
605
def _details_to_str(details):
606
"""Convert a details dict to a string."""
608
# sorted is for testing, may want to remove that and use a dict
609
# subclass with defined order for items instead.
610
for key, content in sorted(details.items()):
611
if content.content_type.type != 'text':
612
chars.append('Binary content: %s\n' % key)
614
chars.append('Text attachment: %s\n' % key)
615
chars.append('------------\n')
616
chars.extend(content.iter_text())
617
if not chars[-1].endswith('\n'):
619
chars.append('------------\n')
620
return _u('').join(chars)