1
# Copyright (c) 2008 testtools developers. See LICENSE for details.
3
"""Test TestResults and related things."""
17
from testtools import (
18
ExtendedToOriginalDecorator,
23
ThreadsafeForwardingResult,
26
from testtools.compat import (
28
_get_exception_encoding,
34
from testtools.content import Content
35
from testtools.content_type import ContentType, UTF8_TEXT
36
from testtools.matchers import (
41
from testtools.tests.helpers import (
48
from testtools.testresult.real import utc
51
class Python26Contract(object):
53
def test_fresh_result_is_successful(self):
54
# A result is considered successful before any tests are run.
55
result = self.makeResult()
56
self.assertTrue(result.wasSuccessful())
58
def test_addError_is_failure(self):
59
# addError fails the test run.
60
result = self.makeResult()
61
result.startTest(self)
62
result.addError(self, an_exc_info)
64
self.assertFalse(result.wasSuccessful())
66
def test_addFailure_is_failure(self):
67
# addFailure fails the test run.
68
result = self.makeResult()
69
result.startTest(self)
70
result.addFailure(self, an_exc_info)
72
self.assertFalse(result.wasSuccessful())
74
def test_addSuccess_is_success(self):
75
# addSuccess does not fail the test run.
76
result = self.makeResult()
77
result.startTest(self)
78
result.addSuccess(self)
80
self.assertTrue(result.wasSuccessful())
83
class Python27Contract(Python26Contract):
85
def test_addExpectedFailure(self):
86
# Calling addExpectedFailure(test, exc_info) completes ok.
87
result = self.makeResult()
88
result.startTest(self)
89
result.addExpectedFailure(self, an_exc_info)
91
def test_addExpectedFailure_is_success(self):
92
# addExpectedFailure does not fail the test run.
93
result = self.makeResult()
94
result.startTest(self)
95
result.addExpectedFailure(self, an_exc_info)
97
self.assertTrue(result.wasSuccessful())
99
def test_addSkipped(self):
100
# Calling addSkip(test, reason) completes ok.
101
result = self.makeResult()
102
result.startTest(self)
103
result.addSkip(self, _u("Skipped for some reason"))
105
def test_addSkip_is_success(self):
106
# addSkip does not fail the test run.
107
result = self.makeResult()
108
result.startTest(self)
109
result.addSkip(self, _u("Skipped for some reason"))
110
result.stopTest(self)
111
self.assertTrue(result.wasSuccessful())
113
def test_addUnexpectedSuccess(self):
114
# Calling addUnexpectedSuccess(test) completes ok.
115
result = self.makeResult()
116
result.startTest(self)
117
result.addUnexpectedSuccess(self)
119
def test_addUnexpectedSuccess_was_successful(self):
120
# addUnexpectedSuccess does not fail the test run in Python 2.7.
121
result = self.makeResult()
122
result.startTest(self)
123
result.addUnexpectedSuccess(self)
124
result.stopTest(self)
125
self.assertTrue(result.wasSuccessful())
127
def test_startStopTestRun(self):
128
# Calling startTestRun completes ok.
129
result = self.makeResult()
130
result.startTestRun()
134
class DetailsContract(Python27Contract):
135
"""Tests for the contract of TestResults."""
137
def test_addExpectedFailure_details(self):
138
# Calling addExpectedFailure(test, details=xxx) completes ok.
139
result = self.makeResult()
140
result.startTest(self)
141
result.addExpectedFailure(self, details={})
143
def test_addError_details(self):
144
# Calling addError(test, details=xxx) completes ok.
145
result = self.makeResult()
146
result.startTest(self)
147
result.addError(self, details={})
149
def test_addFailure_details(self):
150
# Calling addFailure(test, details=xxx) completes ok.
151
result = self.makeResult()
152
result.startTest(self)
153
result.addFailure(self, details={})
155
def test_addSkipped_details(self):
156
# Calling addSkip(test, reason) completes ok.
157
result = self.makeResult()
158
result.startTest(self)
159
result.addSkip(self, details={})
161
def test_addUnexpectedSuccess_details(self):
162
# Calling addUnexpectedSuccess(test) completes ok.
163
result = self.makeResult()
164
result.startTest(self)
165
result.addUnexpectedSuccess(self, details={})
167
def test_addSuccess_details(self):
168
# Calling addSuccess(test) completes ok.
169
result = self.makeResult()
170
result.startTest(self)
171
result.addSuccess(self, details={})
174
class FallbackContract(DetailsContract):
175
"""When we fallback we take our policy choice to map calls.
177
For instance, we map unexpectedSuccess to an error code, not to success.
180
def test_addUnexpectedSuccess_was_successful(self):
181
# addUnexpectedSuccess fails test run in testtools.
182
result = self.makeResult()
183
result.startTest(self)
184
result.addUnexpectedSuccess(self)
185
result.stopTest(self)
186
self.assertFalse(result.wasSuccessful())
189
class StartTestRunContract(FallbackContract):
190
"""Defines the contract for testtools policy choices.
192
That is things which are not simply extensions to unittest but choices we
193
have made differently.
196
def test_startTestRun_resets_unexpected_success(self):
197
result = self.makeResult()
198
result.startTest(self)
199
result.addUnexpectedSuccess(self)
200
result.stopTest(self)
201
result.startTestRun()
202
self.assertTrue(result.wasSuccessful())
204
def test_startTestRun_resets_failure(self):
205
result = self.makeResult()
206
result.startTest(self)
207
result.addFailure(self, an_exc_info)
208
result.stopTest(self)
209
result.startTestRun()
210
self.assertTrue(result.wasSuccessful())
212
def test_startTestRun_resets_errors(self):
213
result = self.makeResult()
214
result.startTest(self)
215
result.addError(self, an_exc_info)
216
result.stopTest(self)
217
result.startTestRun()
218
self.assertTrue(result.wasSuccessful())
221
class TestTestResultContract(TestCase, StartTestRunContract):
223
def makeResult(self):
227
class TestMultiTestResultContract(TestCase, StartTestRunContract):
229
def makeResult(self):
230
return MultiTestResult(TestResult(), TestResult())
233
class TestTextTestResultContract(TestCase, StartTestRunContract):
235
def makeResult(self):
236
return TextTestResult(StringIO())
239
class TestThreadSafeForwardingResultContract(TestCase, StartTestRunContract):
241
def makeResult(self):
242
result_semaphore = threading.Semaphore(1)
243
target = TestResult()
244
return ThreadsafeForwardingResult(target, result_semaphore)
247
class TestExtendedTestResultContract(TestCase, StartTestRunContract):
249
def makeResult(self):
250
return ExtendedTestResult()
253
class TestPython26TestResultContract(TestCase, Python26Contract):
255
def makeResult(self):
256
return Python26TestResult()
259
class TestAdaptedPython26TestResultContract(TestCase, FallbackContract):
261
def makeResult(self):
262
return ExtendedToOriginalDecorator(Python26TestResult())
265
class TestPython27TestResultContract(TestCase, Python27Contract):
267
def makeResult(self):
268
return Python27TestResult()
271
class TestAdaptedPython27TestResultContract(TestCase, DetailsContract):
273
def makeResult(self):
274
return ExtendedToOriginalDecorator(Python27TestResult())
277
class TestTestResult(TestCase):
278
"""Tests for 'TestResult'."""
280
def makeResult(self):
281
"""Make an arbitrary result for testing."""
284
def test_addSkipped(self):
285
# Calling addSkip on a TestResult records the test that was skipped in
286
# its skip_reasons dict.
287
result = self.makeResult()
288
result.addSkip(self, _u("Skipped for some reason"))
289
self.assertEqual({_u("Skipped for some reason"):[self]},
291
result.addSkip(self, _u("Skipped for some reason"))
292
self.assertEqual({_u("Skipped for some reason"):[self, self]},
294
result.addSkip(self, _u("Skipped for another reason"))
295
self.assertEqual({_u("Skipped for some reason"):[self, self],
296
_u("Skipped for another reason"):[self]},
299
def test_now_datetime_now(self):
300
result = self.makeResult()
301
olddatetime = testresult.real.datetime
303
testresult.real.datetime = olddatetime
304
self.addCleanup(restore)
307
now = datetime.datetime.now(utc)
308
stubdatetime = Module()
309
stubdatetime.datetime = Module()
310
stubdatetime.datetime.now = lambda tz: now
311
testresult.real.datetime = stubdatetime
312
# Calling _now() looks up the time.
313
self.assertEqual(now, result._now())
314
then = now + datetime.timedelta(0, 1)
315
# Set an explicit datetime, which gets returned from then on.
317
self.assertNotEqual(now, result._now())
318
self.assertEqual(then, result._now())
319
# go back to looking it up.
321
self.assertEqual(now, result._now())
323
def test_now_datetime_time(self):
324
result = self.makeResult()
325
now = datetime.datetime.now(utc)
327
self.assertEqual(now, result._now())
330
class TestWithFakeExceptions(TestCase):
332
def makeExceptionInfo(self, exceptionFactory, *args, **kwargs):
334
raise exceptionFactory(*args, **kwargs)
336
return sys.exc_info()
339
class TestMultiTestResult(TestWithFakeExceptions):
340
"""Tests for 'MultiTestResult'."""
343
TestWithFakeExceptions.setUp(self)
344
self.result1 = LoggingResult([])
345
self.result2 = LoggingResult([])
346
self.multiResult = MultiTestResult(self.result1, self.result2)
348
def assertResultLogsEqual(self, expectedEvents):
349
"""Assert that our test results have received the expected events."""
350
self.assertEqual(expectedEvents, self.result1._events)
351
self.assertEqual(expectedEvents, self.result2._events)
353
def test_empty(self):
354
# Initializing a `MultiTestResult` doesn't do anything to its
356
self.assertResultLogsEqual([])
358
def test_startTest(self):
359
# Calling `startTest` on a `MultiTestResult` calls `startTest` on all
361
self.multiResult.startTest(self)
362
self.assertResultLogsEqual([('startTest', self)])
364
def test_stopTest(self):
365
# Calling `stopTest` on a `MultiTestResult` calls `stopTest` on all
367
self.multiResult.stopTest(self)
368
self.assertResultLogsEqual([('stopTest', self)])
370
def test_addSkipped(self):
371
# Calling `addSkip` on a `MultiTestResult` calls addSkip on its
373
reason = _u("Skipped for some reason")
374
self.multiResult.addSkip(self, reason)
375
self.assertResultLogsEqual([('addSkip', self, reason)])
377
def test_addSuccess(self):
378
# Calling `addSuccess` on a `MultiTestResult` calls `addSuccess` on
379
# all its `TestResult`s.
380
self.multiResult.addSuccess(self)
381
self.assertResultLogsEqual([('addSuccess', self)])
384
# Calling `done` on a `MultiTestResult` calls `done` on all its
386
self.multiResult.done()
387
self.assertResultLogsEqual([('done')])
389
def test_addFailure(self):
390
# Calling `addFailure` on a `MultiTestResult` calls `addFailure` on
391
# all its `TestResult`s.
392
exc_info = self.makeExceptionInfo(AssertionError, 'failure')
393
self.multiResult.addFailure(self, exc_info)
394
self.assertResultLogsEqual([('addFailure', self, exc_info)])
396
def test_addError(self):
397
# Calling `addError` on a `MultiTestResult` calls `addError` on all
399
exc_info = self.makeExceptionInfo(RuntimeError, 'error')
400
self.multiResult.addError(self, exc_info)
401
self.assertResultLogsEqual([('addError', self, exc_info)])
403
def test_startTestRun(self):
404
# Calling `startTestRun` on a `MultiTestResult` forwards to all its
406
self.multiResult.startTestRun()
407
self.assertResultLogsEqual([('startTestRun')])
409
def test_stopTestRun(self):
410
# Calling `stopTestRun` on a `MultiTestResult` forwards to all its
412
self.multiResult.stopTestRun()
413
self.assertResultLogsEqual([('stopTestRun')])
415
def test_stopTestRun_returns_results(self):
416
# `MultiTestResult.stopTestRun` returns a tuple of all of the return
417
# values the `stopTestRun`s that it forwards to.
418
class Result(LoggingResult):
419
def stopTestRun(self):
420
super(Result, self).stopTestRun()
422
multi_result = MultiTestResult(Result([]), Result([]))
423
result = multi_result.stopTestRun()
424
self.assertEqual(('foo', 'foo'), result)
427
# the time call is dispatched, not eaten by the base class
428
self.multiResult.time('foo')
429
self.assertResultLogsEqual([('time', 'foo')])
432
class TestTextTestResult(TestCase):
433
"""Tests for 'TextTestResult'."""
436
super(TestTextTestResult, self).setUp()
437
self.result = TextTestResult(StringIO())
439
def make_erroring_test(self):
440
class Test(TestCase):
445
def make_failing_test(self):
446
class Test(TestCase):
449
return Test("failed")
451
def make_unexpectedly_successful_test(self):
452
class Test(TestCase):
454
self.expectFailure("yo!", lambda: None)
455
return Test("succeeded")
458
class Test(TestCase):
464
return self.result.stream.getvalue()
466
def test__init_sets_stream(self):
467
result = TextTestResult("fp")
468
self.assertEqual("fp", result.stream)
470
def reset_output(self):
471
self.result.stream = StringIO()
473
def test_startTestRun(self):
474
self.result.startTestRun()
475
self.assertEqual("Tests running...\n", self.getvalue())
477
def test_stopTestRun_count_many(self):
478
test = self.make_test()
479
self.result.startTestRun()
480
self.result.startTest(test)
481
self.result.stopTest(test)
482
self.result.startTest(test)
483
self.result.stopTest(test)
484
self.result.stream = StringIO()
485
self.result.stopTestRun()
486
self.assertThat(self.getvalue(),
487
DocTestMatches("Ran 2 tests in ...s\n...", doctest.ELLIPSIS))
489
def test_stopTestRun_count_single(self):
490
test = self.make_test()
491
self.result.startTestRun()
492
self.result.startTest(test)
493
self.result.stopTest(test)
495
self.result.stopTestRun()
496
self.assertThat(self.getvalue(),
497
DocTestMatches("Ran 1 test in ...s\n\nOK\n", doctest.ELLIPSIS))
499
def test_stopTestRun_count_zero(self):
500
self.result.startTestRun()
502
self.result.stopTestRun()
503
self.assertThat(self.getvalue(),
504
DocTestMatches("Ran 0 tests in ...s\n\nOK\n", doctest.ELLIPSIS))
506
def test_stopTestRun_current_time(self):
507
test = self.make_test()
508
now = datetime.datetime.now(utc)
509
self.result.time(now)
510
self.result.startTestRun()
511
self.result.startTest(test)
512
now = now + datetime.timedelta(0, 0, 0, 1)
513
self.result.time(now)
514
self.result.stopTest(test)
516
self.result.stopTestRun()
517
self.assertThat(self.getvalue(),
518
DocTestMatches("... in 0.001s\n...", doctest.ELLIPSIS))
520
def test_stopTestRun_successful(self):
521
self.result.startTestRun()
522
self.result.stopTestRun()
523
self.assertThat(self.getvalue(),
524
DocTestMatches("...\n\nOK\n", doctest.ELLIPSIS))
526
def test_stopTestRun_not_successful_failure(self):
527
test = self.make_failing_test()
528
self.result.startTestRun()
529
test.run(self.result)
530
self.result.stopTestRun()
531
self.assertThat(self.getvalue(),
532
DocTestMatches("...\n\nFAILED (failures=1)\n", doctest.ELLIPSIS))
534
def test_stopTestRun_not_successful_error(self):
535
test = self.make_erroring_test()
536
self.result.startTestRun()
537
test.run(self.result)
538
self.result.stopTestRun()
539
self.assertThat(self.getvalue(),
540
DocTestMatches("...\n\nFAILED (failures=1)\n", doctest.ELLIPSIS))
542
def test_stopTestRun_not_successful_unexpected_success(self):
543
test = self.make_unexpectedly_successful_test()
544
self.result.startTestRun()
545
test.run(self.result)
546
self.result.stopTestRun()
547
self.assertThat(self.getvalue(),
548
DocTestMatches("...\n\nFAILED (failures=1)\n", doctest.ELLIPSIS))
550
def test_stopTestRun_shows_details(self):
551
self.result.startTestRun()
552
self.make_erroring_test().run(self.result)
553
self.make_unexpectedly_successful_test().run(self.result)
554
self.make_failing_test().run(self.result)
556
self.result.stopTestRun()
557
self.assertThat(self.getvalue(),
558
DocTestMatches("""...======================================================================
559
ERROR: testtools.tests.test_testresult.Test.error
560
----------------------------------------------------------------------
561
Text attachment: traceback
563
Traceback (most recent call last):
564
File "...testtools...runtest.py", line ..., in _run_user...
565
return fn(*args, **kwargs)
566
File "...testtools...testcase.py", line ..., in _run_test_method
567
return self._get_test_method()()
568
File "...testtools...tests...test_testresult.py", line ..., in error
570
ZeroDivisionError:... divi... by zero...
572
======================================================================
573
FAIL: testtools.tests.test_testresult.Test.failed
574
----------------------------------------------------------------------
575
Text attachment: traceback
577
Traceback (most recent call last):
578
File "...testtools...runtest.py", line ..., in _run_user...
579
return fn(*args, **kwargs)
580
File "...testtools...testcase.py", line ..., in _run_test_method
581
return self._get_test_method()()
582
File "...testtools...tests...test_testresult.py", line ..., in failed
586
======================================================================
587
UNEXPECTED SUCCESS: testtools.tests.test_testresult.Test.succeeded
588
----------------------------------------------------------------------
589
...""", doctest.ELLIPSIS | doctest.REPORT_NDIFF))
592
class TestThreadSafeForwardingResult(TestWithFakeExceptions):
593
"""Tests for `TestThreadSafeForwardingResult`."""
596
TestWithFakeExceptions.setUp(self)
597
self.result_semaphore = threading.Semaphore(1)
598
self.target = LoggingResult([])
599
self.result1 = ThreadsafeForwardingResult(self.target,
600
self.result_semaphore)
602
def test_nonforwarding_methods(self):
603
# startTest and stopTest are not forwarded because they need to be
605
self.result1.startTest(self)
606
self.result1.stopTest(self)
607
self.assertEqual([], self.target._events)
609
def test_startTestRun(self):
610
self.result1.startTestRun()
611
self.result2 = ThreadsafeForwardingResult(self.target,
612
self.result_semaphore)
613
self.result2.startTestRun()
614
self.assertEqual(["startTestRun", "startTestRun"], self.target._events)
616
def test_stopTestRun(self):
617
self.result1.stopTestRun()
618
self.result2 = ThreadsafeForwardingResult(self.target,
619
self.result_semaphore)
620
self.result2.stopTestRun()
621
self.assertEqual(["stopTestRun", "stopTestRun"], self.target._events)
623
def test_forwarding_methods(self):
624
# error, failure, skip and success are forwarded in batches.
625
exc_info1 = self.makeExceptionInfo(RuntimeError, 'error')
626
starttime1 = datetime.datetime.utcfromtimestamp(1.489)
627
endtime1 = datetime.datetime.utcfromtimestamp(51.476)
628
self.result1.time(starttime1)
629
self.result1.startTest(self)
630
self.result1.time(endtime1)
631
self.result1.addError(self, exc_info1)
632
exc_info2 = self.makeExceptionInfo(AssertionError, 'failure')
633
starttime2 = datetime.datetime.utcfromtimestamp(2.489)
634
endtime2 = datetime.datetime.utcfromtimestamp(3.476)
635
self.result1.time(starttime2)
636
self.result1.startTest(self)
637
self.result1.time(endtime2)
638
self.result1.addFailure(self, exc_info2)
639
reason = _u("Skipped for some reason")
640
starttime3 = datetime.datetime.utcfromtimestamp(4.489)
641
endtime3 = datetime.datetime.utcfromtimestamp(5.476)
642
self.result1.time(starttime3)
643
self.result1.startTest(self)
644
self.result1.time(endtime3)
645
self.result1.addSkip(self, reason)
646
starttime4 = datetime.datetime.utcfromtimestamp(6.489)
647
endtime4 = datetime.datetime.utcfromtimestamp(7.476)
648
self.result1.time(starttime4)
649
self.result1.startTest(self)
650
self.result1.time(endtime4)
651
self.result1.addSuccess(self)
653
('time', starttime1),
656
('addError', self, exc_info1),
658
('time', starttime2),
661
('addFailure', self, exc_info2),
663
('time', starttime3),
666
('addSkip', self, reason),
668
('time', starttime4),
671
('addSuccess', self),
673
], self.target._events)
676
class TestExtendedToOriginalResultDecoratorBase(TestCase):
678
def make_26_result(self):
679
self.result = Python26TestResult()
680
self.make_converter()
682
def make_27_result(self):
683
self.result = Python27TestResult()
684
self.make_converter()
686
def make_converter(self):
687
self.converter = ExtendedToOriginalDecorator(self.result)
689
def make_extended_result(self):
690
self.result = ExtendedTestResult()
691
self.make_converter()
693
def check_outcome_details(self, outcome):
694
"""Call an outcome with a details dict to be passed through."""
695
# This dict is /not/ convertible - thats deliberate, as it should
696
# not hit the conversion code path.
697
details = {'foo': 'bar'}
698
getattr(self.converter, outcome)(self, details=details)
699
self.assertEqual([(outcome, self, details)], self.result._events)
701
def get_details_and_string(self):
702
"""Get a details dict and expected string."""
703
text1 = lambda: [_b("1\n2\n")]
704
text2 = lambda: [_b("3\n4\n")]
705
bin1 = lambda: [_b("5\n")]
706
details = {'text 1': Content(ContentType('text', 'plain'), text1),
707
'text 2': Content(ContentType('text', 'strange'), text2),
708
'bin 1': Content(ContentType('application', 'binary'), bin1)}
709
return (details, "Binary content: bin 1\n"
710
"Text attachment: text 1\n------------\n1\n2\n"
711
"------------\nText attachment: text 2\n------------\n"
712
"3\n4\n------------\n")
714
def check_outcome_details_to_exec_info(self, outcome, expected=None):
715
"""Call an outcome with a details dict to be made into exc_info."""
716
# The conversion is a done using RemoteError and the string contents
717
# of the text types in the details dict.
720
details, err_str = self.get_details_and_string()
721
getattr(self.converter, outcome)(self, details=details)
722
err = self.converter._details_to_exc_info(details)
723
self.assertEqual([(expected, self, err)], self.result._events)
725
def check_outcome_details_to_nothing(self, outcome, expected=None):
726
"""Call an outcome with a details dict to be swallowed."""
729
details = {'foo': 'bar'}
730
getattr(self.converter, outcome)(self, details=details)
731
self.assertEqual([(expected, self)], self.result._events)
733
def check_outcome_details_to_string(self, outcome):
734
"""Call an outcome with a details dict to be stringified."""
735
details, err_str = self.get_details_and_string()
736
getattr(self.converter, outcome)(self, details=details)
737
self.assertEqual([(outcome, self, err_str)], self.result._events)
739
def check_outcome_details_to_arg(self, outcome, arg, extra_detail=None):
740
"""Call an outcome with a details dict to have an arg extracted."""
741
details, _ = self.get_details_and_string()
743
details.update(extra_detail)
744
getattr(self.converter, outcome)(self, details=details)
745
self.assertEqual([(outcome, self, arg)], self.result._events)
747
def check_outcome_exc_info(self, outcome, expected=None):
748
"""Check that calling a legacy outcome still works."""
749
# calling some outcome with the legacy exc_info style api (no keyword
750
# parameters) gets passed through.
754
getattr(self.converter, outcome)(self, err)
755
self.assertEqual([(expected, self, err)], self.result._events)
757
def check_outcome_exc_info_to_nothing(self, outcome, expected=None):
758
"""Check that calling a legacy outcome on a fallback works."""
759
# calling some outcome with the legacy exc_info style api (no keyword
760
# parameters) gets passed through.
764
getattr(self.converter, outcome)(self, err)
765
self.assertEqual([(expected, self)], self.result._events)
767
def check_outcome_nothing(self, outcome, expected=None):
768
"""Check that calling a legacy outcome still works."""
771
getattr(self.converter, outcome)(self)
772
self.assertEqual([(expected, self)], self.result._events)
774
def check_outcome_string_nothing(self, outcome, expected):
775
"""Check that calling outcome with a string calls expected."""
776
getattr(self.converter, outcome)(self, "foo")
777
self.assertEqual([(expected, self)], self.result._events)
779
def check_outcome_string(self, outcome):
780
"""Check that calling outcome with a string works."""
781
getattr(self.converter, outcome)(self, "foo")
782
self.assertEqual([(outcome, self, "foo")], self.result._events)
785
class TestExtendedToOriginalResultDecorator(
786
TestExtendedToOriginalResultDecoratorBase):
788
def test_progress_py26(self):
789
self.make_26_result()
790
self.converter.progress(1, 2)
792
def test_progress_py27(self):
793
self.make_27_result()
794
self.converter.progress(1, 2)
796
def test_progress_pyextended(self):
797
self.make_extended_result()
798
self.converter.progress(1, 2)
799
self.assertEqual([('progress', 1, 2)], self.result._events)
801
def test_shouldStop(self):
802
self.make_26_result()
803
self.assertEqual(False, self.converter.shouldStop)
804
self.converter.decorated.stop()
805
self.assertEqual(True, self.converter.shouldStop)
807
def test_startTest_py26(self):
808
self.make_26_result()
809
self.converter.startTest(self)
810
self.assertEqual([('startTest', self)], self.result._events)
812
def test_startTest_py27(self):
813
self.make_27_result()
814
self.converter.startTest(self)
815
self.assertEqual([('startTest', self)], self.result._events)
817
def test_startTest_pyextended(self):
818
self.make_extended_result()
819
self.converter.startTest(self)
820
self.assertEqual([('startTest', self)], self.result._events)
822
def test_startTestRun_py26(self):
823
self.make_26_result()
824
self.converter.startTestRun()
825
self.assertEqual([], self.result._events)
827
def test_startTestRun_py27(self):
828
self.make_27_result()
829
self.converter.startTestRun()
830
self.assertEqual([('startTestRun',)], self.result._events)
832
def test_startTestRun_pyextended(self):
833
self.make_extended_result()
834
self.converter.startTestRun()
835
self.assertEqual([('startTestRun',)], self.result._events)
837
def test_stopTest_py26(self):
838
self.make_26_result()
839
self.converter.stopTest(self)
840
self.assertEqual([('stopTest', self)], self.result._events)
842
def test_stopTest_py27(self):
843
self.make_27_result()
844
self.converter.stopTest(self)
845
self.assertEqual([('stopTest', self)], self.result._events)
847
def test_stopTest_pyextended(self):
848
self.make_extended_result()
849
self.converter.stopTest(self)
850
self.assertEqual([('stopTest', self)], self.result._events)
852
def test_stopTestRun_py26(self):
853
self.make_26_result()
854
self.converter.stopTestRun()
855
self.assertEqual([], self.result._events)
857
def test_stopTestRun_py27(self):
858
self.make_27_result()
859
self.converter.stopTestRun()
860
self.assertEqual([('stopTestRun',)], self.result._events)
862
def test_stopTestRun_pyextended(self):
863
self.make_extended_result()
864
self.converter.stopTestRun()
865
self.assertEqual([('stopTestRun',)], self.result._events)
867
def test_tags_py26(self):
868
self.make_26_result()
869
self.converter.tags(1, 2)
871
def test_tags_py27(self):
872
self.make_27_result()
873
self.converter.tags(1, 2)
875
def test_tags_pyextended(self):
876
self.make_extended_result()
877
self.converter.tags(1, 2)
878
self.assertEqual([('tags', 1, 2)], self.result._events)
880
def test_time_py26(self):
881
self.make_26_result()
882
self.converter.time(1)
884
def test_time_py27(self):
885
self.make_27_result()
886
self.converter.time(1)
888
def test_time_pyextended(self):
889
self.make_extended_result()
890
self.converter.time(1)
891
self.assertEqual([('time', 1)], self.result._events)
894
class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase):
898
def test_outcome_Original_py26(self):
899
self.make_26_result()
900
self.check_outcome_exc_info(self.outcome)
902
def test_outcome_Original_py27(self):
903
self.make_27_result()
904
self.check_outcome_exc_info(self.outcome)
906
def test_outcome_Original_pyextended(self):
907
self.make_extended_result()
908
self.check_outcome_exc_info(self.outcome)
910
def test_outcome_Extended_py26(self):
911
self.make_26_result()
912
self.check_outcome_details_to_exec_info(self.outcome)
914
def test_outcome_Extended_py27(self):
915
self.make_27_result()
916
self.check_outcome_details_to_exec_info(self.outcome)
918
def test_outcome_Extended_pyextended(self):
919
self.make_extended_result()
920
self.check_outcome_details(self.outcome)
922
def test_outcome__no_details(self):
923
self.make_extended_result()
925
lambda: getattr(self.converter, self.outcome)(self),
926
Raises(MatchesException(ValueError)))
929
class TestExtendedToOriginalAddFailure(
930
TestExtendedToOriginalAddError):
932
outcome = 'addFailure'
935
class TestExtendedToOriginalAddExpectedFailure(
936
TestExtendedToOriginalAddError):
938
outcome = 'addExpectedFailure'
940
def test_outcome_Original_py26(self):
941
self.make_26_result()
942
self.check_outcome_exc_info_to_nothing(self.outcome, 'addSuccess')
944
def test_outcome_Extended_py26(self):
945
self.make_26_result()
946
self.check_outcome_details_to_nothing(self.outcome, 'addSuccess')
950
class TestExtendedToOriginalAddSkip(
951
TestExtendedToOriginalResultDecoratorBase):
955
def test_outcome_Original_py26(self):
956
self.make_26_result()
957
self.check_outcome_string_nothing(self.outcome, 'addSuccess')
959
def test_outcome_Original_py27(self):
960
self.make_27_result()
961
self.check_outcome_string(self.outcome)
963
def test_outcome_Original_pyextended(self):
964
self.make_extended_result()
965
self.check_outcome_string(self.outcome)
967
def test_outcome_Extended_py26(self):
968
self.make_26_result()
969
self.check_outcome_string_nothing(self.outcome, 'addSuccess')
971
def test_outcome_Extended_py27_no_reason(self):
972
self.make_27_result()
973
self.check_outcome_details_to_string(self.outcome)
975
def test_outcome_Extended_py27_reason(self):
976
self.make_27_result()
977
self.check_outcome_details_to_arg(self.outcome, 'foo',
978
{'reason': Content(UTF8_TEXT, lambda:[_b('foo')])})
980
def test_outcome_Extended_pyextended(self):
981
self.make_extended_result()
982
self.check_outcome_details(self.outcome)
984
def test_outcome__no_details(self):
985
self.make_extended_result()
987
lambda: getattr(self.converter, self.outcome)(self),
988
Raises(MatchesException(ValueError)))
991
class TestExtendedToOriginalAddSuccess(
992
TestExtendedToOriginalResultDecoratorBase):
994
outcome = 'addSuccess'
995
expected = 'addSuccess'
997
def test_outcome_Original_py26(self):
998
self.make_26_result()
999
self.check_outcome_nothing(self.outcome, self.expected)
1001
def test_outcome_Original_py27(self):
1002
self.make_27_result()
1003
self.check_outcome_nothing(self.outcome)
1005
def test_outcome_Original_pyextended(self):
1006
self.make_extended_result()
1007
self.check_outcome_nothing(self.outcome)
1009
def test_outcome_Extended_py26(self):
1010
self.make_26_result()
1011
self.check_outcome_details_to_nothing(self.outcome, self.expected)
1013
def test_outcome_Extended_py27(self):
1014
self.make_27_result()
1015
self.check_outcome_details_to_nothing(self.outcome)
1017
def test_outcome_Extended_pyextended(self):
1018
self.make_extended_result()
1019
self.check_outcome_details(self.outcome)
1022
class TestExtendedToOriginalAddUnexpectedSuccess(
1023
TestExtendedToOriginalResultDecoratorBase):
1025
outcome = 'addUnexpectedSuccess'
1026
expected = 'addFailure'
1028
def test_outcome_Original_py26(self):
1029
self.make_26_result()
1030
getattr(self.converter, self.outcome)(self)
1031
[event] = self.result._events
1032
self.assertEqual((self.expected, self), event[:2])
1034
def test_outcome_Original_py27(self):
1035
self.make_27_result()
1036
self.check_outcome_nothing(self.outcome)
1038
def test_outcome_Original_pyextended(self):
1039
self.make_extended_result()
1040
self.check_outcome_nothing(self.outcome)
1042
def test_outcome_Extended_py26(self):
1043
self.make_26_result()
1044
getattr(self.converter, self.outcome)(self)
1045
[event] = self.result._events
1046
self.assertEqual((self.expected, self), event[:2])
1048
def test_outcome_Extended_py27(self):
1049
self.make_27_result()
1050
self.check_outcome_details_to_nothing(self.outcome)
1052
def test_outcome_Extended_pyextended(self):
1053
self.make_extended_result()
1054
self.check_outcome_details(self.outcome)
1057
class TestExtendedToOriginalResultOtherAttributes(
1058
TestExtendedToOriginalResultDecoratorBase):
1060
def test_other_attribute(self):
1061
class OtherExtendedResult:
1065
self.result = OtherExtendedResult()
1066
self.make_converter()
1067
self.assertEqual(1, self.converter.bar)
1068
self.assertEqual(2, self.converter.foo())
1071
class TestNonAsciiResults(TestCase):
1072
"""Test all kinds of tracebacks are cleanly interpreted as unicode
1074
Currently only uses weak "contains" assertions, would be good to be much
1075
stricter about the expected output. This would add a few failures for the
1076
current release of IronPython for instance, which gets some traceback
1081
_u("pa\u026a\u03b8\u0259n"), # Unicode encodings only
1082
_u("\u5357\u7121"), # In ISO 2022 encodings
1083
_u("\xa7\xa7\xa7"), # In ISO 8859 encodings
1085
# Everything but Jython shows syntax errors on the current character
1086
_error_on_character = os.name != "java"
1088
def _run(self, stream, test):
1089
"""Run the test, the same as in testtools.run but not to stdout"""
1090
result = TextTestResult(stream)
1091
result.startTestRun()
1093
return test.run(result)
1095
result.stopTestRun()
1097
def _write_module(self, name, encoding, contents):
1098
"""Create Python module on disk with contents in given encoding"""
1100
# Need to pre-check that the coding is valid or codecs.open drops
1101
# the file without closing it which breaks non-refcounted pythons
1102
codecs.lookup(encoding)
1104
self.skip("Encoding unsupported by implementation: %r" % encoding)
1105
f = codecs.open(os.path.join(self.dir, name + ".py"), "w", encoding)
1111
def _test_external_case(self, testline, coding="ascii", modulelevel="",
1113
"""Create and run a test case in a seperate module"""
1114
self._setup_external_case(testline, coding, modulelevel, suffix)
1115
return self._run_external_case()
1117
def _setup_external_case(self, testline, coding="ascii", modulelevel="",
1119
"""Create a test case in a seperate module"""
1120
_, prefix, self.modname = self.id().rsplit(".", 2)
1121
self.dir = tempfile.mkdtemp(prefix=prefix, suffix=suffix)
1122
self.addCleanup(shutil.rmtree, self.dir)
1123
self._write_module(self.modname, coding,
1124
# Older Python 2 versions don't see a coding declaration in a
1125
# docstring so it has to be in a comment, but then we can't
1126
# workaround bug: <http://ironpython.codeplex.com/workitem/26940>
1128
"import testtools\n"
1130
"class Test(testtools.TestCase):\n"
1131
" def runTest(self):\n"
1132
" %s\n" % (coding, modulelevel, testline))
1134
def _run_external_case(self):
1135
"""Run the prepared test case in a seperate module"""
1136
sys.path.insert(0, self.dir)
1137
self.addCleanup(sys.path.remove, self.dir)
1138
module = __import__(self.modname)
1139
self.addCleanup(sys.modules.pop, self.modname)
1141
self._run(stream, module.Test())
1142
return stream.getvalue()
1144
def _silence_deprecation_warnings(self):
1145
"""Shut up DeprecationWarning for this test only"""
1146
warnings.simplefilter("ignore", DeprecationWarning)
1147
self.addCleanup(warnings.filters.remove, warnings.filters[0])
1149
def _get_sample_text(self, encoding="unicode_internal"):
1150
if encoding is None and str_is_unicode:
1151
encoding = "unicode_internal"
1152
for u in self._sample_texts:
1154
b = u.encode(encoding)
1155
if u == b.decode(encoding):
1159
except (LookupError, UnicodeError):
1161
self.skip("Could not find a sample text for encoding: %r" % encoding)
1163
def _as_output(self, text):
1166
def test_non_ascii_failure_string(self):
1167
"""Assertion contents can be non-ascii and should get decoded"""
1168
text, raw = self._get_sample_text(_get_exception_encoding())
1169
textoutput = self._test_external_case("self.fail(%s)" % _r(raw))
1170
self.assertIn(self._as_output(text), textoutput)
1172
def test_non_ascii_failure_string_via_exec(self):
1173
"""Assertion via exec can be non-ascii and still gets decoded"""
1174
text, raw = self._get_sample_text(_get_exception_encoding())
1175
textoutput = self._test_external_case(
1176
testline='exec ("self.fail(%s)")' % _r(raw))
1177
self.assertIn(self._as_output(text), textoutput)
1179
def test_control_characters_in_failure_string(self):
1180
"""Control characters in assertions should be escaped"""
1181
textoutput = self._test_external_case("self.fail('\\a\\a\\a')")
1182
self.expectFailure("Defense against the beeping horror unimplemented",
1183
self.assertNotIn, self._as_output("\a\a\a"), textoutput)
1184
self.assertIn(self._as_output(_u("\uFFFD\uFFFD\uFFFD")), textoutput)
1186
def test_os_error(self):
1187
"""Locale error messages from the OS shouldn't break anything"""
1188
textoutput = self._test_external_case(
1189
modulelevel="import os",
1190
testline="os.mkdir('/')")
1191
if os.name != "nt" or sys.version_info < (2, 5):
1192
self.assertIn(self._as_output("OSError: "), textoutput)
1194
self.assertIn(self._as_output("WindowsError: "), textoutput)
1196
def test_assertion_text_shift_jis(self):
1197
"""A terminal raw backslash in an encoded string is weird but fine"""
1198
example_text = _u("\u5341")
1199
textoutput = self._test_external_case(
1201
testline="self.fail('%s')" % example_text)
1203
output_text = example_text
1205
output_text = example_text.encode("shift_jis").decode(
1206
_get_exception_encoding(), "replace")
1207
self.assertIn(self._as_output("AssertionError: %s" % output_text),
1210
def test_file_comment_iso2022_jp(self):
1211
"""Control character escapes must be preserved if valid encoding"""
1212
example_text, _ = self._get_sample_text("iso2022_jp")
1213
textoutput = self._test_external_case(
1214
coding="iso2022_jp",
1215
testline="self.fail('Simple') # %s" % example_text)
1216
self.assertIn(self._as_output(example_text), textoutput)
1218
def test_unicode_exception(self):
1219
"""Exceptions that can be formated losslessly as unicode should be"""
1220
example_text, _ = self._get_sample_text()
1222
"class FancyError(Exception):\n"
1223
# A __unicode__ method does nothing on py3k but the default works
1224
" def __unicode__(self):\n"
1225
" return self.args[0]\n")
1226
textoutput = self._test_external_case(
1227
modulelevel=exception_class,
1228
testline="raise FancyError(%s)" % _r(example_text))
1229
self.assertIn(self._as_output(example_text), textoutput)
1231
def test_unprintable_exception(self):
1232
"""A totally useless exception instance still prints something"""
1234
"class UnprintableError(Exception):\n"
1235
" def __str__(self):\n"
1236
" raise RuntimeError\n"
1237
" def __unicode__(self):\n"
1238
" raise RuntimeError\n"
1239
" def __repr__(self):\n"
1240
" raise RuntimeError\n")
1241
textoutput = self._test_external_case(
1242
modulelevel=exception_class,
1243
testline="raise UnprintableError")
1244
self.assertIn(self._as_output(
1245
"UnprintableError: <unprintable UnprintableError object>\n"),
1248
def test_string_exception(self):
1249
"""Raise a string rather than an exception instance if supported"""
1250
if sys.version_info > (2, 6):
1251
self.skip("No string exceptions in Python 2.6 or later")
1252
elif sys.version_info > (2, 5):
1253
self._silence_deprecation_warnings()
1254
textoutput = self._test_external_case(testline="raise 'plain str'")
1255
self.assertIn(self._as_output("\nplain str\n"), textoutput)
1257
def test_non_ascii_dirname(self):
1258
"""Script paths in the traceback can be non-ascii"""
1259
text, raw = self._get_sample_text(sys.getfilesystemencoding())
1260
textoutput = self._test_external_case(
1261
# Avoid bug in Python 3 by giving a unicode source encoding rather
1262
# than just ascii which raises a SyntaxError with no other details
1264
testline="self.fail('Simple')",
1266
self.assertIn(self._as_output(text), textoutput)
1268
def test_syntax_error(self):
1269
"""Syntax errors should still have fancy special-case formatting"""
1270
textoutput = self._test_external_case("exec ('f(a, b c)')")
1271
self.assertIn(self._as_output(
1272
' File "<string>", line 1\n'
1274
+ ' ' * self._error_on_character +
1279
def test_syntax_error_malformed(self):
1280
"""Syntax errors with bogus parameters should break anything"""
1281
textoutput = self._test_external_case("raise SyntaxError(3, 2, 1)")
1282
self.assertIn(self._as_output("\nSyntaxError: "), textoutput)
1284
def test_syntax_error_import_binary(self):
1285
"""Importing a binary file shouldn't break SyntaxError formatting"""
1286
if sys.version_info < (2, 5):
1287
# Python 2.4 assumes the file is latin-1 and tells you off
1288
self._silence_deprecation_warnings()
1289
self._setup_external_case("import bad")
1290
f = open(os.path.join(self.dir, "bad.py"), "wb")
1292
f.write(_b("x\x9c\xcb*\xcd\xcb\x06\x00\x04R\x01\xb9"))
1295
textoutput = self._run_external_case()
1296
self.assertIn(self._as_output("\nSyntaxError: "), textoutput)
1298
def test_syntax_error_line_iso_8859_1(self):
1299
"""Syntax error on a latin-1 line shows the line decoded"""
1300
text, raw = self._get_sample_text("iso-8859-1")
1301
textoutput = self._setup_external_case("import bad")
1302
self._write_module("bad", "iso-8859-1",
1303
"# coding: iso-8859-1\n! = 0 # %s\n" % text)
1304
textoutput = self._run_external_case()
1305
self.assertIn(self._as_output(_u(
1306
#'bad.py", line 2\n'
1310
(text,)), textoutput)
1312
def test_syntax_error_line_iso_8859_5(self):
1313
"""Syntax error on a iso-8859-5 line shows the line decoded"""
1314
text, raw = self._get_sample_text("iso-8859-5")
1315
textoutput = self._setup_external_case("import bad")
1316
self._write_module("bad", "iso-8859-5",
1317
"# coding: iso-8859-5\n%% = 0 # %s\n" % text)
1318
textoutput = self._run_external_case()
1319
self.assertIn(self._as_output(_u(
1320
#'bad.py", line 2\n'
1322
+ ' ' * self._error_on_character +
1325
(text,)), textoutput)
1327
def test_syntax_error_line_euc_jp(self):
1328
"""Syntax error on a euc_jp line shows the line decoded"""
1329
text, raw = self._get_sample_text("euc_jp")
1330
textoutput = self._setup_external_case("import bad")
1331
self._write_module("bad", "euc_jp",
1332
"# coding: euc_jp\n$ = 0 # %s\n" % text)
1333
textoutput = self._run_external_case()
1334
self.assertIn(self._as_output(_u(
1335
#'bad.py", line 2\n'
1337
+ ' ' * self._error_on_character +
1340
(text,)), textoutput)
1342
def test_syntax_error_line_utf_8(self):
1343
"""Syntax error on a utf-8 line shows the line decoded"""
1344
text, raw = self._get_sample_text("utf-8")
1345
textoutput = self._setup_external_case("import bad")
1346
self._write_module("bad", "utf-8", _u("\ufeff^ = 0 # %s\n") % text)
1347
textoutput = self._run_external_case()
1348
self.assertIn(self._as_output(_u(
1351
+ ' ' * self._error_on_character +
1357
class TestNonAsciiResultsWithUnittest(TestNonAsciiResults):
1358
"""Test that running under unittest produces clean ascii strings"""
1360
def _run(self, stream, test):
1361
from unittest import TextTestRunner as _Runner
1362
return _Runner(stream).run(test)
1364
def _as_output(self, text):
1367
return text.encode("utf-8")
1371
from unittest import TestLoader
1372
return TestLoader().loadTestsFromName(__name__)