1
import gc, StringIO, sys
3
from twisted.python import log
4
from twisted.trial import unittest, runner, reporter, util
5
from twisted.trial.test import erroneous, suppression
8
class ResultsTestMixin:
9
def loadSuite(self, suite):
10
self.loader = runner.TestLoader()
11
self.suite = self.loader.loadClass(suite)
12
self.reporter = reporter.TestResult()
15
self.failUnless(self.reporter.wasSuccessful())
16
self.failUnlessEqual(self.reporter.errors, [])
17
self.failUnlessEqual(self.reporter.failures, [])
18
self.failUnlessEqual(self.reporter.skips, [])
20
def assertCount(self, numTests):
21
self.failUnlessEqual(self.suite.countTestCases(), numTests)
22
self.suite(self.reporter)
23
self.failUnlessEqual(self.reporter.testsRun, numTests)
27
class TestSuccess(unittest.TestCase):
29
Test that successful tests are reported as such.
33
self.result = reporter.TestResult()
36
def test_successful(self):
38
A successful test, used by other tests.
42
def assertSuccessful(self, test, result):
43
self.assertEqual(result.successes, [(test,)])
44
self.assertEqual(result.failures, [])
45
self.assertEqual(result.errors, [])
46
self.assertEqual(result.expectedFailures, [])
47
self.assertEqual(result.unexpectedSuccesses, [])
48
self.assertEqual(result.skips, [])
51
def test_successfulIsReported(self):
53
Test that when a successful test is run, it is reported as a success,
54
and not as any other kind of result.
56
test = TestSuccess('test_successful')
58
self.assertSuccessful(test, self.result)
61
def test_defaultIsSuccessful(self):
63
Test that L{unittest.TestCase} itself can be instantiated, run, and
64
reported as being successful.
66
test = unittest.TestCase()
68
self.assertSuccessful(test, self.result)
72
class TestSkipMethods(unittest.TestCase, ResultsTestMixin):
73
class SkippingTests(unittest.TestCase):
75
raise unittest.SkipTest('skip1')
78
raise RuntimeError("I should not get raised")
79
test_skip2.skip = 'skip2'
82
self.fail('I should not fail')
83
test_skip3.skip = 'skip3'
85
class SkippingSetUp(unittest.TestCase):
87
raise unittest.SkipTest('skipSetUp')
96
self.loadSuite(TestSkipMethods.SkippingTests)
98
def test_counting(self):
101
def test_results(self):
102
self.suite(self.reporter)
103
self.failUnless(self.reporter.wasSuccessful())
104
self.failUnlessEqual(self.reporter.errors, [])
105
self.failUnlessEqual(self.reporter.failures, [])
106
self.failUnlessEqual(len(self.reporter.skips), 3)
108
def test_setUp(self):
109
self.loadSuite(TestSkipMethods.SkippingSetUp)
110
self.suite(self.reporter)
111
self.failUnless(self.reporter.wasSuccessful())
112
self.failUnlessEqual(self.reporter.errors, [])
113
self.failUnlessEqual(self.reporter.failures, [])
114
self.failUnlessEqual(len(self.reporter.skips), 2)
116
def test_reasons(self):
117
self.suite(self.reporter)
119
# whiteboxing reporter
120
for test, reason in self.reporter.skips:
121
self.failUnlessEqual(test.shortDescription()[len(prefix):],
125
class TestSkipClasses(unittest.TestCase, ResultsTestMixin):
126
class SkippedClass(unittest.TestCase):
128
def setUpClass(self):
129
self.__class__._setUpClassRan = True
131
self.__class__._setUpRan = True
132
def tearDownClass(self):
133
self.__class__._tearDownClassRan = True
134
def test_skip1(self):
135
raise unittest.SkipTest('skip1')
136
def test_skip2(self):
137
raise RuntimeError("Ought to skip me")
138
test_skip2.skip = 'skip2'
139
def test_skip3(self):
141
def test_skip4(self):
142
raise RuntimeError("Skip me too")
145
self.loadSuite(TestSkipClasses.SkippedClass)
146
TestSkipClasses.SkippedClass._setUpRan = False
147
TestSkipClasses.SkippedClass._setUpClassRan = False
148
TestSkipClasses.SkippedClass._tearDownClassRan = False
150
def test_counting(self):
153
def test_setUpRan(self):
154
self.suite(self.reporter)
155
self.failUnlessEqual(TestSkipClasses.SkippedClass._setUpRan, False)
156
self.failUnlessEqual(TestSkipClasses.SkippedClass._setUpClassRan,
158
self.failUnlessEqual(TestSkipClasses.SkippedClass._tearDownClassRan,
161
def test_results(self):
162
self.suite(self.reporter)
163
self.failUnless(self.reporter.wasSuccessful())
164
self.failUnlessEqual(self.reporter.errors, [])
165
self.failUnlessEqual(self.reporter.failures, [])
166
self.failUnlessEqual(len(self.reporter.skips), 4)
168
def test_reasons(self):
169
self.suite(self.reporter)
170
expectedReasons = ['class', 'skip2', 'class', 'class']
172
reasonsGiven = [ reason for test, reason in self.reporter.skips ]
173
self.failUnlessEqual(expectedReasons, reasonsGiven)
176
class TestSkipClassesRaised(unittest.TestCase, ResultsTestMixin):
177
class SkippedClass(unittest.TestCase):
178
def setUpClass(self):
179
raise unittest.SkipTest("class")
181
self.__class__._setUpRan = True
182
def tearDownClass(self):
183
self.__class__._tearDownClassRan = True
184
def test_skip1(self):
185
raise unittest.SkipTest('skip1')
186
def test_skip2(self):
187
raise RuntimeError("Ought to skip me")
188
test_skip2.skip = 'skip2'
189
def test_skip3(self):
191
def test_skip4(self):
192
raise RuntimeError("Skip me too")
195
if hasattr(TestSkipClassesRaised.SkippedClass, 'skip'):
196
delattr(TestSkipClassesRaised.SkippedClass, 'skip')
197
self.loadSuite(TestSkipClassesRaised.SkippedClass)
198
TestSkipClassesRaised.SkippedClass._setUpRan = False
199
TestSkipClassesRaised.SkippedClass._tearDownClassRan = False
201
def test_counting(self):
204
def test_setUpRan(self):
205
self.suite(self.reporter)
206
self.failUnlessEqual(
207
TestSkipClassesRaised.SkippedClass._setUpRan, False)
209
def test_tearDownClassRan(self):
210
self.suite(self.reporter)
211
self.failUnlessEqual(
212
TestSkipClassesRaised.SkippedClass._tearDownClassRan, False)
214
def test_results(self):
215
self.suite(self.reporter)
216
self.failUnless(self.reporter.wasSuccessful())
217
self.failUnlessEqual(self.reporter.errors, [])
218
self.failUnlessEqual(self.reporter.failures, [])
219
self.failUnlessEqual(len(self.reporter.skips), 4)
221
def test_reasons(self):
222
self.suite(self.reporter)
223
expectedReasons = ['class', 'skip2', 'class', 'class']
225
reasonsGiven = [ reason for test, reason in self.reporter.skips ]
226
self.failUnlessEqual(expectedReasons, reasonsGiven)
229
class TestTodo(unittest.TestCase, ResultsTestMixin):
230
class TodoTests(unittest.TestCase):
231
def test_todo1(self):
232
self.fail("deliberate failure")
233
test_todo1.todo = "todo1"
235
def test_todo2(self):
236
raise RuntimeError("deliberate error")
237
test_todo2.todo = "todo2"
239
def test_todo3(self):
240
"""unexpected success"""
241
test_todo3.todo = 'todo3'
244
self.loadSuite(TestTodo.TodoTests)
246
def test_counting(self):
249
def test_results(self):
250
self.suite(self.reporter)
251
self.failUnless(self.reporter.wasSuccessful())
252
self.failUnlessEqual(self.reporter.errors, [])
253
self.failUnlessEqual(self.reporter.failures, [])
254
self.failUnlessEqual(self.reporter.skips, [])
255
self.failUnlessEqual(len(self.reporter.expectedFailures), 2)
256
self.failUnlessEqual(len(self.reporter.unexpectedSuccesses), 1)
258
def test_expectedFailures(self):
259
self.suite(self.reporter)
260
expectedReasons = ['todo1', 'todo2']
261
reasonsGiven = [ r.reason
262
for t, e, r in self.reporter.expectedFailures ]
263
self.failUnlessEqual(expectedReasons, reasonsGiven)
265
def test_unexpectedSuccesses(self):
266
self.suite(self.reporter)
267
expectedReasons = ['todo3']
268
reasonsGiven = [ r.reason
269
for t, r in self.reporter.unexpectedSuccesses ]
270
self.failUnlessEqual(expectedReasons, reasonsGiven)
273
class TestTodoClass(unittest.TestCase, ResultsTestMixin):
274
class TodoClass(unittest.TestCase):
275
def test_todo1(self):
277
test_todo1.todo = "method"
278
def test_todo2(self):
280
def test_todo3(self):
281
self.fail("Deliberate Failure")
282
test_todo3.todo = "method"
283
def test_todo4(self):
284
self.fail("Deliberate Failure")
285
TodoClass.todo = "class"
288
self.loadSuite(TestTodoClass.TodoClass)
290
def test_counting(self):
293
def test_results(self):
294
self.suite(self.reporter)
295
self.failUnless(self.reporter.wasSuccessful())
296
self.failUnlessEqual(self.reporter.errors, [])
297
self.failUnlessEqual(self.reporter.failures, [])
298
self.failUnlessEqual(self.reporter.skips, [])
299
self.failUnlessEqual(len(self.reporter.expectedFailures), 2)
300
self.failUnlessEqual(len(self.reporter.unexpectedSuccesses), 2)
302
def test_expectedFailures(self):
303
self.suite(self.reporter)
304
expectedReasons = ['method', 'class']
305
reasonsGiven = [ r.reason
306
for t, e, r in self.reporter.expectedFailures ]
307
self.failUnlessEqual(expectedReasons, reasonsGiven)
309
def test_unexpectedSuccesses(self):
310
self.suite(self.reporter)
311
expectedReasons = ['method', 'class']
312
reasonsGiven = [ r.reason
313
for t, r in self.reporter.unexpectedSuccesses ]
314
self.failUnlessEqual(expectedReasons, reasonsGiven)
317
class TestStrictTodo(unittest.TestCase, ResultsTestMixin):
318
class Todos(unittest.TestCase):
319
def test_todo1(self):
320
raise RuntimeError, "expected failure"
321
test_todo1.todo = (RuntimeError, "todo1")
323
def test_todo2(self):
324
raise RuntimeError, "expected failure"
325
test_todo2.todo = ((RuntimeError, OSError), "todo2")
327
def test_todo3(self):
328
raise RuntimeError, "we had no idea!"
329
test_todo3.todo = (OSError, "todo3")
331
def test_todo4(self):
332
raise RuntimeError, "we had no idea!"
333
test_todo4.todo = ((OSError, SyntaxError), "todo4")
335
def test_todo5(self):
336
self.fail("deliberate failure")
337
test_todo5.todo = (unittest.FailTest, "todo5")
339
def test_todo6(self):
340
self.fail("deliberate failure")
341
test_todo6.todo = (RuntimeError, "todo6")
343
def test_todo7(self):
345
test_todo7.todo = (RuntimeError, "todo7")
348
self.loadSuite(TestStrictTodo.Todos)
350
def test_counting(self):
353
def test_results(self):
354
self.suite(self.reporter)
355
self.failIf(self.reporter.wasSuccessful())
356
self.failUnlessEqual(len(self.reporter.errors), 2)
357
self.failUnlessEqual(len(self.reporter.failures), 1)
358
self.failUnlessEqual(len(self.reporter.expectedFailures), 3)
359
self.failUnlessEqual(len(self.reporter.unexpectedSuccesses), 1)
360
self.failUnlessEqual(self.reporter.skips, [])
362
def test_expectedFailures(self):
363
self.suite(self.reporter)
364
expectedReasons = ['todo1', 'todo2', 'todo5']
365
reasonsGotten = [ r.reason
366
for t, e, r in self.reporter.expectedFailures ]
367
self.failUnlessEqual(expectedReasons, reasonsGotten)
369
def test_unexpectedSuccesses(self):
370
self.suite(self.reporter)
371
expectedReasons = [([RuntimeError], 'todo7')]
372
reasonsGotten = [ (r.errors, r.reason)
373
for t, r in self.reporter.unexpectedSuccesses ]
374
self.failUnlessEqual(expectedReasons, reasonsGotten)
377
class _CleanUpReporter(reporter.Reporter):
379
super(_CleanUpReporter, self).__init__(StringIO.StringIO(), 'default',
382
def cleanupErrors(self, errs):
383
self.cleanerrs = errs
386
class TestCleanup(unittest.TestCase):
388
self.result = _CleanUpReporter()
389
self.loader = runner.TestLoader()
391
def testLeftoverSockets(self):
392
suite = self.loader.loadMethod(
393
erroneous.SocketOpenTest.test_socketsLeftOpen)
394
suite.run(self.result)
395
self.assert_(self.result.cleanerrs)
396
self.assert_(isinstance(self.result.cleanerrs.value,
397
util.DirtyReactorError))
399
def testLeftoverPendingCalls(self):
400
suite = erroneous.ReactorCleanupTests('test_leftoverPendingCalls')
401
suite.run(self.result)
402
self.assert_(self.result.cleanerrs)
403
self.assert_(isinstance(self.result.cleanerrs.value,
404
util.PendingTimedCallsError))
407
class BogusReporter(reporter.Reporter):
409
super(BogusReporter, self).__init__(StringIO.StringIO(), 'default',
412
def upDownError(self, method, error, warn, printStatus):
413
super(BogusReporter, self).upDownError(method, error, False,
415
self.udeMethod = method
418
class FixtureTest(unittest.TestCase):
420
self.reporter = BogusReporter()
421
self.loader = runner.TestLoader()
423
def testBrokenSetUp(self):
424
self.loader.loadClass(erroneous.TestFailureInSetUp).run(self.reporter)
425
imi = self.reporter.udeMethod
426
self.assertEqual(imi, 'setUp')
427
self.assert_(len(self.reporter.errors) > 0)
428
self.assert_(isinstance(self.reporter.errors[0][1].value,
429
erroneous.FoolishError))
431
def testBrokenTearDown(self):
432
suite = self.loader.loadClass(erroneous.TestFailureInTearDown)
433
suite.run(self.reporter)
434
imi = self.reporter.udeMethod
435
self.assertEqual(imi, 'tearDown')
436
errors = self.reporter.errors
437
self.assert_(len(errors) > 0)
438
self.assert_(isinstance(errors[0][1].value, erroneous.FoolishError))
440
def testBrokenSetUpClass(self):
441
suite = self.loader.loadClass(erroneous.TestFailureInSetUpClass)
442
suite.run(self.reporter)
443
imi = self.reporter.udeMethod
444
self.assertEqual(imi, 'setUpClass')
445
self.assert_(self.reporter.errors)
447
def testBrokenTearDownClass(self):
448
suite = self.loader.loadClass(erroneous.TestFailureInTearDownClass)
449
suite.run(self.reporter)
450
imi = self.reporter.udeMethod
451
self.assertEqual(imi, 'tearDownClass')
454
class FixtureMetaTest(unittest.TestCase):
455
def test_testBrokenTearDownClass(self):
456
"""FixtureTest.testBrokenTearDownClass succeeds when run twice
458
test = FixtureTest('testBrokenTearDownClass')
459
result = reporter.TestResult()
461
self.failUnless(result.wasSuccessful())
462
result2 = reporter.TestResult()
464
self.failUnless(result2.wasSuccessful())
467
class SuppressionTest(unittest.TestCase):
468
def runTests(self, suite):
469
suite.run(reporter.TestResult())
472
self.stream = StringIO.StringIO()
473
self._stdout, sys.stdout = sys.stdout, self.stream
474
self.loader = runner.TestLoader()
477
sys.stdout = self._stdout
481
return self.stream.getvalue()
483
def testSuppressMethod(self):
484
self.runTests(self.loader.loadMethod(
485
suppression.TestSuppression.testSuppressMethod))
486
self.assertNotSubstring(suppression.METHOD_WARNING_MSG, self.getIO())
487
self.assertSubstring(suppression.CLASS_WARNING_MSG, self.getIO())
488
self.assertSubstring(suppression.MODULE_WARNING_MSG, self.getIO())
490
def testSuppressClass(self):
491
self.runTests(self.loader.loadMethod(
492
suppression.TestSuppression.testSuppressClass))
493
self.assertSubstring(suppression.METHOD_WARNING_MSG, self.getIO())
494
self.assertNotSubstring(suppression.CLASS_WARNING_MSG, self.getIO())
495
self.assertSubstring(suppression.MODULE_WARNING_MSG, self.getIO())
497
def testSuppressModule(self):
498
self.runTests(self.loader.loadMethod(
499
suppression.TestSuppression2.testSuppressModule))
500
self.assertSubstring(suppression.METHOD_WARNING_MSG, self.getIO())
501
self.assertSubstring(suppression.CLASS_WARNING_MSG, self.getIO())
502
self.assertNotSubstring(suppression.MODULE_WARNING_MSG, self.getIO())
504
def testOverrideSuppressClass(self):
505
self.runTests(self.loader.loadMethod(
506
suppression.TestSuppression.testOverrideSuppressClass))
507
self.assertSubstring(suppression.CLASS_WARNING_MSG, self.getIO())
508
self.assertSubstring(suppression.MODULE_WARNING_MSG, self.getIO())
509
self.assertSubstring(suppression.METHOD_WARNING_MSG, self.getIO())
513
"""I provide a few mock tests that log setUp, tearDown, test execution and
514
garbage collection. I'm used to test whether gc.collect gets called.
517
class BasicTest(unittest.TestCase):
523
self._log('tearDown')
525
class ClassTest(unittest.TestCase):
530
def tearDownClass(self):
531
self._log('tearDownClass')
534
self._collectCalled.append(msg)
537
"""Fake gc.collect"""
541
self._collectCalled = []
542
self.BasicTest._log = self.ClassTest._log = self._log
543
self._oldCollect = gc.collect
544
gc.collect = self.collect
547
gc.collect = self._oldCollect
550
class TestGarbageCollectionDefault(GCMixin, unittest.TestCase):
551
def test_collectNotDefault(self):
553
By default, tests should not force garbage collection.
555
test = self.BasicTest('test_foo')
556
result = reporter.TestResult()
558
self.failUnlessEqual(self._collectCalled, ['setUp', 'test', 'tearDown'])
561
class TestGarbageCollection(GCMixin, unittest.TestCase):
562
def test_collectCalled(self):
563
"""test gc.collect is called before and after each test
565
test = TestGarbageCollection.BasicTest('test_foo')
566
test.forceGarbageCollection = True
567
result = reporter.TestResult()
569
self.failUnlessEqual(
571
['collect', 'setUp', 'test', 'tearDown', 'collect'])
573
def test_collectCalledWhenTearDownClass(self):
574
"""test gc.collect is called after tearDownClasss"""
575
tests = [TestGarbageCollection.ClassTest('test_1'),
576
TestGarbageCollection.ClassTest('test_2')]
578
t.forceGarbageCollection = True
579
test = runner.TestSuite(tests)
580
result = reporter.TestResult()
582
# check that collect gets called after individual tests, and
583
# after tearDownClass
584
self.failUnlessEqual(
586
['collect', 'test1', 'collect',
587
'collect', 'test2', 'tearDownClass', 'collect'])
590
class TestUnhandledDeferred(unittest.TestCase):
592
from twisted.trial.test import weird
593
# test_unhandledDeferred creates a cycle. we need explicit control of gc
595
self.test1 = weird.TestBleeding('test_unhandledDeferred')
596
self.test1.forceGarbageCollection = True
598
def test_isReported(self):
600
Forcing garbage collection should cause unhandled Deferreds to be
603
result = reporter.TestResult()
605
self.assertEqual(len(result.errors), 1,
606
'Unhandled deferred passed without notice')
608
def test_doesntBleed(self):
610
Forcing garbage collection in the test should mean that there are
611
no unreachable cycles immediately after the test completes.
613
result = reporter.TestResult()
615
self.flushLoggedErrors() # test1 logs errors that get caught be us.
616
# test1 created unreachable cycle.
617
# it & all others should have been collected by now.
619
self.assertEqual(n, 0, 'unreachable cycle still existed')
620
# check that last gc.collect didn't log more errors
621
x = self.flushLoggedErrors()
622
self.assertEqual(len(x), 0, 'Errors logged after gc.collect')
627
self.flushLoggedErrors()
added
removed
src/twisted/trial/test/test_tests.py
