← Back to branch summary

~ubuntu-branches/ubuntu/utopic/autopilot-legacy/utopic-proposed

« back to all changes in this revision

Viewing changes to autopilot/tests/unit/test_testresults.py

  • Committer: Package Import Robot
  • Author(s): Ubuntu daily release
  • Date: 2014-04-30 00:05:58 UTC
  • Revision ID: package-import@ubuntu.com-20140430000558-zmi37a3yf8podvwr
Tags: upstream-1.4.1+14.10.20140430
Import upstream version 1.4.1+14.10.20140430

Show diffs side-by-side

added added

removed removed

Lines of Context:
autopilot/tests/unit/test_testresults.py
 
1
# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
 
2
#
 
3
# Autopilot Functional Test Tool
 
4
# Copyright (C) 2013 Canonical
 
5
#
 
6
# This program is free software: you can redistribute it and/or modify
 
7
# it under the terms of the GNU General Public License as published by
 
8
# the Free Software Foundation, either version 3 of the License, or
 
9
# (at your option) any later version.
 
10
#
 
11
# This program is distributed in the hope that it will be useful,
 
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
14
# GNU General Public License for more details.
 
15
#
 
16
# You should have received a copy of the GNU General Public License
 
17
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
18
#
 
19
 
 
20
import codecs
 
21
from mock import Mock
 
22
import os
 
23
import tempfile
 
24
from testtools import TestCase, PlaceHolder
 
25
from testtools.content import text_content
 
26
from testtools.matchers import Contains, raises, NotEquals
 
27
from testscenarios import WithScenarios
 
28
import unittest
 
29
 
 
30
from autopilot import testresult
 
31
from autopilot import run
 
32
 
 
33
 
 
34
class LoggedTestResultDecoratorTests(TestCase):
 
35
 
 
36
    def construct_simple_content_object(self):
 
37
        return text_content(self.getUniqueString)
 
38
 
 
39
    def test_can_construct(self):
 
40
        testresult.LoggedTestResultDecorator(Mock())
 
41
 
 
42
    def test_addSuccess_calls_decorated_test(self):
 
43
        wrapped = Mock()
 
44
        result = testresult.LoggedTestResultDecorator(wrapped)
 
45
        fake_test = PlaceHolder('fake_test')
 
46
        fake_details = self.construct_simple_content_object()
 
47
 
 
48
        result.addSuccess(fake_test, fake_details)
 
49
 
 
50
        wrapped.addSuccess.assert_called_once_with(
 
51
            fake_test,
 
52
            details=fake_details
 
53
        )
 
54
 
 
55
    def test_addError_calls_decorated_test(self):
 
56
        wrapped = Mock()
 
57
        result = testresult.LoggedTestResultDecorator(wrapped)
 
58
        fake_test = PlaceHolder('fake_test')
 
59
        fake_error = object()
 
60
        fake_details = self.construct_simple_content_object()
 
61
 
 
62
        result.addError(fake_test, fake_error, fake_details)
 
63
 
 
64
        wrapped.addError.assert_called_once_with(
 
65
            fake_test,
 
66
            fake_error,
 
67
            details=fake_details
 
68
        )
 
69
 
 
70
    def test_addFailure_calls_decorated_test(self):
 
71
        wrapped = Mock()
 
72
        result = testresult.LoggedTestResultDecorator(wrapped)
 
73
        fake_test = PlaceHolder('fake_test')
 
74
        fake_error = object()
 
75
        fake_details = self.construct_simple_content_object()
 
76
 
 
77
        result.addFailure(fake_test, fake_error, fake_details)
 
78
 
 
79
        wrapped.addFailure.assert_called_once_with(
 
80
            fake_test,
 
81
            fake_error,
 
82
            details=fake_details
 
83
        )
 
84
 
 
85
 
 
86
class OutputFormatFactoryTests(TestCase):
 
87
 
 
88
    def test_has_text_format(self):
 
89
        self.assertTrue('text' in testresult.get_output_formats())
 
90
 
 
91
    def test_has_xml_format(self):
 
92
        self.assertTrue('xml' in testresult.get_output_formats())
 
93
 
 
94
    def test_has_subunit_format(self):
 
95
        self.assertTrue('subunit' in testresult.get_output_formats())
 
96
 
 
97
    def test_default_format_is_available(self):
 
98
        self.assertThat(
 
99
            testresult.get_output_formats(),
 
100
            Contains(testresult.get_default_format())
 
101
        )
 
102
 
 
103
 
 
104
class TestResultOutputStreamTests(WithScenarios, TestCase):
 
105
 
 
106
    scenarios = [
 
107
        (f, dict(format=f)) for f in testresult.get_output_formats().keys()
 
108
    ]
 
109
 
 
110
    def get_supported_options(self, **kwargs):
 
111
        """Get a dictionary of all supported keyword arguments for the current
 
112
        result class.
 
113
 
 
114
        Pass in keyword arguments to override default options.
 
115
        """
 
116
        output_path = tempfile.mktemp()
 
117
        self.addCleanup(remove_if_exists, output_path)
 
118
        options = {
 
119
            'stream': run.get_output_stream(self.format, output_path),
 
120
            'failfast': False
 
121
        }
 
122
        options.update(kwargs)
 
123
        return options
 
124
 
 
125
    def run_test_with_result(self, test_suite, **kwargs):
 
126
        """Run the given test with the current result object.
 
127
 
 
128
        Returns the test result and output file path.
 
129
        Use keyword arguments to alter result object options.
 
130
 
 
131
        """
 
132
        ResultClass = testresult.get_output_formats()[self.format]
 
133
        result_options = self.get_supported_options(**kwargs)
 
134
        output_path = result_options['stream'].name
 
135
        result = ResultClass(**result_options)
 
136
        result.startTestRun()
 
137
        test_result = test_suite.run(result)
 
138
        result.stopTestRun()
 
139
        result_options['stream'].flush()
 
140
        return test_result, output_path
 
141
 
 
142
    def test_factory_function_is_a_callable(self):
 
143
        self.assertTrue(
 
144
            callable(testresult.get_output_formats()[self.format])
 
145
        )
 
146
 
 
147
    def test_factory_callable_raises_on_unknown_kwargs(self):
 
148
        factory_fn = testresult.get_output_formats()[self.format]
 
149
        options = self.get_supported_options()
 
150
        options['unknown_kwarg'] = True
 
151
 
 
152
        self.assertThat(
 
153
            lambda: factory_fn(**options),
 
154
            raises(ValueError)
 
155
        )
 
156
 
 
157
    def test_creates_non_empty_file_on_passing_test(self):
 
158
        class PassingTests(TestCase):
 
159
 
 
160
            def test_passes(self):
 
161
                pass
 
162
 
 
163
        test_result, output_path = self.run_test_with_result(
 
164
            PassingTests('test_passes')
 
165
        )
 
166
        self.assertTrue(test_result.wasSuccessful())
 
167
        self.assertThat(open(output_path, 'rb').read(), NotEquals(b''))
 
168
 
 
169
    def test_creates_non_empty_file_on_failing_test(self):
 
170
        class FailingTests(TestCase):
 
171
 
 
172
            def test_fails(self):
 
173
                self.fail("Failing Test: ")
 
174
 
 
175
        test_result, output_path = self.run_test_with_result(
 
176
            FailingTests('test_fails')
 
177
        )
 
178
        self.assertFalse(test_result.wasSuccessful())
 
179
        self.assertThat(open(output_path, 'rb').read(), NotEquals(b''))
 
180
 
 
181
    def test_creates_non_empty_file_on_erroring_test(self):
 
182
        class ErroringTests(TestCase):
 
183
 
 
184
            def test_errors(self):
 
185
                raise RuntimeError("Uncaught Exception!")
 
186
 
 
187
        test_result, output_path = self.run_test_with_result(
 
188
            ErroringTests('test_errors')
 
189
        )
 
190
        self.assertFalse(test_result.wasSuccessful())
 
191
        self.assertThat(open(output_path, 'rb').read(), NotEquals(b''))
 
192
 
 
193
    def test_creates_non_empty_log_file_when_failing_with_unicode(self):
 
194
        class FailingTests(TestCase):
 
195
 
 
196
            def test_fails_unicode(self):
 
197
                self.fail(
 
198
                    u'\xa1pl\u0279oM \u01ddpo\u0254\u0131u\u2229 oll\u01ddH'
 
199
                )
 
200
        test_result, output_path = self.run_test_with_result(
 
201
            FailingTests('test_fails_unicode')
 
202
        )
 
203
        # We need to specify 'errors="ignore"' because subunit write non-valid
 
204
        # unicode data.
 
205
        log_contents = codecs.open(
 
206
            output_path,
 
207
            'r',
 
208
            encoding='utf-8',
 
209
            errors='ignore'
 
210
        ).read()
 
211
        self.assertFalse(test_result.wasSuccessful())
 
212
        self.assertThat(
 
213
            log_contents,
 
214
            Contains(u'\xa1pl\u0279oM \u01ddpo\u0254\u0131u\u2229 oll\u01ddH')
 
215
        )
 
216
 
 
217
    def test_result_object_supports_many_tests(self):
 
218
        class ManyFailingTests(TestCase):
 
219
 
 
220
            def test_fail1(self):
 
221
                self.fail("Failing test")
 
222
 
 
223
            def test_fail2(self):
 
224
                self.fail("Failing test")
 
225
        suite = unittest.TestSuite(
 
226
            tests=(
 
227
                ManyFailingTests('test_fail1'),
 
228
                ManyFailingTests('test_fail2'),
 
229
            )
 
230
        )
 
231
        test_result, output_path = self.run_test_with_result(suite)
 
232
        self.assertFalse(test_result.wasSuccessful())
 
233
        self.assertEqual(2, test_result.testsRun)
 
234
 
 
235
    def test_result_object_supports_failfast(self):
 
236
        class ManyFailingTests(TestCase):
 
237
 
 
238
            def test_fail1(self):
 
239
                self.fail("Failing test")
 
240
 
 
241
            def test_fail2(self):
 
242
                self.fail("Failing test")
 
243
        suite = unittest.TestSuite(
 
244
            tests=(
 
245
                ManyFailingTests('test_fail1'),
 
246
                ManyFailingTests('test_fail2'),
 
247
            )
 
248
        )
 
249
        test_result, output_path = self.run_test_with_result(
 
250
            suite,
 
251
            failfast=True
 
252
        )
 
253
        self.assertFalse(test_result.wasSuccessful())
 
254
        self.assertEqual(1, test_result.testsRun)
 
255
 
 
256
 
 
257
def remove_if_exists(path):
 
258
    if os.path.exists(path):
 
259
        os.remove(path)