← Back to branch summary

~ubuntu-branches/ubuntu/raring/qtwebkit-source/raring-proposed

« back to all changes in this revision

Viewing changes to Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

  • Committer: Package Import Robot
  • Author(s): Jonathan Riddell
  • Date: 2013-02-18 14:24:18 UTC
  • Revision ID: package-import@ubuntu.com-20130218142418-eon0jmjg3nj438uy
Tags: upstream-2.3
Import upstream version 2.3

Show diffs side-by-side

added added

removed removed

Lines of Context:
Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
 
1
# Copyright (C) 2011 Google Inc. All rights reserved.
 
2
#
 
3
# Redistribution and use in source and binary forms, with or without
 
4
# modification, are permitted provided that the following conditions are
 
5
# met:
 
6
#
 
7
#     * Redistributions of source code must retain the above copyright
 
8
# notice, this list of conditions and the following disclaimer.
 
9
#     * Redistributions in binary form must reproduce the above
 
10
# copyright notice, this list of conditions and the following disclaimer
 
11
# in the documentation and/or other materials provided with the
 
12
# distribution.
 
13
#     * Neither the name of Google Inc. nor the names of its
 
14
# contributors may be used to endorse or promote products derived from
 
15
# this software without specific prior written permission.
 
16
#
 
17
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 
18
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 
19
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 
20
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 
21
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 
24
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 
25
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 
26
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 
27
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
28
 
 
29
 
 
30
import logging
 
31
import re
 
32
import time
 
33
 
 
34
from webkitpy.layout_tests.controllers import test_result_writer
 
35
from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
 
36
from webkitpy.layout_tests.models import test_expectations
 
37
from webkitpy.layout_tests.models import test_failures
 
38
from webkitpy.layout_tests.models.test_results import TestResult
 
39
 
 
40
 
 
41
_log = logging.getLogger(__name__)
 
42
 
 
43
 
 
44
def run_single_test(port, options, test_input, driver, worker_name, stop_when_done):
 
45
    runner = SingleTestRunner(options, port, driver, test_input, worker_name, stop_when_done)
 
46
    return runner.run()
 
47
 
 
48
 
 
49
class SingleTestRunner(object):
 
50
    (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update')
 
51
 
 
52
    def __init__(self, options, port, driver, test_input, worker_name, stop_when_done):
 
53
        self._options = options
 
54
        self._port = port
 
55
        self._filesystem = port.host.filesystem
 
56
        self._driver = driver
 
57
        self._timeout = test_input.timeout
 
58
        self._worker_name = worker_name
 
59
        self._test_name = test_input.test_name
 
60
        self._should_run_pixel_test = test_input.should_run_pixel_test
 
61
        self._reference_files = test_input.reference_files
 
62
        self._stop_when_done = stop_when_done
 
63
 
 
64
        if self._reference_files:
 
65
            # Detect and report a test which has a wrong combination of expectation files.
 
66
            # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
 
67
            # 'foo-expected.txt', we should warn users. One test file must be used exclusively
 
68
            # in either layout tests or reftests, but not in both.
 
69
            for suffix in ('.txt', '.png', '.wav'):
 
70
                expected_filename = self._port.expected_filename(self._test_name, suffix)
 
71
                if self._filesystem.exists(expected_filename):
 
72
                    _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.',
 
73
                        self._test_name, expected_filename)
 
74
 
 
75
    def _expected_driver_output(self):
 
76
        return DriverOutput(self._port.expected_text(self._test_name),
 
77
                                 self._port.expected_image(self._test_name),
 
78
                                 self._port.expected_checksum(self._test_name),
 
79
                                 self._port.expected_audio(self._test_name))
 
80
 
 
81
    def _should_fetch_expected_checksum(self):
 
82
        return self._should_run_pixel_test and not (self._options.new_baseline or self._options.reset_results)
 
83
 
 
84
    def _driver_input(self):
 
85
        # The image hash is used to avoid doing an image dump if the
 
86
        # checksums match, so it should be set to a blank value if we
 
87
        # are generating a new baseline.  (Otherwise, an image from a
 
88
        # previous run will be copied into the baseline."""
 
89
        image_hash = None
 
90
        if self._should_fetch_expected_checksum():
 
91
            image_hash = self._port.expected_checksum(self._test_name)
 
92
        return DriverInput(self._test_name, self._timeout, image_hash, self._should_run_pixel_test)
 
93
 
 
94
    def run(self):
 
95
        if self._reference_files:
 
96
            if self._port.get_option('no_ref_tests') or self._options.reset_results:
 
97
                reftest_type = set([reference_file[0] for reference_file in self._reference_files])
 
98
                result = TestResult(self._test_name, reftest_type=reftest_type)
 
99
                result.type = test_expectations.SKIP
 
100
                return result
 
101
            return self._run_reftest()
 
102
        if self._options.reset_results:
 
103
            return self._run_rebaseline()
 
104
        return self._run_compare_test()
 
105
 
 
106
    def _run_compare_test(self):
 
107
        driver_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
 
108
        expected_driver_output = self._expected_driver_output()
 
109
 
 
110
        if self._options.ignore_metrics:
 
111
            expected_driver_output.strip_metrics()
 
112
            driver_output.strip_metrics()
 
113
 
 
114
        test_result = self._compare_output(expected_driver_output, driver_output)
 
115
        if self._options.new_test_results:
 
116
            self._add_missing_baselines(test_result, driver_output)
 
117
        test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, expected_driver_output, test_result.failures)
 
118
        return test_result
 
119
 
 
120
    def _run_rebaseline(self):
 
121
        driver_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
 
122
        failures = self._handle_error(driver_output)
 
123
        test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, None, failures)
 
124
        # FIXME: It the test crashed or timed out, it might be better to avoid
 
125
        # to write new baselines.
 
126
        self._overwrite_baselines(driver_output)
 
127
        return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 
128
 
 
129
    _render_tree_dump_pattern = re.compile(r"^layer at \(\d+,\d+\) size \d+x\d+\n")
 
130
 
 
131
    def _add_missing_baselines(self, test_result, driver_output):
 
132
        missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash)
 
133
        if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
 
134
            self._save_baseline_data(driver_output.text, '.txt', self._location_for_new_baseline(driver_output.text, '.txt'))
 
135
        if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
 
136
            self._save_baseline_data(driver_output.audio, '.wav', self._location_for_new_baseline(driver_output.audio, '.wav'))
 
137
        if missingImage:
 
138
            self._save_baseline_data(driver_output.image, '.png', self._location_for_new_baseline(driver_output.image, '.png'))
 
139
 
 
140
    def _location_for_new_baseline(self, data, extension):
 
141
        if self._options.add_platform_exceptions:
 
142
            return self.VERSION_DIR
 
143
        if extension == '.png':
 
144
            return self.PLATFORM_DIR
 
145
        if extension == '.wav':
 
146
            return self.ALONGSIDE_TEST
 
147
        if extension == '.txt' and self._render_tree_dump_pattern.match(data):
 
148
            return self.PLATFORM_DIR
 
149
        return self.ALONGSIDE_TEST
 
150
 
 
151
    def _overwrite_baselines(self, driver_output):
 
152
        location = self.VERSION_DIR if self._options.add_platform_exceptions else self.UPDATE
 
153
        self._save_baseline_data(driver_output.text, '.txt', location)
 
154
        self._save_baseline_data(driver_output.audio, '.wav', location)
 
155
        if self._should_run_pixel_test:
 
156
            self._save_baseline_data(driver_output.image, '.png', location)
 
157
 
 
158
    def _save_baseline_data(self, data, extension, location):
 
159
        if data is None:
 
160
            return
 
161
        port = self._port
 
162
        fs = self._filesystem
 
163
        if location == self.ALONGSIDE_TEST:
 
164
            output_dir = fs.dirname(port.abspath_for_test(self._test_name))
 
165
        elif location == self.VERSION_DIR:
 
166
            output_dir = fs.join(port.baseline_version_dir(), fs.dirname(self._test_name))
 
167
        elif location == self.PLATFORM_DIR:
 
168
            output_dir = fs.join(port.baseline_platform_dir(), fs.dirname(self._test_name))
 
169
        elif location == self.UPDATE:
 
170
            output_dir = fs.dirname(port.expected_filename(self._test_name, extension))
 
171
        else:
 
172
            raise AssertionError('unrecognized baseline location: %s' % location)
 
173
 
 
174
        fs.maybe_make_directory(output_dir)
 
175
        output_basename = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + extension)
 
176
        output_path = fs.join(output_dir, output_basename)
 
177
        _log.info('Writing new expected result "%s"' % port.relative_test_filename(output_path))
 
178
        port.update_baseline(output_path, data)
 
179
 
 
180
    def _handle_error(self, driver_output, reference_filename=None):
 
181
        """Returns test failures if some unusual errors happen in driver's run.
 
182
 
 
183
        Args:
 
184
          driver_output: The output from the driver.
 
185
          reference_filename: The full path to the reference file which produced the driver_output.
 
186
              This arg is optional and should be used only in reftests until we have a better way to know
 
187
              which html file is used for producing the driver_output.
 
188
        """
 
189
        failures = []
 
190
        fs = self._filesystem
 
191
        if driver_output.timeout:
 
192
            failures.append(test_failures.FailureTimeout(bool(reference_filename)))
 
193
 
 
194
        if reference_filename:
 
195
            testname = self._port.relative_test_filename(reference_filename)
 
196
        else:
 
197
            testname = self._test_name
 
198
 
 
199
        if driver_output.crash:
 
200
            failures.append(test_failures.FailureCrash(bool(reference_filename),
 
201
                                                       driver_output.crashed_process_name,
 
202
                                                       driver_output.crashed_pid))
 
203
            if driver_output.error:
 
204
                _log.debug("%s %s crashed, (stderr lines):" % (self._worker_name, testname))
 
205
            else:
 
206
                _log.debug("%s %s crashed, (no stderr)" % (self._worker_name, testname))
 
207
        elif driver_output.error:
 
208
            _log.debug("%s %s output stderr lines:" % (self._worker_name, testname))
 
209
        for line in driver_output.error.splitlines():
 
210
            _log.debug("  %s" % line)
 
211
        return failures
 
212
 
 
213
    def _compare_output(self, expected_driver_output, driver_output):
 
214
        failures = []
 
215
        failures.extend(self._handle_error(driver_output))
 
216
 
 
217
        if driver_output.crash:
 
218
            # Don't continue any more if we already have a crash.
 
219
            # In case of timeouts, we continue since we still want to see the text and image output.
 
220
            return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 
221
 
 
222
        failures.extend(self._compare_text(expected_driver_output.text, driver_output.text))
 
223
        failures.extend(self._compare_audio(expected_driver_output.audio, driver_output.audio))
 
224
        if self._should_run_pixel_test:
 
225
            failures.extend(self._compare_image(expected_driver_output, driver_output))
 
226
        return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 
227
 
 
228
    def _compare_text(self, expected_text, actual_text):
 
229
        failures = []
 
230
        if (expected_text and actual_text and
 
231
            # Assuming expected_text is already normalized.
 
232
            self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))):
 
233
            failures.append(test_failures.FailureTextMismatch())
 
234
        elif actual_text and not expected_text:
 
235
            failures.append(test_failures.FailureMissingResult())
 
236
        return failures
 
237
 
 
238
    def _compare_audio(self, expected_audio, actual_audio):
 
239
        failures = []
 
240
        if (expected_audio and actual_audio and
 
241
            self._port.do_audio_results_differ(expected_audio, actual_audio)):
 
242
            failures.append(test_failures.FailureAudioMismatch())
 
243
        elif actual_audio and not expected_audio:
 
244
            failures.append(test_failures.FailureMissingAudio())
 
245
        return failures
 
246
 
 
247
    def _get_normalized_output_text(self, output):
 
248
        """Returns the normalized text output, i.e. the output in which
 
249
        the end-of-line characters are normalized to "\n"."""
 
250
        # Running tests on Windows produces "\r\n".  The "\n" part is helpfully
 
251
        # changed to "\r\n" by our system (Python/Cygwin), resulting in
 
252
        # "\r\r\n", when, in fact, we wanted to compare the text output with
 
253
        # the normalized text expectation files.
 
254
        return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n")
 
255
 
 
256
    # FIXME: This function also creates the image diff. Maybe that work should
 
257
    # be handled elsewhere?
 
258
    def _compare_image(self, expected_driver_output, driver_output):
 
259
        failures = []
 
260
        # If we didn't produce a hash file, this test must be text-only.
 
261
        if driver_output.image_hash is None:
 
262
            return failures
 
263
        if not expected_driver_output.image:
 
264
            failures.append(test_failures.FailureMissingImage())
 
265
        elif not expected_driver_output.image_hash:
 
266
            failures.append(test_failures.FailureMissingImageHash())
 
267
        elif driver_output.image_hash != expected_driver_output.image_hash:
 
268
            diff_result = self._port.diff_image(expected_driver_output.image, driver_output.image)
 
269
            err_str = diff_result[2]
 
270
            if err_str:
 
271
                _log.warning('  %s : %s' % (self._test_name, err_str))
 
272
                failures.append(test_failures.FailureImageHashMismatch())
 
273
                driver_output.error = (driver_output.error or '') + err_str
 
274
            else:
 
275
                driver_output.image_diff = diff_result[0]
 
276
                if driver_output.image_diff:
 
277
                    failures.append(test_failures.FailureImageHashMismatch(diff_result[1]))
 
278
                else:
 
279
                    # See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure.
 
280
                    _log.warning('  %s -> pixel hash failed (but diff passed)' % self._test_name)
 
281
        return failures
 
282
 
 
283
    def _run_reftest(self):
 
284
        test_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
 
285
        total_test_time = 0
 
286
        reference_output = None
 
287
        test_result = None
 
288
 
 
289
        # A reftest can have multiple match references and multiple mismatch references;
 
290
        # the test fails if any mismatch matches and all of the matches don't match.
 
291
        # To minimize the number of references we have to check, we run all of the mismatches first,
 
292
        # then the matches, and short-circuit out as soon as we can.
 
293
        # Note that sorting by the expectation sorts "!=" before "==" so this is easy to do.
 
294
 
 
295
        putAllMismatchBeforeMatch = sorted
 
296
        for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files):
 
297
            reference_test_name = self._port.relative_test_filename(reference_filename)
 
298
            reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, None, should_run_pixel_test=True), self._stop_when_done)
 
299
            test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=')
 
300
 
 
301
            if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):
 
302
                break
 
303
            total_test_time += test_result.test_run_time
 
304
 
 
305
        assert(reference_output)
 
306
        test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, test_output, reference_output, test_result.failures)
 
307
        reftest_type = set([reference_file[0] for reference_file in self._reference_files])
 
308
        return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr, reftest_type=reftest_type)
 
309
 
 
310
    def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
 
311
        total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
 
312
        has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
 
313
        failures = []
 
314
        failures.extend(self._handle_error(actual_driver_output))
 
315
        if failures:
 
316
            # Don't continue any more if we already have crash or timeout.
 
317
            return TestResult(self._test_name, failures, total_test_time, has_stderr)
 
318
        failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename))
 
319
        if failures:
 
320
            return TestResult(self._test_name, failures, total_test_time, has_stderr)
 
321
 
 
322
        if not reference_driver_output.image_hash and not actual_driver_output.image_hash:
 
323
            failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
 
324
        elif mismatch:
 
325
            if reference_driver_output.image_hash == actual_driver_output.image_hash:
 
326
                diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
 
327
                if not diff_result[0]:
 
328
                    failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
 
329
                else:
 
330
                    _log.warning("  %s -> ref test hashes matched but diff failed" % self._test_name)
 
331
 
 
332
        elif reference_driver_output.image_hash != actual_driver_output.image_hash:
 
333
            diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
 
334
            if diff_result[0]:
 
335
                failures.append(test_failures.FailureReftestMismatch(reference_filename))
 
336
            else:
 
337
                _log.warning("  %s -> ref test hashes didn't match but diff passed" % self._test_name)
 
338
 
 
339
        return TestResult(self._test_name, failures, total_test_time, has_stderr)