1
# Copyright (C) 2009 Google Inc. All rights reserved.
3
# Redistribution and use in source and binary forms, with or without
4
# modification, are permitted provided that the following conditions are
7
# * Redistributions of source code must retain the above copyright
8
# notice, this list of conditions and the following disclaimer.
9
# * Redistributions in binary form must reproduce the above
10
# copyright notice, this list of conditions and the following disclaimer
11
# in the documentation and/or other materials provided with the
13
# * Neither the name of Google Inc. nor the names of its
14
# contributors may be used to endorse or promote products derived from
15
# this software without specific prior written permission.
17
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
from webkitpy.common.net.layouttestresults import LayoutTestResults
32
from webkitpy.common.net.buildbot import BuildBot, Builder, Build
33
from webkitpy.layout_tests.models import test_results
34
from webkitpy.layout_tests.models import test_failures
35
from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
38
class BuilderTest(unittest.TestCase):
39
def _mock_test_result(self, testname):
40
return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
42
def _install_fetch_build(self, failure):
43
def _mock_fetch_build(build_number):
46
build_number=build_number,
47
revision=build_number + 1000,
48
is_green=build_number < 4
50
results = [self._mock_test_result(testname) for testname in failure(build_number)]
51
layout_test_results = LayoutTestResults(results)
52
def mock_layout_test_results():
53
return layout_test_results
54
build.layout_test_results = mock_layout_test_results
56
self.builder._fetch_build = _mock_fetch_build
59
self.buildbot = BuildBot()
60
self.builder = Builder(u"Test Builder \u2661", self.buildbot)
61
self._install_fetch_build(lambda build_number: ["test1", "test2"])
63
def test_latest_layout_test_results(self):
64
self.builder.fetch_layout_test_results = lambda results_url: LayoutTestResults([self._mock_test_result(testname) for testname in ["test1", "test2"]])
65
self.builder.accumulated_results_url = lambda: "http://dummy_url.org"
66
self.assertTrue(self.builder.latest_layout_test_results())
68
def test_find_regression_window(self):
69
regression_window = self.builder.find_regression_window(self.builder.build(10))
70
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
71
self.assertEqual(regression_window.failing_build().revision(), 1004)
73
regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
74
self.assertEqual(regression_window.build_before_failure(), None)
75
self.assertEqual(regression_window.failing_build().revision(), 1008)
77
def test_none_build(self):
78
self.builder._fetch_build = lambda build_number: None
79
regression_window = self.builder.find_regression_window(self.builder.build(10))
80
self.assertEqual(regression_window.build_before_failure(), None)
81
self.assertEqual(regression_window.failing_build(), None)
83
def test_flaky_tests(self):
84
self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
85
regression_window = self.builder.find_regression_window(self.builder.build(10))
86
self.assertEqual(regression_window.build_before_failure().revision(), 1009)
87
self.assertEqual(regression_window.failing_build().revision(), 1010)
89
def test_failure_and_flaky(self):
90
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
91
regression_window = self.builder.find_regression_window(self.builder.build(10))
92
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
93
self.assertEqual(regression_window.failing_build().revision(), 1004)
95
def test_no_results(self):
96
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
97
regression_window = self.builder.find_regression_window(self.builder.build(10))
98
self.assertEqual(regression_window.build_before_failure().revision(), 1003)
99
self.assertEqual(regression_window.failing_build().revision(), 1004)
101
def test_failure_after_flaky(self):
102
self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
103
regression_window = self.builder.find_regression_window(self.builder.build(10))
104
self.assertEqual(regression_window.build_before_failure().revision(), 1006)
105
self.assertEqual(regression_window.failing_build().revision(), 1007)
107
def test_find_blameworthy_regression_window(self):
108
self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
109
self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
110
# Flakey test avoidance requires at least 2 red builds:
111
self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
112
self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
114
self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
116
def test_build_caching(self):
117
self.assertEqual(self.builder.build(10), self.builder.build(10))
119
def test_build_and_revision_for_filename(self):
121
"r47483 (1)/" : (47483, 1),
122
"r47483 (1).zip" : (47483, 1),
125
for filename, revision_and_build in expectations.items():
126
self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
128
def test_file_info_list_to_revision_to_build_list(self):
130
{"filename": "r47483 (1)/"},
131
{"filename": "r47483 (1).zip"},
132
{"filename": "random junk"},
134
builds_and_revisions_list = [(47483, 1), (47483, 1)]
135
self.assertEqual(self.builder._file_info_list_to_revision_to_build_list(file_info_list), builds_and_revisions_list)
137
def test_fetch_build(self):
138
buildbot = BuildBot()
139
builder = Builder(u"Test Builder \u2661", buildbot)
141
def mock_fetch_build_dictionary(self, build_number):
144
"revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
146
"number": int(build_number),
147
# Intentionally missing the 'results' key, meaning it's a "pass" build.
149
return build_dictionary
150
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
151
self.assertNotEqual(builder._fetch_build(1), None)
154
class BuildTest(unittest.TestCase):
155
def test_layout_test_results(self):
156
buildbot = BuildBot()
157
builder = Builder(u"Foo Builder (test)", buildbot)
158
builder._fetch_file_from_results = lambda results_url, file_name: None
159
build = Build(builder, None, None, None)
160
# Test that layout_test_results() returns None if the fetch fails.
161
self.assertEqual(build.layout_test_results(), None)
164
class BuildBotTest(unittest.TestCase):
166
_example_one_box_status = '''
169
<td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td>
170
<td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td>
171
<td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td>
173
<td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td>
174
<td class="LastBuild box" >no build</td>
175
<td align="center" class="Activity building">building<br />< 1 min</td>
177
<td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td>
178
<td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td>
179
<td align="center" class="Activity idle">idle<br />3 pending</td>
181
<td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td>
182
<td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td>
183
<td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td>
186
_expected_example_one_box_parsings = [
189
'build_number' : 3693,
190
'name': u'Windows Debug (Tests)',
191
'built_revision': 47380,
192
'activity': 'building',
197
'build_number' : None,
198
'name': u'SnowLeopard Intel Release',
199
'built_revision': None,
200
'activity': 'building',
205
'build_number' : 654,
206
'name': u'Qt Linux Release',
207
'built_revision': 47383,
213
'build_number' : 2090,
214
'name': u'Qt Windows 32-bit Debug',
215
'built_revision': 60563,
216
'activity': 'building',
221
def test_status_parsing(self):
222
buildbot = BuildBot()
224
soup = BeautifulSoup(self._example_one_box_status)
225
status_table = soup.find("table")
226
input_rows = status_table.findAll('tr')
228
for x in range(len(input_rows)):
229
status_row = input_rows[x]
230
expected_parsing = self._expected_example_one_box_parsings[x]
232
builder = buildbot._parse_builder_status_from_row(status_row)
234
# Make sure we aren't parsing more or less than we expect
235
self.assertEqual(builder.keys(), expected_parsing.keys())
237
for key, expected_value in expected_parsing.items():
238
self.assertEqual(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value)))
240
def test_builder_with_name(self):
241
buildbot = BuildBot()
243
builder = buildbot.builder_with_name("Test Builder")
244
self.assertEqual(builder.name(), "Test Builder")
245
self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder")
246
self.assertEqual(builder.url_encoded_name(), "Test%20Builder")
247
self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder")
249
# Override _fetch_build_dictionary function to not touch the network.
250
def mock_fetch_build_dictionary(self, build_number):
253
"revision" : 2 * build_number,
255
"number" : int(build_number),
256
"results" : build_number % 2, # 0 means pass
258
return build_dictionary
259
buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
261
build = builder.build(10)
262
self.assertEqual(build.builder(), builder)
263
self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10")
264
self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29")
265
self.assertEqual(build.revision(), 20)
266
self.assertEqual(build.is_green(), True)
268
build = build.previous_build()
269
self.assertEqual(build.builder(), builder)
270
self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9")
271
self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29")
272
self.assertEqual(build.revision(), 18)
273
self.assertEqual(build.is_green(), False)
275
self.assertEqual(builder.build(None), None)
277
_example_directory_listing = '''
278
<h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1>
284
<th>Content type</th>
285
<th>Content encoding</th>
287
<tr class="directory ">
288
<td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td>
290
<td><b>[Directory]</b></td>
293
<tr class="file alt">
294
<td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td>
296
<td>[application/zip]</td>
302
"filename" : "r47483 (1)/",
304
"type" : "[Directory]",
308
"filename" : "r47484 (2).zip",
310
"type" : "[application/zip]",
315
def test_parse_build_to_revision_map(self):
316
buildbot = BuildBot()
317
files = buildbot._parse_twisted_directory_listing(self._example_directory_listing)
318
self.assertEqual(self._expected_files, files)
320
_fake_builder_page = '''
322
<div class="content">
323
<h1>Some Builder</h1>
324
<p>(<a href="../waterfall?show=Some Builder">view in waterfall</a>)</p>
326
<h2>Recent Builds:</h2>
331
<th>Result</th> <th>Build #</th>
335
<td>Jan 10 15:49</td>
336
<td><span class="revision" title="Revision 104643"><a href="http://trac.webkit.org/changeset/104643">104643</a></span></td>
337
<td class="success">failure</td> <td><a href=".../37604">#37604</a></td>
338
<td class="left">Build successful</td>
341
<td>Jan 10 15:32</td>
342
<td><span class="revision" title="Revision 104636"><a href="http://trac.webkit.org/changeset/104636">104636</a></span></td>
343
<td class="success">failure</td> <td><a href=".../37603">#37603</a></td>
344
<td class="left">Build successful</td>
347
<td>Jan 10 15:18</td>
348
<td><span class="revision" title="Revision 104635"><a href="http://trac.webkit.org/changeset/104635">104635</a></span></td>
349
<td class="success">success</td> <td><a href=".../37602">#37602</a></td>
350
<td class="left">Build successful</td>
353
<td>Jan 10 14:51</td>
354
<td><span class="revision" title="Revision 104633"><a href="http://trac.webkit.org/changeset/104633">104633</a></span></td>
355
<td class="failure">failure</td> <td><a href=".../37601">#37601</a></td>
356
<td class="left">Failed compile-webkit</td>
360
_fake_builder_page_without_success = '''
364
<td>Jan 10 15:49</td>
365
<td><span class="revision" title="Revision 104643"><a href="http://trac.webkit.org/changeset/104643">104643</a></span></td>
366
<td class="success">failure</td>
369
<td>Jan 10 15:32</td>
370
<td><span class="revision" title="Revision 104636"><a href="http://trac.webkit.org/changeset/104636">104636</a></span></td>
371
<td class="success">failure</td>
374
<td>Jan 10 15:18</td>
375
<td><span class="revision" title="Revision 104635"><a href="http://trac.webkit.org/changeset/104635">104635</a></span></td>
376
<td class="success">failure</td>
379
<td>Jan 10 11:58</td>
380
<td><span class="revision" title="Revision ??"><a href="http://trac.webkit.org/changeset/%3F%3F">??</a></span></td>
381
<td class="retry">retry</td>
384
<td>Jan 10 14:51</td>
385
<td><span class="revision" title="Revision 104633"><a href="http://trac.webkit.org/changeset/104633">104633</a></span></td>
386
<td class="failure">failure</td>
391
def test_revisions_for_builder(self):
392
buildbot = BuildBot()
393
buildbot._fetch_builder_page = lambda builder: builder.page
394
builder_with_success = Builder('Some builder', None)
395
builder_with_success.page = self._fake_builder_page
396
self.assertEqual(buildbot._revisions_for_builder(builder_with_success), [(104643, False), (104636, False), (104635, True), (104633, False)])
398
builder_without_success = Builder('Some builder', None)
399
builder_without_success.page = self._fake_builder_page_without_success
400
self.assertEqual(buildbot._revisions_for_builder(builder_without_success), [(104643, False), (104636, False), (104635, False), (104633, False)])
402
def test_find_green_revision(self):
403
buildbot = BuildBot()
404
self.assertEqual(buildbot._find_green_revision({
405
'Builder 1': [(1, True), (3, True)],
406
'Builder 2': [(1, True), (3, False)],
407
'Builder 3': [(1, True), (3, True)],
409
self.assertEqual(buildbot._find_green_revision({
410
'Builder 1': [(1, False), (3, True)],
411
'Builder 2': [(1, True), (3, True)],
412
'Builder 3': [(1, True), (3, True)],
414
self.assertEqual(buildbot._find_green_revision({
415
'Builder 1': [(1, True), (2, True)],
416
'Builder 2': [(1, False), (2, True), (3, True)],
417
'Builder 3': [(1, True), (3, True)],
419
self.assertEqual(buildbot._find_green_revision({
420
'Builder 1': [(1, True), (2, True)],
421
'Builder 2': [(1, True), (2, True), (3, True)],
422
'Builder 3': [(1, True), (3, True)],
424
self.assertEqual(buildbot._find_green_revision({
425
'Builder 1': [(1, False), (2, True)],
426
'Builder 2': [(1, True), (3, True)],
427
'Builder 3': [(1, True), (3, True)],
429
self.assertEqual(buildbot._find_green_revision({
430
'Builder 1': [(1, True), (3, True)],
431
'Builder 2': [(1, False), (2, True), (3, True), (4, True)],
432
'Builder 3': [(2, True), (4, True)],
434
self.assertEqual(buildbot._find_green_revision({
435
'Builder 1': [(1, True), (3, True)],
436
'Builder 2': [(1, False), (2, True), (3, True), (4, False)],
437
'Builder 3': [(2, True), (4, True)],
439
self.assertEqual(buildbot._find_green_revision({
440
'Builder 1': [(1, True), (3, True)],
441
'Builder 2': [(1, False), (2, True), (3, True), (4, False)],
442
'Builder 3': [(2, True), (3, True), (4, True)],
444
self.assertEqual(buildbot._find_green_revision({
445
'Builder 1': [(1, True), (2, True)],
447
'Builder 3': [(1, True), (2, True)],
449
self.assertEqual(buildbot._find_green_revision({
450
'Builder 1': [(1, True), (3, False), (5, True), (10, True), (12, False)],
451
'Builder 2': [(1, True), (3, False), (7, True), (9, True), (12, False)],
452
'Builder 3': [(1, True), (3, True), (7, True), (11, False), (12, True)],
455
def _fetch_build(self, build_number):
456
if build_number == 5:
457
return "correct build"
460
def _fetch_revision_to_build_map(self):
461
return {'r5': 5, 'r2': 2, 'r3': 3}
463
def test_latest_cached_build(self):
464
b = Builder('builder', BuildBot())
465
b._fetch_build = self._fetch_build
466
b._fetch_revision_to_build_map = self._fetch_revision_to_build_map
467
self.assertEqual("correct build", b.latest_cached_build())
469
def results_url(self):
472
def test_results_zip_url(self):
473
b = Build(None, 123, 123, False)
474
b.results_url = self.results_url
475
self.assertEqual("some-url.zip", b.results_zip_url())
478
if __name__ == '__main__':