6689.1.1
by Holger Rapp
First unfinished version of a regression test suite. |
1 |
#!/usr/bin/env python
|
2 |
# encoding: utf-8
|
|
3 |
||
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
4 |
from glob import glob |
5 |
import argparse |
|
6 |
import os |
|
7 |
import re |
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
8 |
import shutil |
9 |
import subprocess |
|
6803
by Holger Rapp
regression_test.py returns an error to the system on failure. |
10 |
import sys |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
11 |
import tempfile |
6689.1.2
by Holger Rapp
Brought back original content of regression_test.py. |
12 |
import unittest |
6849
by Tino
allow regression test on windows |
13 |
import platform |
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
14 |
|
7945.1.1
by Tino
allow regression tests to be run with python 3 |
15 |
#Python2/3 compat code for iterating items
|
16 |
try: |
|
17 |
dict.iteritems |
|
18 |
except AttributeError: |
|
19 |
# Python 3
|
|
20 |
def itervalues(d): |
|
21 |
return iter(d.values()) |
|
22 |
def iteritems(d): |
|
23 |
return iter(d.items()) |
|
24 |
else: |
|
25 |
# Python 2
|
|
26 |
def itervalues(d): |
|
27 |
return d.itervalues() |
|
28 |
def iteritems(d): |
|
29 |
return d.iteritems() |
|
7945.1.2
by Tino
use logfile output on all platforms |
30 |
|
7293.1.52
by Holger Rapp
Adds a --datadir_for_testing flag which works as an extra --datadir flag. That |
31 |
def datadir(): |
32 |
return os.path.join(os.path.dirname(__file__), "data") |
|
33 |
||
34 |
def datadir_for_testing(): |
|
7782.3.1
by fios at foramnagaidhlig
Fixed datadir_for_testing in regression_test.py. |
35 |
return os.path.relpath(".", os.path.dirname(__file__)) |
7293.1.52
by Holger Rapp
Adds a --datadir_for_testing flag which works as an extra --datadir flag. That |
36 |
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
37 |
def out(string): |
38 |
sys.stdout.write(string) |
|
39 |
sys.stdout.flush() |
|
40 |
||
41 |
class WidelandsTestCase(unittest.TestCase): |
|
42 |
do_use_random_directory = True |
|
43 |
path_to_widelands_binary = None |
|
44 |
keep_output_around = False |
|
45 |
||
46 |
def __init__(self, test_script, **wlargs): |
|
47 |
unittest.TestCase.__init__(self) |
|
48 |
self._test_script = test_script |
|
49 |
self._wlargs = wlargs |
|
50 |
||
51 |
def __str__(self): |
|
52 |
return self._test_script |
|
53 |
||
54 |
def setUp(self): |
|
55 |
if self.do_use_random_directory: |
|
56 |
self.run_dir = tempfile.mkdtemp(prefix="widelands_regression_test") |
|
57 |
else: |
|
58 |
self.run_dir = os.path.join(tempfile.gettempdir(), "widelands_regression_test", self.__class__.__name__) |
|
59 |
if os.path.exists(self.run_dir): |
|
60 |
if not self.keep_output_around: |
|
61 |
shutil.rmtree(self.run_dir) |
|
6812.1.18
by Holger Rapp
No more multiple inheritance in scripting layer. All clearer now. |
62 |
os.makedirs(self.run_dir) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
63 |
else: |
64 |
os.makedirs(self.run_dir) |
|
65 |
self.widelands_returncode = 0 |
|
66 |
||
67 |
def run(self, result=None): |
|
68 |
self.currentResult = result # remember result for use in tearDown |
|
69 |
unittest.TestCase.run(self, result) |
|
70 |
||
71 |
def tearDown(self): |
|
72 |
if self.currentResult.wasSuccessful() and not self.keep_output_around: |
|
73 |
shutil.rmtree(self.run_dir) |
|
74 |
||
6820.2.10
by Holger Rapp
Fixed a bug in the test runner and added a new test case. |
75 |
def run_widelands(self, wlargs, which_time): |
76 |
"""Run Widelands with the given 'wlargs'. 'which_time' is an integer
|
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
77 |
defining the number of times Widelands has been run this test case
|
78 |
(i.e. because we might load a saved game from an earlier run. This will
|
|
79 |
impact the filenames for stdout.txt.
|
|
80 |
||
81 |
Returns the stdout filename."""
|
|
7945.1.2
by Tino
use logfile output on all platforms |
82 |
stdout_filename = os.path.join(self.run_dir, "stdout_{:02d}.txt".format(which_time)) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
83 |
if (os.path.exists(stdout_filename)): |
84 |
os.unlink(stdout_filename) |
|
85 |
||
86 |
with open(stdout_filename, 'a') as stdout_file: |
|
7293.1.52
by Holger Rapp
Adds a --datadir_for_testing flag which works as an extra --datadir flag. That |
87 |
args = [self.path_to_widelands_binary, |
88 |
'--verbose=true', |
|
7945.1.2
by Tino
use logfile output on all platforms |
89 |
'--datadir={}'.format(datadir()), |
90 |
'--datadir_for_testing={}'.format(datadir_for_testing()), |
|
91 |
'--homedir={}'.format(self.run_dir), |
|
7293.1.52
by Holger Rapp
Adds a --datadir_for_testing flag which works as an extra --datadir flag. That |
92 |
'--disable_fx=true', |
93 |
'--disable_music=true', |
|
8063.1.3
by Holger Rapp
No longer use --logfile in test runner. |
94 |
'--language=en_US' ] |
7945.1.2
by Tino
use logfile output on all platforms |
95 |
args += [ "--{}={}".format(key, value) for key, value in iteritems(wlargs) ] |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
96 |
|
97 |
widelands = subprocess.Popen( |
|
98 |
args, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
|
99 |
while 1: |
|
100 |
line = widelands.stdout.readline() |
|
101 |
if not line: |
|
102 |
break
|
|
7945.1.2
by Tino
use logfile output on all platforms |
103 |
stdout_file.write(str(line)) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
104 |
stdout_file.flush() |
105 |
widelands.communicate() |
|
106 |
self.widelands_returncode = widelands.returncode |
|
107 |
return stdout_filename |
|
108 |
||
109 |
def runTest(self): |
|
110 |
out("\n Running Widelands ... ") |
|
111 |
stdout_filename = self.run_widelands(self._wlargs, 0) |
|
112 |
stdout = open(stdout_filename, "r").read() |
|
113 |
self.verify_success(stdout, stdout_filename) |
|
114 |
||
115 |
find_saves = lambda stdout: re.findall("Script requests save to: (\w+)$", stdout, re.M) |
|
116 |
savegame_done = { fn: False for fn in find_saves(stdout) } |
|
117 |
which_time = 1 |
|
118 |
while not all(savegame_done.values()): |
|
6820.2.11
by Holger Rapp
Added two more test that remove the ship when a ware should be transported. |
119 |
for savegame in sorted(savegame_done): |
120 |
if not savegame_done[savegame]: break |
|
7945.1.2
by Tino
use logfile output on all platforms |
121 |
out(" Loading savegame: {} ... ".format(savegame)) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
122 |
stdout_filename = self.run_widelands({ "loadgame": os.path.join( |
7945.1.2
by Tino
use logfile output on all platforms |
123 |
self.run_dir, "save", "{}.wgf".format(savegame))}, which_time) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
124 |
which_time += 1 |
125 |
stdout = open(stdout_filename, "r").read() |
|
126 |
for new_save in find_saves(stdout): |
|
127 |
if new_save not in savegame_done: |
|
128 |
savegame_done[new_save] = False |
|
129 |
savegame_done[savegame] = True |
|
130 |
self.verify_success(stdout, stdout_filename) |
|
131 |
||
132 |
def verify_success(self, stdout, stdout_filename): |
|
7945.1.2
by Tino
use logfile output on all platforms |
133 |
common_msg = "Analyze the files in {} to see why this test case failed. Stdout is\n {}\n\nstdout:\n{}".format( |
7707.1.1
by Holger Rapp
Make travis stricter, run regression tests on travis, adjust warnings. |
134 |
self.run_dir, stdout_filename, stdout) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
135 |
self.assertTrue(self.widelands_returncode == 0, |
7945.1.2
by Tino
use logfile output on all platforms |
136 |
"Widelands exited abnormally. {}".format(common_msg) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
137 |
)
|
138 |
self.assertTrue("All Tests passed" in stdout, |
|
7945.1.2
by Tino
use logfile output on all platforms |
139 |
"Not all tests pass. {}.".format(common_msg) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
140 |
)
|
141 |
out("done.\n") |
|
142 |
if self.keep_output_around: |
|
7945.1.2
by Tino
use logfile output on all platforms |
143 |
out(" stdout: {}\n".format(stdout_filename)) |
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
144 |
|
145 |
def parse_args(): |
|
146 |
p = argparse.ArgumentParser(description= |
|
147 |
"Run the regression tests suite."
|
|
148 |
)
|
|
149 |
||
150 |
p.add_argument("-r", "--regexp", type=str, |
|
151 |
help = "Run only the tests from the files which filename matches." |
|
152 |
)
|
|
153 |
p.add_argument("-n", "--nonrandom", action="store_true", default = False, |
|
154 |
help = "Do not randomize the directories for the tests. This is useful " |
|
155 |
"if you want to run a test more often than once and not reopen stdout.txt "
|
|
156 |
"in your editor."
|
|
157 |
)
|
|
6801
by Holger Rapp
Fix more bugs around seafaring, especially related to persistence. |
158 |
p.add_argument("-k", "--keep-around", action="store_true", default = False, |
159 |
help = "Keep the output files around even when a test terminates successfully." |
|
160 |
)
|
|
6689.1.6
by Holger Rapp
Let the caller specify the widelands binary to use for regression testing. |
161 |
p.add_argument("-b", "--binary", type=str, |
162 |
help = "Run this binary as Widelands. Otherwise some default paths are searched." |
|
163 |
)
|
|
164 |
||
165 |
args = p.parse_args() |
|
166 |
||
167 |
if args.binary is None: |
|
168 |
potential_binaries = ( |
|
169 |
glob("widelands") + |
|
170 |
glob("src/widelands") + |
|
171 |
glob("../*/src/widelands") |
|
172 |
)
|
|
173 |
if not potential_binaries: |
|
174 |
p.error("No widelands binary found. Please specify with -b.") |
|
175 |
args.binary = potential_binaries[0] |
|
176 |
return args |
|
177 |
||
6689.1.2
by Holger Rapp
Brought back original content of regression_test.py. |
178 |
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
179 |
def discover_loadgame_tests(regexp, suite): |
180 |
"""Add all tests using --loadgame to the 'suite'."""
|
|
181 |
for fixture in glob(os.path.join("test", "save", "*")): |
|
182 |
if not os.path.isdir(fixture): |
|
183 |
continue
|
|
184 |
savegame = glob(os.path.join(fixture, "*.wgf"))[0] |
|
185 |
for test_script in glob(os.path.join(fixture, "test*.lua")): |
|
186 |
if regexp is not None and not re.search(regexp, test_script): |
|
187 |
continue
|
|
188 |
suite.addTest( |
|
189 |
WidelandsTestCase(test_script, |
|
190 |
loadgame=savegame, script=test_script)) |
|
191 |
||
192 |
def discover_scenario_tests(regexp, suite): |
|
193 |
"""Add all tests using --scenario to the 'suite'."""
|
|
194 |
for wlmap in glob(os.path.join("test", "maps", "*")): |
|
195 |
if not os.path.isdir(wlmap): |
|
196 |
continue
|
|
197 |
for test_script in glob(os.path.join(wlmap, "scripting", "test*.lua")): |
|
198 |
if regexp is not None and not re.search(regexp, test_script): |
|
199 |
continue
|
|
200 |
suite.addTest( |
|
201 |
WidelandsTestCase(test_script, |
|
202 |
scenario=wlmap, script=test_script)) |
|
203 |
||
204 |
def discover_editor_tests(regexp, suite): |
|
205 |
"""Add all tests needing --editor to the 'suite'."""
|
|
206 |
for wlmap in glob(os.path.join("test", "maps", "*")): |
|
207 |
if not os.path.isdir(wlmap): |
|
208 |
continue
|
|
209 |
for test_script in glob(os.path.join(wlmap, "scripting", "editor_test*.lua")): |
|
210 |
if regexp is not None and not re.search(regexp, test_script): |
|
211 |
continue
|
|
212 |
suite.addTest( |
|
213 |
WidelandsTestCase(test_script, |
|
214 |
editor=wlmap, script=test_script)) |
|
215 |
||
6689.1.2
by Holger Rapp
Brought back original content of regression_test.py. |
216 |
def main(): |
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
217 |
args = parse_args() |
218 |
||
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
219 |
WidelandsTestCase.path_to_widelands_binary = args.binary |
7945.1.1
by Tino
allow regression tests to be run with python 3 |
220 |
print("Using '{}' binary.".format(args.binary)) |
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
221 |
WidelandsTestCase.do_use_random_directory = not args.nonrandom |
222 |
WidelandsTestCase.keep_output_around = args.keep_around |
|
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
223 |
|
224 |
all_files = [os.path.basename(filename) for filename in glob("test/test_*.py") ] |
|
225 |
if args.regexp: |
|
226 |
all_files = [filename for filename in all_files if re.search(args.regexp, filename) ] |
|
227 |
||
7945.1.2
by Tino
use logfile output on all platforms |
228 |
all_modules = [ "test.{}".format(filename[:-3]) for filename in all_files ] |
6689.1.4
by Holger Rapp
All tests now run and pass on my system. |
229 |
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
230 |
suite = unittest.TestSuite() |
231 |
discover_loadgame_tests(args.regexp, suite) |
|
232 |
discover_scenario_tests(args.regexp, suite) |
|
233 |
discover_editor_tests(args.regexp, suite) |
|
6689.1.2
by Holger Rapp
Brought back original content of regression_test.py. |
234 |
|
6820.2.7
by Holger Rapp
Finally found a better way to define tests that do not need all the redundant python files. |
235 |
return unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() |
6689.1.2
by Holger Rapp
Brought back original content of regression_test.py. |
236 |
|
237 |
if __name__ == '__main__': |
|
6803
by Holger Rapp
regression_test.py returns an error to the system on failure. |
238 |
sys.exit(0 if main() else 1) |