3
# Quick'n'dirty regression check for dejagnu testsuites
4
# Copyright (C) 2003, 2004, 2005, 2006, 2007 James Troup <james@nocrew.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU;5B General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
################################################################################
26
################################################################################
28
def fubar(msg, exit_code=1):
29
sys.stderr.write("E: %s\n" % (msg))
33
sys.stderr.write("W: %s\n" % (msg))
36
sys.stderr.write("I: %s\n" % (msg))
38
################################################################################
40
def read_testsummary(filename):
43
for line in file.readlines():
46
if line.startswith("Running"):
50
if x.find("/testsuite/") == -1:
51
fubar("Can't find /testsuite/ in '%s'." % (x))
52
# 'Running /home/james/debian/packages/binutils/binutils-2.14.90.0.7/gas/testsuite/gas/hppa/unsorted/unsorted.exp ...' -> 'gas/hppa/unsorted/unsorted.exp'
53
# ... since using basename() isn't dupe safe.
54
section = x[x.find("/testsuite/"):].replace("/testsuite/","").split()[0]
56
# Tests can be duplicated, e.g. hppa/basic/basic.exp
57
# is run twice, once for hppa-linux and once for
58
# hppa64-linux. This is of course a horrible bodge,
59
# but I can't think of anything trivial and better off
62
if section in results:
65
while section in results and extra < too_many:
66
section = "%s.%s" % (section, extra)
69
fubar("gave up trying to unduplicate %s." % (section))
75
for state in [ "PASS", "XPASS", "FAIL", "XFAIL", "UNRESOLVED",
76
"UNTESTED", "UNSUPPORTED" ]:
77
if line.startswith(state):
80
test = ':'.join(s[1:]).strip()
82
warn("%s/%s is duplicated." % (section, test))
83
results[section][test] = state
92
################################################################################
94
def compare_results(old, new):
101
progression_count = 0
104
for section in list(new.keys()):
105
for test in list(new[section].keys()):
106
state = new[section][test]
110
if state == "PASS" or state == "XPASS":
112
elif state == "FAIL" or state == "UNRESOLVED":
114
elif state == "XFAIL":
116
elif state == "UNTESTED":
120
if section not in old:
122
if test not in old[section]:
124
old_state = old[section][test]
126
if old_state != "PASS":
127
progression_count += 1
128
info("[%s] progression (%s -> %s): %s" % (section, old_state, state, test))
129
elif state == "XPASS":
130
if old_state != "XPASS" and old_state != "PASS":
131
progression_count += 1
132
warn("[%s] %s: %s" % (section, state, test))
133
elif state == "FAIL":
134
if old_state != "FAIL":
135
regression_count += 1
136
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
137
elif state == "XFAIL":
138
if old_state != "XFAIL":
140
info("[%s] change (%s -> %s): %s" % (section, old_state, state, test))
141
elif state == "UNRESOLVED":
142
if old_state != "UNRESOLVED" and old_state != "FAIL":
143
regression_count += 1
144
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
145
if old_state == "FAIL":
147
info("[%s] change (%s -> %s): %s" % (section, old_state, state, test))
148
elif state == "UNTESTED":
149
if old_state != "UNTESTED":
151
warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test))
154
print("%d REGRESSIONS (%.2f%%)." % (regression_count, (float(regression_count)/total_num)*100))
155
if progression_count:
156
print("%d progressions (%.2f%%)." % (progression_count, (float(progression_count)/total_num)*100))
159
print("%d changes (%.2f%%)." % (change_count, (float(change_count)/total_num)*100))
161
print("%d tests: %d pass (%.2f%%), %d fail (%.2f%%), %d xfail (%.2f%%) %d untested (%.2f%%)." \
162
% (total_num, pass_count, (float(pass_count)/total_num)*100,
163
fail_count, (float(fail_count)/total_num)*100,
164
xfail_count, (float(xfail_count)/total_num)*100,
165
untested_count, (float(untested_count)/total_num)*100))
170
################################################################################
172
def compare_multiple(directory, first_version, second_version):
173
architectures = [ "alpha", "arm", "hppa", "i386", "ia64", "mips",
174
"m68k", "mipsel", "powerpc", "s390", "sparc" ]
176
for arch in architectures:
177
print("*********************************** %s ******************************" % (arch))
178
second_filename = "%s/%s_%s" % (directory, second_version, arch)
179
if not os.path.exists(second_filename):
180
print(" -- NOT AVAILABLE --")
183
new = read_testsummary(second_filename)
184
first_filename = "%s/%s_%s" % (directory, first_version, arch)
185
old = read_testsummary(first_filename)
186
compare_results(old, new)
188
################################################################################
191
"""Initalization, including parsing of options."""
193
usage = """usage: %prog [OPTIONS] <OLD> <NEW>
194
compare (binutils) dejagnu testsuite results.
198
test-suite-compare.py binutils-2.17/test-summary binutils-2.18/test-summary
200
Or to compare across all architectures (with test results stored in a
201
'test-summary' directory):
203
test-suite-compare.py -mtest-summary 2.17-3 2.18-1"""
204
parser = optparse.OptionParser(usage)
205
parser.add_option("-m", "--multiple", dest="multiple",
206
nargs=1, type="string",
207
help="compare multiple architectures")
208
(options, args) = parser.parse_args()
210
if len(args) > 2 or len(args) < 2:
211
parser.error("takes 2 arguments (old and new)")
212
(old_version, new_version) = args
214
return options, old_version, new_version
216
################################################################################
219
(options, old_version, new_version) = init()
221
compare_multiple(options.multiple, old_version, new_version)
223
old = read_testsummary(old_version)
224
new = read_testsummary(new_version)
225
compare_results(old, new)
227
################################################################################
229
if __name__ == '__main__':
added
removed
debian/test-suite-compare.py
