2
# This file is part of Checkbox.
4
# Copyright 2015 Canonical Ltd.
6
# Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
8
# Checkbox is free software: you can redistribute it and/or modify
9
# it under the terms of the GNU General Public License version 3,
10
# as published by the Free Software Foundation.
12
# Checkbox is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
17
# You should have received a copy of the GNU General Public License
18
# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.
20
Tool for running piglit tests.
22
This script is designed to wrap ``piglit run`` to provide consistent outcome
23
and testing experience for plainbox.
26
This script was written to work with ``piglit-0~git20150312-530724b-1`` and
27
may not work with older or more recent version.
41
_logger = logging.getLogger("piglit-wrapper")
46
""" Wrapper around the piglit tool. """
48
OUTCOME_CRASH = 'crash'
53
def __init__(self, tests):
55
Initialize the wrapper around piglit.
58
List of patterns to match tests against.
59
Only tests matching any of the patterns will be started.
62
self._is_supported = None # Optional[bool]
63
self._outcome_stats = collections.defaultdict(int) # Dict[str, int]
64
# Dict[str, List[str]]
65
self._outcome_list = collections.defaultdict(list)
66
self._test_count = None # Optional[int]
69
def is_supported(self):
70
""" Flag indicating if this version if piglit is supported. """
71
return bool(self._is_supported)
74
def outcome_stats(self):
75
""" Mapping from test outcome to number of occurrences. """
76
return self._outcome_stats
79
def outcome_list(self):
80
""" Mapping from test outcome to a list of tests. """
81
return self._outcome_list
85
""" Total number of tests. """
86
return self._test_count
89
def is_successful(self):
90
""" Flag indicating if the run was successful. """
91
return (self._outcome_stats[self.OUTCOME_FAIL] == 0 and
92
self._outcome_stats[self.OUTCOME_CRASH] == 0)
94
def run(self, dirname=None):
95
""" Run piglit and all the desired tests. """
96
if dirname is not None:
97
self._run_in_directory(dirname)
99
with tempfile.TemporaryDirectory() as tmpdir:
100
_logger.info(_("Created temporary directory: %s"), tmpdir)
101
self._run_in_directory(tmpdir)
103
def _run_in_directory(self, dirname):
104
""" Run piglit and all the desired tests in a specific directory. """
108
# Using the json backend that we understand
110
for test in self._tests:
111
# Include tests that we've been asked to run.
112
cmd.extend(["-t", test])
113
# Out of all the tests in general.
115
# Save results to a hard-coded file in this directory
117
_logger.info(_("Starting program: %r"), cmd)
119
# redirect stdout to /dev/null as we don't care about the
120
# spinner that piglit prints
121
cmd, stdout=subprocess.DEVNULL)
122
# NOTE: the "results.json" filename is hard-coded into piglit
123
result_filename = os.path.join(dirname, "results.json")
124
self._analyze_results(result_filename)
126
def _analyze_results(self, result_filename):
127
""" Analyze raw piglit json data. """
128
if not os.path.isfile(result_filename):
129
self._is_supported = False
130
_logger.errr(_("Piglit didn't create the test result file?"))
132
_logger.info(_("Analyzing piglit test results from %s"),
134
with open(result_filename, 'rt', encoding='UTF-8') as stream:
135
result_json = json.load(stream)
136
version = result_json.get('results_version')
138
self._is_supported = True
139
self._analyze_v4(result_json)
141
self._is_supported = False
142
_logger.errr(_("Unsupported piglit result format (%r)"), version)
144
def _analyze_v4(self, result_json):
145
""" Analyze raw piglit json data (format 4). """
146
_logger.info(_("Analyzing piglit test results (format 4)"))
147
self._test_count = len(result_json['tests'])
148
for test_id, test_result in result_json['tests'].items():
149
outcome = test_result['result']
150
self._outcome_stats[outcome] += 1
151
self._outcome_list[outcome].append(test_id)
155
""" Main function. """
156
gettext.textdomain('plainbox-provider-piglit')
157
gettext.bindtextdomain('plainbox-provider-piglit',
158
os.getenv('PLAINBOX_PROVIDER_LOCALE_DIR'))
159
parser = argparse.ArgumentParser(
160
description=_("Tool for running piglit tests"))
162
'-d', '--dirname', metavar=_("DIR"), default=None,
163
help=_("save piglit results to DIR"))
165
"--test", "-t", metavar=_("PATTERN"), required=True, action='append',
166
help=_("run piglit tests matching given PATTERN"))
170
help=_("be more verbose during testing"))
171
ns = parser.parse_args()
173
level=logging.INFO if ns.verbose else logging.WARNING,
174
format="{name}:{levelname}: {message}", style='{')
175
piglit = PiglitWrapper(ns.test)
176
piglit.run(ns.dirname)
177
if not piglit.is_supported:
178
print(_("This version of piglit is not supported"))
180
stats = piglit.outcome_stats
181
print(_("Summary of results (by outcome)"))
182
for outcome in sorted(stats.keys()):
183
print(" - {}: {}".format(outcome, stats[outcome]))
185
for test_id in sorted(piglit.outcome_list[outcome]):
186
print(" * {}".format(test_id))
187
if piglit.is_successful:
188
print(_("Tests successful"))
191
print(_("Tests unsuccessful"))
195
if __name__ == "__main__":
added
removed
providers/plainbox-provider-piglit/bin/piglit-wrapper
