2
# -*- mode: c; c-basic-offset: 2; indent-tabs-mode: nil; -*-
3
# vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
5
# Copyright (C) 2010 Patrick Crews
7
## This program is free software; you can redistribute it and/or modify
8
# it under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 2 of the License, or
10
# (at your option) any later version.
12
# This program is distributed in the hope that it will be useful,
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
17
# You should have received a copy of the GNU General Public License
18
# along with this program; if not, write to the Free Software
19
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21
""" dtr_test_management:
22
code related to the gathering / analysis / management of
24
ie - collecting the list of tests in each suite, then
25
gathering additional, relevant information for the test-runner's dtr
26
mode. (traditional diff-based testing)
35
import lib.test_mgmt.test_management as test_management
40
"""Holds info on a per .test file basis
41
Attributes contain information necessary to execute / validate
42
the test file when it is executed.
45
def __init__(self, system_manager, test_case=None, test_name=None, suite_name=None
46
, suite_path=None, test_server_options=[], test_path=None, result_path=None
47
, comment=None, master_sh=None
48
, disable=0, innodb_test=1
49
, need_debug=0, debug=0):
50
self.system_manager = system_manager
51
self.logging = self.system_manager.logging
52
self.skip_keys = ['system_manager']
53
self.testcase = test_case
54
self.testname = test_name
55
self.suitename = suite_name
56
self.suitepath = suite_path
57
self.fullname = "%s.%s" %(suite_name, test_name)
58
self.testpath = test_path
59
self.resultpath = result_path
62
self.timezone = "GMT-3"
63
self.component_id = "drizzled"
66
self.server_options = test_server_options
67
self.comment = comment
68
self.master_sh = master_sh
69
self.disable = disable
70
self.innodb_test = innodb_test
71
self.need_debug = need_debug
73
self.system_manager.logging.debug_class(self)
76
if self.skip_flag or self.disable:
85
class testManager(test_management.testManager):
86
"""Deals with scanning test directories, gathering test cases, and
87
collecting per-test information (opt files, etc) for use by the
92
def process_suite(self,suite_dir):
93
"""Process a test suite.
94
This includes searching for tests in test_list and only
95
working with the named tests (all tests in suite is the default)
96
Further processing includes reading the disabled.def file
97
to know which tests to skip, processing the suite.opt file,
98
and processing the individual test cases for data relevant
99
to the rest of the test-runner
104
self.system_manager.logging.verbose("Processing suite: %s" %(suite_dir))
106
# Generate our test and result files:
107
testdir = os.path.join(suite_dir, 't')
108
resultdir = os.path.join(suite_dir, 'r')
110
# Do basic checks to make sure this is worth further work
111
self.check_suite(suite_dir, testdir, resultdir)
113
# Get suite-level options
115
suite_options = self.process_suite_options(suite_dir)
117
# Get the 'name' of the suite. This can require some processing
118
# But the name is useful for reporting and whatnot
119
suite_name = self.get_suite_name(suite_dir)
121
# Get the contents of the testdir and filter it accordingly
122
# This applies do-test / skip-test filters and any specific
124
testlist = self.testlist_filter(os.listdir(testdir))
125
# sort our list if no individual tests were specified
126
if not self.desired_tests:
129
# gather deeper information on tests we are interested in
131
# We have tests we want to process, we gather additional information
132
# that is useful at the suite level. This includes disabled tests
133
# Gather disabled test list.
134
# This is used in process_test_file()
136
disabled_tests = self.process_disabled_test_file(testdir)
137
for test_case in testlist:
138
self.add_test(self.process_test_file(suite_dir,
139
suite_name, suite_options
140
, disabled_tests, testdir
141
, resultdir, test_case))
143
def process_test_file(self, suite_dir, suite_name, suite_options
144
, disabled_tests, testdir
145
, resultdir, test_case):
146
""" We generate / find / store all the relevant information per-test.
147
This information will be used when actually executing the test
148
We store the data in a testCase object
152
test_server_options = suite_options
153
test_name = test_case.replace('.test','')
155
self.system_manager.logging.verbose("Processing test: %s.%s" %(suite_name,test_name))
158
# Fix this , create a testCase with gather_test_data() passed
160
# Ensure we pass everything we need and use it all
166
, test_server_options
169
, need_debug) = self.gather_test_data(test_case, test_name,
170
suite_name, test_server_options,testdir,
171
resultdir, disabled_tests)
172
test_case = testCase(self.system_manager, test_case, test_name, suite_name,
173
suite_dir, test_server_options,test_path, result_path,
174
master_sh=master_sh, debug=self.debug)
178
########################################################################
181
# Stuff that helps us out and simplifies our main functions
182
# But isn't that important unless you need to dig deep
183
########################################################################
185
def gather_test_data(self, test_case, test_name, suite_name,
186
test_server_options, testdir, resultdir, disabled_tests):
187
""" We gather all of the data needed to produce a testCase for
192
(self, test_case=None, test_name=None, suite_name=None
193
, test_server_options=[], test_path=None, result_path=None
194
, comment=None, master_sh=None, disable=0, innodb_test=0
195
, need_debug=0, debug=0):
197
test_path = os.path.join(testdir,test_case)
198
result_file_name = test_name+'.result'
199
result_path = self.find_result_path(resultdir, result_file_name)
201
master_sh_path = test_path.replace('.test','-master.sh')
202
if os.path.exists(master_sh_path):
203
master_sh = master_sh_path
206
master_opt_path = test_path.replace('.test','-master.opt')
207
test_server_options = test_server_options + self.process_opt_file(
209
(disable, comment) = self.check_if_disabled(disabled_tests, test_name)
212
return (test_path, result_file_name, result_path, comment, master_sh,
213
test_server_options, disable, innodb_test, need_debug)
215
def check_suite(self, suite_dir, testdir, resultdir):
216
"""Handle basic checks of the suite:
217
does the suite exist?
223
# We expect suite to be a path name, no fuzzy searching
224
if not os.path.exists(suite_dir):
225
self.system_manager.logging.error("Suite: %s does not exist" %(suite_dir))
228
# Ensure our test and result directories are present
229
if not os.path.exists(testdir):
230
self.system_manager.logging.error("Suite: %s does not have a 't' directory (expected location for test files)" %(suite_dir))
232
if not os.path.exists(resultdir):
233
self.system_manager.logging.error("Suite: %s does not have an 'r' directory (expected location for result files)" %(suite_dir))
236
def get_suite_name(self, suite_dir):
237
""" Get the 'name' of the suite
238
This can either be the path basename or one directory up, as
239
in the case of files in the drizzle/plugins directory
243
# We trim any trailing path delimiters as python returns
244
# '' for basedir if the path ends that way
245
# BEGIN horrible hack to accomodate bad location of main suite : /
246
if suite_dir == self.testdir:
248
# END horrible hack : /
249
if suite_dir.endswith('/'):
250
suite_dir=suite_dir[:-1]
251
suite_dir_root,suite_dir_basename = os.path.split(suite_dir)
252
if suite_dir_basename == 'tests' or suite_dir_basename == 'drizzle-tests':
253
suite_name = os.path.basename(suite_dir_root)
255
suite_name = suite_dir_basename
257
self.system_manager.logging.debug("Suite_name: %s" %(suite_name))
260
def process_suite_options(self, suite_dir):
261
""" Process the suite.opt and master.opt files
262
that reside at the suite-level if they exist.
263
Return a list of the options found
267
opt_files = ['t/master.opt','t/suite.opt']
268
for opt_file in opt_files:
269
found_options = found_options + self.process_opt_file(os.path.join(suite_dir,opt_file))
272
def process_disabled_test_file(self, testdir):
273
""" Checks and processes the suite's disabled.def
274
file. This file must reside in the suite/t directory
275
It must be in the format:
276
test-name : comment (eg BugNNNN - bug on hold, test disabled)
277
In reality a test should *never* be disabled. EVER.
278
However, we keep this as a bit of utility
282
disabled_def_path = os.path.join(testdir,'disabled.def')
283
if not os.path.exists(disabled_def_path):
284
return disabled_tests
287
disabled_test_file = open(disabled_def_path,'r')
289
self.system_manager.logging.error("Problem opening disabled.def file: %s" %(disabled_def_path))
293
self.system_manager.logging.debug("Processing disabled.def file: %s" %(disabled_def_path))
294
disabled_bug_pattern = re.compile("[\S]+[\s]+:[\s]+[\S]")
296
for line in disabled_test_file:
298
if not line.startswith('#'): # comment
299
if re.match(disabled_test_pattern,line):
301
self.system_manager.logging.debug("found disabled test - %s" %(line))
302
test_name, test_comment = line.split(':')
303
disabled_tests[test_name.strip()]=test_comment.strip()
305
disabled_test_file.close()
306
return disabled_tests
309
def process_opt_file(self, opt_file_path):
310
""" Process a test-run '.opt' file.
311
These files contain test and suite-specific server options
312
(ie what options the server needs to use for the test)
314
Returns a list of the options (we don't really validate...yet)
316
NOTE: test-run.pl allows for server *and* system options
317
(eg timezone, slave_count, etc) in opt files. We don't.
318
None of our tests use this and we should probably avoid it
319
We can introduce server and system .opt files or even better
320
would be to use config files as we do with drizzle-automation
321
This would allow us to specify options for several different
322
things in a single file, but in a clean and standardized manner
326
if not os.path.exists(opt_file_path):
330
opt_file = open(opt_file_path,'r')
332
self.system_manager.logging.error("Problem opening option file: %s" %(opt_file_path))
336
self.system_manager.logging.debug("Processing opt file: %s" %(opt_file_path))
337
for line in opt_file:
338
options = line.split('--')
340
for option in options:
342
if 'restart' in option or '#' in option:
344
found_options.append('--%s' %(option.strip()))
348
def testlist_filter(self, testlist):
349
""" Filter our list of testdir contents based on several
350
criteria. This looks for user-specified test-cases
351
and applies the do-test and skip-test filters
353
Returns the list of tests that we want to execute
354
for further processing
358
# We want only .test files
359
# Possible TODO: allow alternate test extensions
360
testlist = [test_file for test_file in testlist if test_file.endswith('.test')]
362
# Search for specific test names
363
if self.desired_tests: # We have specific, named tests we want from the suite(s)
365
for test in self.desired_tests:
366
if test.endswith('.test'):
371
tests_to_use.append(test)
372
testlist = tests_to_use
374
# TODO: Allow for regex?
375
# Apply do-test filter
377
testlist = [test_file for test_file in testlist if test_file.startswith(self.dotest)]
378
# Apply skip-test filter
380
testlist = [test_file for test_file in testlist if not test_file.startswith(self.skiptest)]
383
def find_result_path(self, result_dir, result_file_name):
384
""" This is copied from test-run.pl dtr_cases.pl
385
If we have an engine option passed in and the
386
path resultdir/engine/testname.result exists, that is
389
Need to check if we really need this - maybe PBXT?
392
result_path = os.path.join(result_dir,result_file_name)
393
if self.default_engine:
394
candidate_path = os.path.join(result_dir, self.default_engine,
396
if os.path.exists(candidate_path):
397
result_path = candidate_path
400
def check_if_disabled(self, disabled_tests, test_name):
401
""" Scan the list of disabled tests if it exists to see
402
if the test is disabled.
407
if test_name in disabled_tests:
409
self.system_manager.logging.debug("%s says - I'm disabled" %(test_name))
410
return (1, disabled_tests[test_name])
413
def sort_testcases(self):
414
""" We sort our testcases according to the server_options they have
415
For each testcase, we sort the list of options, so if a test has
416
--plugin-add=csv --abracadabra, we would get
417
--abracadabra --plugin-add=csv
419
This results in tests that have similar options being run in order
420
this minimizes server restarts which can be costly
423
test_management.testManager.sort_testcases(self)
426
for testcase in self.test_list:
427
key = " ".join(sorted(testcase.server_options))
429
organizer[key].append(testcase)
431
organizer[key] = [testcase]
432
for value_list in organizer.values():
433
ordered_list = ordered_list + value_list
434
self.test_list = ordered_list
added
removed
tests/lib/drizzle_test_run/dtr_test_management.py
