« back to all changes in this revision
Viewing changes to util/simplify-grammar.pl
- browse files at revision 140.3.2
- compare with another revision
- download diff
- download tarball
- view history from revision 140.3.2
- Committer: Bernt M. Johnsen
- Date: 2009-11-20 10:29:35 UTC
- mto: (140.3.6 rqg-bernt)
- mto: This revision was merged to the branch mainline in revision 157.
- Revision ID: bernt.johnsen@sun.com-20091120102935-yldf173mresd20b3
New config
- files added:
- lib/GenTest/Properties.pm
- files modified:
- simplify-grammar_template.cfg
added
removed
util/simplify-grammar.pl
1
# use strict; mleich: disabled because of eval `cat $config_file`;
1
use strict;
2
2
use lib 'lib';
3
3
use lib '../lib';
4
4
use DBI;
5
use Carp;
6
use Getopt::Long;
7
use Data::Dumper;
5
8
6
9
use GenTest;
7
10
use GenTest::Constants;
8
11
use GenTest::Grammar;
12
use GenTest::Properties;
9
13
use GenTest::Simplifier::Grammar;
10
14
use Time::HiRes;
11
15
12
16
#
13
17
# RQG grammar simplification with an oracle() function based on
14
# 1. RQG exit status codes (-> @desired_status_codes)
15
# 2. expected RQG protocol output (-> @expected_output)
18
# 1. RQG exit status codes (-> desired_status_codes)
19
# 2. expected RQG protocol output (-> expected_output)
16
20
# Hint: 2. will be not checked if 1. already failed
17
21
#
18
# You need to adjusted parameters to your use case and environment.
19
# 1. Copy simplify-grammar_template.cfg to for exanple 1.cfg
22
# You need to adjust parameters to your use case and environment.
23
# 1. Copy simplify-grammar_template.cfg to for example 1.cfg
20
24
# 2. Adjust the settings
21
# 2. perl util/simplify-grammar.pl 1.cfg
25
# 2. perl util/simplify-grammar.pl --config 1.cfg
22
26
#
23
# This script is used to simplify grammar files to the smallest form that will still reproduce the desired outcome
24
# For the purpose, the GenTest::Simplifier::Grammar module provides progressively simple grammars, and we
25
# define an oracle() function that runs those grammars with the RQG and reports if the RQG returns the desired
26
# status code (usually something like STATUS_SERVER_CRASHED
27
# This script is used to simplify grammar files to the smallest form
28
# that will still reproduce the desired outcome
29
# For the purpose, the GenTest::Simplifier::Grammar module provides
30
# progressively simple grammars, and we define an oracle() function
31
# that runs those grammars with the RQG and reports if the RQG returns
32
# the desired status code (usually something like
33
# STATUS_SERVER_CRASHED
27
34
#
28
35
# For more information, please see:
29
36
#
30
37
# http://forge.mysql.com/wiki/RandomQueryGeneratorSimplification
31
38
#
32
39
33
# Preload user-modifiable settings
34
#---------------------------------
35
my $config_file = $ARGV[0];
36
if ( ! -f $config_file ) {
37
say ("config_file ('$config_file') is not a plain file");
38
say ("abort");
39
exit;
40
}
41
42
my $initial_grammar_file;
43
my $vardir_prefix;
44
my $storage;
45
my @rqg_options;
46
my @expected_output;
47
my $trials;
48
my $initial_seed;
49
my $grammar_flags;
50
my $search_var_size;
51
my @desired_status_codes;
52
53
eval `cat $config_file`;
40
# get configuration
41
42
my $options = {};
43
GetOptions($options, 'config=s', 'trials=i','storage=s','expected_output=s@');
44
my $config = GenTest::Properties->new(options => $options,
45
legal => ['desired_status_codes',
46
'expected_output',
47
'initial_grammar_file',
48
'grammar_flags',
49
'trials',
50
'initial_seed',
51
'search_var_size',
52
'rqg_options',
53
'vardir_prefix',
54
'storage'],
55
defaults => {expected_output => []},
56
);
57
58
# Dump settings
59
say("SIMPLIFY RQG GRAMMAR BASED ON EXPECTED CONTENT WITHIN SOME FILE");
60
say("---------------------------------------------------------------");
61
$config->printProps;
62
say("---------------------------------------------------------------");
63
64
## Calculate mysqld and rqg options
65
66
my $mysqlopt = $config->genOpt('--mysqld=--', $config->rqg_options->{mysqld});
67
68
## The one below is a hack.... Need support for nested options like these
69
delete $config->rqg_options->{mysqld};
70
71
my $rqgoptions = $config->genOpt('--', 'rqg_options');
54
72
55
73
# Determine some runtime parameter, check parameters, ....
56
74
58
76
59
77
say("The ID of this run is $run_id.");
60
78
61
open(INITIAL_GRAMMAR, $initial_grammar_file) or die "Umable to open '$initial_grammar_file': $!";;
62
read(INITIAL_GRAMMAR, my $initial_grammar , -s $initial_grammar_file);
79
open(INITIAL_GRAMMAR, $config->initial_grammar_file)
80
or croak "Unable to open '$config->initial_grammar_file': $!";;
81
read(INITIAL_GRAMMAR, my $initial_grammar , -s $config->initial_grammar_file);
63
82
close(INITIAL_GRAMMAR);
64
83
65
if ( ! -d $vardir_prefix ) {
66
say ("vardir_prefix '$vardir_prefix' is not an existing directory");
67
say ("abort");
68
exit;
84
if ( ! -d $config->vardir_prefix ) {
85
croak("vardir_prefix '".
86
$config->vardir_prefix.
87
"' is not an existing directory");
69
88
}
89
70
90
# Calculate a unique vardir (use $MTR_BUILD_THREAD or $run_id)
71
my $vardir = $vardir_prefix.'/var_'.$run_id;
91
92
my $vardir = $config->vardir_prefix.'/var_'.$run_id;
72
93
mkdir ($vardir);
73
push @mtr_options, "--vardir=$vardir";
94
push my @mtr_options, "--vardir=$vardir";
74
95
75
if ( ! -d $storage) {
76
say ("storage ('$storage') is not an existing directory");
77
say ("abort");
78
exit;
96
if ( ! -d $config->storage) {
97
croak("storage ('$config->storage') is not an existing directory");
79
98
}
80
$storage = $storage.'/'.$run_id;
99
my $storage = $config->storage.'/'.$run_id;
100
say "Storage is $storage";
81
101
mkdir ($storage);
82
102
83
103
84
# Dump settings
85
say("SIMPLIFY RQG GRAMMAR BASED ON EXPECTED CONTENT WITHIN SOME FILE");
86
say("---------------------------------------------------------------");
87
say("rqg_options : @rqg_options ");
88
say("initial_grammar_file : $initial_grammar_file ");
89
say("desired_status_codes : @desired_status_codes ");
90
say("expected_output : @expected_output ");
91
say("trials : $trials ");
92
say("initial_seed : $initial_seed ");
93
say("storage : $storage ");
94
say("vardir_prefix : $vardir_prefix ");
95
say("run_id : $run_id");
96
say("vardir : $vardir ");
97
say("---------------------------------------------------------------");
98
99
104
my $iteration;
100
my $good_seed = $initial_seed;
105
my $good_seed = $config->initial_seed;
101
106
102
107
my $simplifier = GenTest::Simplifier::Grammar->new(
103
grammar_flags => $grammar_flags,
104
oracle => sub {
105
106
$iteration++;
107
my $oracle_grammar = shift;
108
109
foreach my $trial (1..$trials) {
110
say("run_id = $run_id; iteration = $iteration; trial = $trial");
111
112
# $current_seed -- The seed value to be used for the next run.
113
# The test results of many grammars are quite sensitive to the seed value.
114
# 1. Run the first trial on the initial grammar with $initial_seed .
115
# This should raise the chance that the initial oracle check passes.
116
# 2. Run the first trial on a just simplified grammar with the last successfull
117
# seed value. In case the last simplification did remove some random determined
118
# we should have a bigger likelihood to reach the expected result.
119
# 3. In case of "threads = 1" it turned out that after a minor simplification the desired
120
# bad effect disappeared sometimes on the next run with the same seed value whereas
121
# a different seed value was again successful. Therefore we manipulate the seed value.
122
# In case of "threads > 1" this manipulation might be not required, but it will not
123
# make the conditions worse.
124
my $current_seed = $good_seed - 1 + $trial;
125
126
# Note(mleich): The grammar used is iteration specific. Don't store per trial.
127
# Shouldn't the next command be outside of the loop ?
128
my $current_grammar = $storage.'/'.$iteration.'.yy';
129
my $current_rqg_log = $storage.'/'.$iteration.'-'.$trial.'.log';
130
my $errfile = $vardir.'/log/master.err';
131
open (GRAMMAR, ">$current_grammar") or die "unable to create $current_grammar : $!";
132
print GRAMMAR $oracle_grammar;
133
close (GRAMMAR);
134
135
my $start_time = Time::HiRes::time();
136
137
# Note(mleich):
138
# In case of "threads = 1" it turned out that after a minor simplification the desired
139
# bad effect disappeared sometimes on the next run with the same seed value whereas
140
# a different seed value was again successful. Therefore we manipulate the seed value.
141
# In case of "threads > 1" this manipulation might be not required, but it will not
142
# make the conditions worse.
143
say("perl runall.pl ".join(' ', @rqg_options)." --grammar=$current_grammar --vardir=$vardir --seed=$current_seed 2>&1 >$current_rqg_log");
144
my $rqg_status = system("perl runall.pl ".join(' ', @rqg_options)." --grammar=$current_grammar --vardir=$vardir --seed=$current_seed 2>&1 >$current_rqg_log");
145
146
$rqg_status = $rqg_status >> 8;
147
148
my $end_time = Time::HiRes::time();
149
my $duration = $end_time - $start_time;
150
151
say("rqg_status = $rqg_status; duration = $duration");
152
153
return ORACLE_ISSUE_NO_LONGER_REPEATABLE if $rqg_status == STATUS_ENVIRONMENT_FAILURE;
154
155
foreach my $desired_status_code (@desired_status_codes) {
156
if (($rqg_status == $desired_status_code) ||
157
(($rqg_status != 0) && ($desired_status_code == STATUS_ANY_ERROR))) {
158
# "backtrace" output (independend of server crash or RQG kills the server) is in $current_rqg_log
159
open (my $my_logfile,'<'.$current_rqg_log) or die "unable to open $current_rqg_log : $!";
160
# If open (above) did not fail than size determination must be successful.
161
my @filestats = stat($current_rqg_log);
162
my $filesize = $filestats[7];
163
my $offset = $filesize - $search_var_size;
164
# Of course read fails if $offset < 0
165
if ( $offset < 0 ) { $offset = 0 } ;
166
read($my_logfile, my $rqgtest_output, $search_var_size, $offset );
167
close ($my_logfile);
168
# Debug print("$rqgtest_output");
169
170
# Every element of @expected_output must be found in $rqgtest_output.
171
my $success = 1;
172
foreach my $expected_output (@expected_output) {
173
if ($rqgtest_output =~ m{$expected_output}sio) {
174
say ("###### Found pattern: $expected_output ######");
175
} else {
176
say ("###### Not found pattern: $expected_output ######");
177
$success = 0;
178
last;
179
}
180
}
181
if ( 1 == $success ) {
182
say ("###### SUCCESS with $current_grammar ######");
183
$good_seed = $current_seed;
184
return ORACLE_ISSUE_STILL_REPEATABLE;
185
}
186
} # End of check if the output matches given string patterns
187
} # End of loop over @desired_status_codes
188
} # End of loop over the trials
189
return ORACLE_ISSUE_NO_LONGER_REPEATABLE;
190
}
191
);
108
grammar_flags => $config->grammar_flags,
109
oracle => sub {
110
$iteration++;
111
my $oracle_grammar = shift;
112
113
foreach my $trial (1..$config->trials) {
114
say("run_id = $run_id; iteration = $iteration; trial = $trial");
115
116
# $current_seed -- The seed value to be used for the next run.
117
# The test results of many grammars are quite sensitive to the
118
# seed value.
119
# 1. Run the first trial on the initial grammar with
120
# $config->initial_seed . This should raise the chance
121
# that the initial oracle check passes.
122
# 2. Run the first trial on a just simplified grammar with the
123
# last successfull seed value. In case the last
124
# simplification did remove some random determined we
125
# should have a bigger likelihood to reach the expected
126
# result.
127
# 3. In case of "threads = 1" it turned out that after a minor
128
# simplification the desired bad effect disappeared
129
# sometimes on the next run with the same seed value
130
# whereas a different seed value was again
131
# successful. Therefore we manipulate the seed value. In
132
# case of "threads > 1" this manipulation might be not
133
# required, but it will not make the conditions worse.
134
my $current_seed = $good_seed - 1 + $trial;
135
136
# Note(mleich): The grammar used is iteration specific. Don't
137
# store per trial. Shouldn't the next command be
138
# outside of the loop ?
139
my $current_grammar = $storage.'/'.$iteration.'.yy';
140
my $current_rqg_log = $storage.'/'.$iteration.'-'.$trial.'.log';
141
my $errfile = $vardir.'/log/master.err';
142
open (GRAMMAR, ">$current_grammar")
143
or croak "unable to create $current_grammar : $!";
144
print GRAMMAR $oracle_grammar;
145
close (GRAMMAR);
146
147
my $start_time = Time::HiRes::time();
148
149
# Note(mleich): In case of "threads = 1" it turned out that
150
# after a minor simplification the desired bad effect
151
# disappeared sometimes on the next run with the same
152
# seed value whereas a different seed value was again
153
# successful. Therefore we manipulate the seed value. In
154
# case of "threads > 1" this manipulation might be not
155
# required, but it will not make the conditions worse.
156
157
my $rqgcmd =
158
"perl runall.pl $rqgoptions $mysqlopt ".
159
"--grammar=$current_grammar ".
160
"--vardir=$vardir ".
161
"--seed=$current_seed 2>&1 >$current_rqg_log";
162
163
say($rqgcmd);
164
my $rqg_status = system($rqgcmd);
165
$rqg_status = $rqg_status >> 8;
166
167
my $end_time = Time::HiRes::time();
168
my $duration = $end_time - $start_time;
169
170
say("rqg_status = $rqg_status; duration = $duration");
171
172
return ORACLE_ISSUE_NO_LONGER_REPEATABLE
173
if $rqg_status == STATUS_ENVIRONMENT_FAILURE;
174
175
foreach my $desired_status_code (@{$config->desired_status_codes}) {
176
if (($rqg_status == $desired_status_code) ||
177
(($rqg_status != 0) && ($desired_status_code == STATUS_ANY_ERROR))) {
178
# "backtrace" output (independend of server crash
179
# or RQG kills the server) is in $current_rqg_log
180
open (my $my_logfile,'<'.$current_rqg_log)
181
or croak "unable to open $current_rqg_log : $!";
182
# If open (above) did not fail than size
183
# determination must be successful.
184
my @filestats = stat($current_rqg_log);
185
my $filesize = $filestats[7];
186
my $offset = $filesize - $config->search_var_size;
187
# Of course read fails if $offset < 0
188
$offset = 0 if $offset < 0;
189
read($my_logfile, my $rqgtest_output, $config->search_var_size, $offset );
190
close ($my_logfile);
191
# Debug print("$rqgtest_output");
192
193
# Every element of @expected_output must be found
194
# in $rqgtest_output.
195
my $success = 1;
196
foreach my $expected_output (@{$config->expected_output}) {
197
if ($rqgtest_output =~ m{$expected_output}sio) {
198
say ("###### Found pattern: $expected_output ######");
199
} else {
200
say ("###### Not found pattern: $expected_output ######");
201
$success = 0;
202
last;
203
}
204
}
205
if ( $success ) {
206
say ("###### SUCCESS with $current_grammar ######");
207
$good_seed = $current_seed;
208
return ORACLE_ISSUE_STILL_REPEATABLE;
209
}
210
} # End of check if the output matches given string patterns
211
} # End of loop over desired_status_codes
212
} # End of loop over the trials
213
return ORACLE_ISSUE_NO_LONGER_REPEATABLE;
214
}
215
);
192
216
193
217
my $simplified_grammar = $simplifier->simplify($initial_grammar);
194
218
Loggerhead is a web-based interface for Breezy
Version: 2.0.1