← Back to branch summary

~ubuntu-branches/ubuntu/trusty/drizzle/trusty

« back to all changes in this revision

Viewing changes to tests/randgen/lib/GenTest/Reporter/DrizzleRecovery.pm

  • Committer: Package Import Robot
  • Author(s): Clint Byrum
  • Date: 2012-06-19 10:46:49 UTC
  • mfrom: (1.1.6)
  • mto: This revision was merged to the branch mainline in revision 29.
  • Revision ID: package-import@ubuntu.com-20120619104649-e2l0ggd4oz3um0f4
Tags: upstream-7.1.36-stable
Import upstream version 7.1.36-stable

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/randgen/lib/GenTest/Reporter/DrizzleRecovery.pm
 
1
# Copyright (C) 2008-2009 Sun Microsystems, Inc. All rights reserved.
 
2
# Use is subject to license terms.
 
3
#
 
4
# This program is free software; you can redistribute it and/or modify
 
5
# it under the terms of the GNU General Public License as published by
 
6
# the Free Software Foundation; version 2 of the License.
 
7
#
 
8
# This program is distributed in the hope that it will be useful, but
 
9
# WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 
11
# General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
 
16
# USA
 
17
 
 
18
package GenTest::Reporter::DrizzleRecovery;
 
19
 
 
20
require Exporter;
 
21
@ISA = qw(GenTest::Reporter);
 
22
 
 
23
use strict;
 
24
use DBI;
 
25
use GenTest;
 
26
use GenTest::Constants;
 
27
use GenTest::Reporter;
 
28
use GenTest::Comparator;
 
29
use Data::Dumper;
 
30
use IPC::Open2;
 
31
use IPC::Open3;
 
32
 
 
33
my $first_reporter;
 
34
 
 
35
sub monitor {
 
36
        my $reporter = shift;
 
37
 
 
38
        # In case of two servers, we will be called twice.
 
39
        # Only kill the first server and ignore the second call.
 
40
        
 
41
        $first_reporter = $reporter if not defined $first_reporter;
 
42
        return STATUS_OK if $reporter ne $first_reporter;
 
43
        
 
44
        my $dbh = DBI->connect($reporter->dsn(), undef, undef, {PrintError => 0});
 
45
 
 
46
        if (time() > $reporter->testEnd() - 19) 
 
47
        {
 
48
                say("Sending shutdown() call to server in order to force a recovery.");
 
49
                $dbh->selectrow_array('SELECT shutdown()');
 
50
                return STATUS_SERVER_KILLED;
 
51
        } 
 
52
        else 
 
53
        {
 
54
                return STATUS_OK;
 
55
        }
 
56
}
 
57
 
 
58
sub report {
 
59
        my $reporter = shift;
 
60
 
 
61
        #
 
62
        # If there is a hang during recovery in one engine, another engine may continue to print
 
63
        # periodic diagnostic output forever. This prevents PB2 timeout mechanisms from kicking in
 
64
        # In order to avoid that, we set our own crude alarm as a stop-gap measure
 
65
        #
 
66
        alarm(3600);
 
67
 
 
68
        $first_reporter = $reporter if not defined $first_reporter;
 
69
        return STATUS_OK if $reporter ne $first_reporter;
 
70
 
 
71
        my $main_port;
 
72
        my $basedir;
 
73
 
 
74
        if (exists $ENV{'MASTER_MYPORT'})
 
75
        {
 
76
            $main_port = $ENV{'MASTER_MYPORT'};
 
77
        }
 
78
        else
 
79
        {
 
80
            $main_port = '9306';
 
81
        }
 
82
        
 
83
        if (exists $ENV{'DRIZZLE_BASEDIR'})
 
84
        {
 
85
            $basedir = $ENV{'DRIZZLE_BASEDIR'};
 
86
        }
 
87
        else
 
88
        {
 
89
            $basedir= $reporter->serverVariable('basedir');
 
90
        }
 
91
        say("$basedir");
 
92
        my $binary = $basedir.'/drizzled/drizzled' ;
 
93
        my $datadir = $reporter->serverVariable('datadir');
 
94
        my $recovery_datadir = $datadir.'_recovery';
 
95
        my $port = $reporter->serverVariable('mysql_protocol_port');
 
96
        
 
97
 
 
98
        my $dbh_prev = DBI->connect($reporter->dsn());
 
99
 
 
100
        if (defined $dbh_prev) {
 
101
                # Server is still running, kill it.
 
102
                $dbh_prev->disconnect();
 
103
 
 
104
                say("Sending shutdown() call to server.");
 
105
                $dbh_prev->selectrow_array('SELECT shutdown()');
 
106
                sleep(5);
 
107
        }
 
108
 
 
109
        say("Copying datadir... (interrupting the copy operation may cause a false recovery failure to be reported below");
 
110
        system("cp -r $datadir $recovery_datadir");
 
111
        system("rm -f $recovery_datadir/core*");        # Remove cores from any previous crash
 
112
 
 
113
        say("Copying complete");
 
114
        say("Attempting database recovery using the server ...");
 
115
 
 
116
        my @drizzled_options = (
 
117
                '--no-defaults',
 
118
                '--core-file',  
 
119
                '--datadir="'.$recovery_datadir.'"',
 
120
                '--basedir="'.$basedir.'"',
 
121
                '--plugin-add=shutdown_function',
 
122
                '--mysql-protocol.port='.$port,
 
123
 
 
124
        );
 
125
 
 
126
        my $drizzled_command = $binary.' '.join(' ', @drizzled_options).' 2>&1';
 
127
        say("Executing $drizzled_command .");
 
128
 
 
129
        my $drizzled_pid = open2(\*RDRFH, \*WTRFH, $drizzled_command);
 
130
 
 
131
        #
 
132
        # Phase1 - the server is running single-threaded. We consume the error log and parse it for
 
133
        # statements that indicate failed recovery
 
134
        # 
 
135
 
 
136
        my $recovery_status = STATUS_OK;
 
137
        while (<RDRFH>) {
 
138
                $_ =~ s{[\r\n]}{}siog;
 
139
                say($_);
 
140
                if ($_ =~ m{registration as a STORAGE ENGINE failed.}sio) {
 
141
                        say("Storage engine registration failed");
 
142
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
 
143
                } elsif ($_ =~ m{corrupt}) {
 
144
                        say("Log message '$_' indicates database corruption");
 
145
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
 
146
                } elsif ($_ =~ m{exception}sio) {
 
147
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
 
148
                } elsif ($_ =~ m{ready for connections}sio) {
 
149
                        say("Server Recovery was apparently successfull.") if $recovery_status == STATUS_OK ;
 
150
                        last;
 
151
                } elsif ($_ =~ m{device full error|no space left on device}sio) {
 
152
                        $recovery_status = STATUS_ENVIRONMENT_FAILURE;
 
153
                        last;
 
154
                } elsif (
 
155
                        ($_ =~ m{got signal}sio) ||
 
156
                        ($_ =~ m{segfault}sio) ||
 
157
                        ($_ =~ m{segmentation fault}sio)
 
158
                ) {
 
159
                        say("Recovery has apparently crashed.");
 
160
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
 
161
                }
 
162
        }
 
163
 
 
164
        my $dbh = DBI->connect($reporter->dsn());
 
165
        $recovery_status = STATUS_DATABASE_CORRUPTION if not defined $dbh && $recovery_status == STATUS_OK;
 
166
 
 
167
        if ($recovery_status > STATUS_OK) {
 
168
                say("Recovery has failed.");
 
169
                return $recovery_status;
 
170
        }
 
171
        
 
172
        # 
 
173
        # Phase 2 - server is now running, so we execute various statements in order to verify table consistency
 
174
        # However, while we do that, we are still responsible for processing the error log and dumping it to our stdout.
 
175
        # If we do not do that, and the server calls flish(stdout) , it will hang waiting for us to consume its stdout, which
 
176
        # we would no longer be doing. So, we call eater(), which forks a separate process to read the log and dump it to stdout.
 
177
        #
 
178
 
 
179
        say("Testing database consistency");
 
180
 
 
181
        my $eater_pid = eater(*RDRFH);
 
182
 
 
183
        my $databases = $dbh->selectcol_arrayref("SHOW DATABASES");
 
184
        foreach my $database (@$databases) {
 
185
                next if $database =~ m{^(pbxt|mysql|information_schema|data_dictionary)$}sio;
 
186
                $dbh->do("USE $database");
 
187
                my $tables = $dbh->selectcol_arrayref("SHOW TABLES");
 
188
                foreach my $table (@$tables) {
 
189
                        say("Verifying table: $table; database: $database");
 
190
 
 
191
                        my $sth_keys = $dbh->prepare("
 
192
                                SHOW KEYS FROM `$database`.`$table`
 
193
                        ");
 
194
 
 
195
                        $sth_keys->execute();
 
196
 
 
197
                        my @walk_queries;
 
198
 
 
199
                        while (my $key_hashref = $sth_keys->fetchrow_hashref()) {
 
200
                                my $key_name = $key_hashref->{Key_name};
 
201
                                my $column_name = $key_hashref->{Column_name};
 
202
 
 
203
                                foreach my $select_type ('*' , "`$column_name`") {
 
204
                                        my $main_predicate;
 
205
                                        if ($column_name =~ m{int}sio) {
 
206
                                                $main_predicate = "WHERE `$column_name` >= -9223372036854775808";
 
207
                                        } elsif ($column_name =~ m{char}sio) {
 
208
                                                $main_predicate = "WHERE `$column_name` = '' OR `$column_name` != ''";
 
209
                                        } elsif ($column_name =~ m{date}sio) {
 
210
                                                $main_predicate = "WHERE (`$column_name` >= '1900-01-01' OR `$column_name` = '0000-00-00') ";
 
211
                                        } elsif ($column_name =~ m{time}sio) {
 
212
                                                $main_predicate = "WHERE (`$column_name` >= '-838:59:59' OR `$column_name` = '00:00:00') ";
 
213
                                        } else {
 
214
                                                next;
 
215
                                        }
 
216
        
 
217
                                        if ($column_name =~m{not_null}) {}
 
218
                                        else
 
219
                                        {
 
220
                                                $main_predicate = $main_predicate." OR `$column_name` IS NULL OR `$column_name` IS NOT NULL";
 
221
                                        }
 
222
                        
 
223
                                        push @walk_queries, "SELECT $select_type FROM `$database`.`$table` FORCE INDEX ($key_name) ".$main_predicate;
 
224
                                }
 
225
                        };
 
226
 
 
227
                        my %rows;
 
228
                        my %data;
 
229
 
 
230
                        foreach my $walk_query (@walk_queries) {
 
231
                                my $sth_rows = $dbh->prepare($walk_query);
 
232
                                $sth_rows->execute();
 
233
 
 
234
                                if (defined $sth_rows->err()) {
 
235
                                        say("Failing query is $walk_query.");
 
236
                                        return STATUS_RECOVERY_FAILURE;
 
237
                                }
 
238
 
 
239
                                my $rows = $sth_rows->rows();
 
240
                                $sth_rows->finish();
 
241
 
 
242
                                push @{$rows{$rows}} , $walk_query;
 
243
                               
 
244
                        }
 
245
 
 
246
                        if (keys %rows > 1) {
 
247
                                say("Table `$database`.`$table` is inconsistent.");
 
248
                                print Dumper \%rows;
 
249
 
 
250
                                my @rows_sorted = grep { $_ > 0 } sort keys %rows;
 
251
                        
 
252
                                my $least_sql = $rows{$rows_sorted[0]}->[0];
 
253
                                my $most_sql  = $rows{$rows_sorted[$#rows_sorted]}->[0];
 
254
                        
 
255
                                say("Query that returned least rows: $least_sql\n");
 
256
                                say("Query that returned most rows: $most_sql\n");
 
257
        
 
258
                                my $least_result_obj = GenTest::Result->new(
 
259
                                        data => $dbh->selectall_arrayref($least_sql)
 
260
                                );
 
261
                                
 
262
                                my $most_result_obj = GenTest::Result->new(
 
263
                                        data => $dbh->selectall_arrayref($most_sql)
 
264
                                );
 
265
 
 
266
                                say(GenTest::Comparator::dumpDiff($least_result_obj, $most_result_obj));
 
267
 
 
268
                                $recovery_status = STATUS_DATABASE_CORRUPTION;
 
269
                        }
 
270
 
 
271
 
 
272
                                foreach my $sql (
 
273
                                        "CHECK TABLE `$database`.`$table` ",
 
274
                                        "ANALYZE TABLE `$database`.`$table`",
 
275
                                        #"OPTIMIZE TABLE `$database`.`$table`",
 
276
                                        #"REPAIR TABLE `$database`.`$table` "
 
277
                                ) 
 
278
                                {
 
279
                                        say("Executing $sql.");
 
280
                                        my $sth = $dbh->prepare($sql);
 
281
                                        if (defined $sth) 
 
282
                                        {
 
283
                                                $sth->execute();
 
284
 
 
285
                                                return STATUS_DATABASE_CORRUPTION if $dbh->err() > 0 && $dbh->err() != 1178;
 
286
                                                if ($sth->{NUM_OF_FIELDS} > 0) 
 
287
                                                {
 
288
                                                        my $result = Dumper($sth->fetchall_arrayref());
 
289
                                                        next if $result =~ m{is not BASE TABLE}sio;     # Do not process VIEWs
 
290
                                                        if ($result =~ m{error'|corrupt|repaired|invalid|crashed}sio) 
 
291
                                                        {
 
292
                                                                print $result;
 
293
                                                                return STATUS_DATABASE_CORRUPTION
 
294
                                                        }
 
295
                                                };
 
296
 
 
297
                                                $sth->finish();
 
298
                                        } 
 
299
                                        else 
 
300
                                        {
 
301
                                                say("Prepare failed: ".$dbh->errrstr());
 
302
                                                return STATUS_DATABASE_CORRUPTION;
 
303
                                        }
 
304
                                }
 
305
                }
 
306
        }
 
307
 
 
308
        close(MYSQLD);
 
309
 
 
310
        if ($recovery_status > STATUS_OK) {
 
311
                say("Recovery has failed.");
 
312
                return $recovery_status;
 
313
        } elsif ($reporter->serverVariable('falcon_error_inject') ne '') {
 
314
                return STATUS_SERVER_KILLED;
 
315
        } else {
 
316
                return STATUS_OK;
 
317
        }
 
318
}
 
319
 
 
320
sub eater {
 
321
        my $fh = shift;
 
322
        if (my $eater_pid = fork()) {
 
323
                # parent
 
324
                return $eater_pid;
 
325
        } else {
 
326
                # child
 
327
                $0 = 'Recovery log eater';
 
328
                while (<$fh>) {
 
329
                        $_ =~ s{[\r\n]}{}siog;
 
330
                        say($_);
 
331
                }
 
332
 
 
333
                exit(0);
 
334
        }
 
335
}
 
336
 
 
337
sub type {
 
338
        return REPORTER_TYPE_ALWAYS | REPORTER_TYPE_PERIODIC;
 
339
}
 
340
 
 
341
1;