← Back to branch summary

~ubuntu-branches/ubuntu/trusty/drizzle/trusty

« back to all changes in this revision

Viewing changes to tests/kewpie/randgen/lib/GenTest/Reporter/DrizzleRecovery.pm

  • Committer: Package Import Robot
  • Author(s): Dmitrijs Ledkovs
  • Date: 2013-10-29 15:43:40 UTC
  • mfrom: (1.2.12) (2.1.19 trusty-proposed)
  • Revision ID: package-import@ubuntu.com-20131029154340-2gp39el6cv8bwf2o
Tags: 1:7.2.3-2ubuntu1
* Merge from debian, remaining changes:
  - Link against boost_system because of boost_thread.
  - Add required libs to message/include.am
  - Add upstart job and adjust init script to be upstart compatible.
  - Disable -floop-parallelize-all due to gcc-4.8/4.9 compiler ICE
    http://gcc.gnu.org/bugzilla/show_bug.cgi?id=57732

Show diffs side-by-side

added added

removed removed

Lines of Context:
tests/kewpie/randgen/lib/GenTest/Reporter/DrizzleRecovery.pm
1
 
# Copyright (C) 2008-2009 Sun Microsystems, Inc. All rights reserved.
2
 
# Use is subject to license terms.
3
 
#
4
 
# This program is free software; you can redistribute it and/or modify
5
 
# it under the terms of the GNU General Public License as published by
6
 
# the Free Software Foundation; version 2 of the License.
7
 
#
8
 
# This program is distributed in the hope that it will be useful, but
9
 
# WITHOUT ANY WARRANTY; without even the implied warranty of
10
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
 
# General Public License for more details.
12
 
#
13
 
# You should have received a copy of the GNU General Public License
14
 
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
16
 
# USA
17
 
 
18
 
package GenTest::Reporter::DrizzleRecovery;
19
 
 
20
 
require Exporter;
21
 
@ISA = qw(GenTest::Reporter);
22
 
 
23
 
use strict;
24
 
use DBI;
25
 
use GenTest;
26
 
use GenTest::Constants;
27
 
use GenTest::Reporter;
28
 
use GenTest::Comparator;
29
 
use Data::Dumper;
30
 
use IPC::Open2;
31
 
use IPC::Open3;
32
 
 
33
 
my $first_reporter;
34
 
 
35
 
sub monitor {
36
 
        my $reporter = shift;
37
 
 
38
 
        # In case of two servers, we will be called twice.
39
 
        # Only kill the first server and ignore the second call.
40
 
        
41
 
        $first_reporter = $reporter if not defined $first_reporter;
42
 
        return STATUS_OK if $reporter ne $first_reporter;
43
 
        
44
 
        my $dbh = DBI->connect($reporter->dsn(), undef, undef, {PrintError => 0});
45
 
 
46
 
        if (time() > $reporter->testEnd() - 19) 
47
 
        {
48
 
                say("Sending shutdown() call to server in order to force a recovery.");
49
 
                $dbh->selectrow_array('SELECT shutdown()');
50
 
                return STATUS_SERVER_KILLED;
51
 
        } 
52
 
        else 
53
 
        {
54
 
                return STATUS_OK;
55
 
        }
56
 
}
57
 
 
58
 
sub report {
59
 
        my $reporter = shift;
60
 
 
61
 
        #
62
 
        # If there is a hang during recovery in one engine, another engine may continue to print
63
 
        # periodic diagnostic output forever. This prevents PB2 timeout mechanisms from kicking in
64
 
        # In order to avoid that, we set our own crude alarm as a stop-gap measure
65
 
        #
66
 
        alarm(3600);
67
 
 
68
 
        $first_reporter = $reporter if not defined $first_reporter;
69
 
        return STATUS_OK if $reporter ne $first_reporter;
70
 
 
71
 
        my $main_port;
72
 
        my $basedir;
73
 
 
74
 
        if (exists $ENV{'MASTER_MYPORT'})
75
 
        {
76
 
            $main_port = $ENV{'MASTER_MYPORT'};
77
 
        }
78
 
        else
79
 
        {
80
 
            $main_port = '9306';
81
 
        }
82
 
        
83
 
        if (exists $ENV{'DRIZZLE_BASEDIR'})
84
 
        {
85
 
            $basedir = $ENV{'DRIZZLE_BASEDIR'};
86
 
        }
87
 
        else
88
 
        {
89
 
            $basedir= $reporter->serverVariable('basedir');
90
 
        }
91
 
        say("$basedir");
92
 
        my $binary = $basedir.'/drizzled/drizzled' ;
93
 
        my $datadir = $reporter->serverVariable('datadir');
94
 
        my $recovery_datadir = $datadir.'_recovery';
95
 
        my $port = $reporter->serverVariable('mysql_protocol_port');
96
 
        
97
 
 
98
 
        my $dbh_prev = DBI->connect($reporter->dsn());
99
 
 
100
 
        if (defined $dbh_prev) {
101
 
                # Server is still running, kill it.
102
 
                $dbh_prev->disconnect();
103
 
 
104
 
                say("Sending shutdown() call to server.");
105
 
                $dbh_prev->selectrow_array('SELECT shutdown()');
106
 
                sleep(5);
107
 
        }
108
 
 
109
 
        say("Copying datadir... (interrupting the copy operation may cause a false recovery failure to be reported below");
110
 
        system("cp -r $datadir $recovery_datadir");
111
 
        system("rm -f $recovery_datadir/core*");        # Remove cores from any previous crash
112
 
 
113
 
        say("Copying complete");
114
 
        say("Attempting database recovery using the server ...");
115
 
 
116
 
        my @drizzled_options = (
117
 
                '--no-defaults',
118
 
                '--core-file',  
119
 
                '--datadir="'.$recovery_datadir.'"',
120
 
                '--basedir="'.$basedir.'"',
121
 
                '--plugin-add=shutdown_function',
122
 
                '--mysql-protocol.port='.$port,
123
 
 
124
 
        );
125
 
 
126
 
        my $drizzled_command = $binary.' '.join(' ', @drizzled_options).' 2>&1';
127
 
        say("Executing $drizzled_command .");
128
 
 
129
 
        my $drizzled_pid = open2(\*RDRFH, \*WTRFH, $drizzled_command);
130
 
 
131
 
        #
132
 
        # Phase1 - the server is running single-threaded. We consume the error log and parse it for
133
 
        # statements that indicate failed recovery
134
 
        # 
135
 
 
136
 
        my $recovery_status = STATUS_OK;
137
 
        while (<RDRFH>) {
138
 
                $_ =~ s{[\r\n]}{}siog;
139
 
                say($_);
140
 
                if ($_ =~ m{registration as a STORAGE ENGINE failed.}sio) {
141
 
                        say("Storage engine registration failed");
142
 
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
143
 
                } elsif ($_ =~ m{corrupt}) {
144
 
                        say("Log message '$_' indicates database corruption");
145
 
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
146
 
                } elsif ($_ =~ m{exception}sio) {
147
 
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
148
 
                } elsif ($_ =~ m{ready for connections}sio) {
149
 
                        say("Server Recovery was apparently successfull.") if $recovery_status == STATUS_OK ;
150
 
                        last;
151
 
                } elsif ($_ =~ m{device full error|no space left on device}sio) {
152
 
                        $recovery_status = STATUS_ENVIRONMENT_FAILURE;
153
 
                        last;
154
 
                } elsif (
155
 
                        ($_ =~ m{got signal}sio) ||
156
 
                        ($_ =~ m{segfault}sio) ||
157
 
                        ($_ =~ m{segmentation fault}sio)
158
 
                ) {
159
 
                        say("Recovery has apparently crashed.");
160
 
                        $recovery_status = STATUS_DATABASE_CORRUPTION;
161
 
                }
162
 
        }
163
 
 
164
 
        my $dbh = DBI->connect($reporter->dsn());
165
 
        $recovery_status = STATUS_DATABASE_CORRUPTION if not defined $dbh && $recovery_status == STATUS_OK;
166
 
 
167
 
        if ($recovery_status > STATUS_OK) {
168
 
                say("Recovery has failed.");
169
 
                return $recovery_status;
170
 
        }
171
 
        
172
 
        # 
173
 
        # Phase 2 - server is now running, so we execute various statements in order to verify table consistency
174
 
        # However, while we do that, we are still responsible for processing the error log and dumping it to our stdout.
175
 
        # If we do not do that, and the server calls flish(stdout) , it will hang waiting for us to consume its stdout, which
176
 
        # we would no longer be doing. So, we call eater(), which forks a separate process to read the log and dump it to stdout.
177
 
        #
178
 
 
179
 
        say("Testing database consistency");
180
 
 
181
 
        my $eater_pid = eater(*RDRFH);
182
 
 
183
 
        my $databases = $dbh->selectcol_arrayref("SHOW DATABASES");
184
 
        foreach my $database (@$databases) {
185
 
                next if $database =~ m{^(pbxt|mysql|information_schema|data_dictionary)$}sio;
186
 
                $dbh->do("USE $database");
187
 
                my $tables = $dbh->selectcol_arrayref("SHOW TABLES");
188
 
                foreach my $table (@$tables) {
189
 
                        say("Verifying table: $table; database: $database");
190
 
 
191
 
                        my $sth_keys = $dbh->prepare("
192
 
                                SHOW KEYS FROM `$database`.`$table`
193
 
                        ");
194
 
 
195
 
                        $sth_keys->execute();
196
 
 
197
 
                        my @walk_queries;
198
 
 
199
 
                        while (my $key_hashref = $sth_keys->fetchrow_hashref()) {
200
 
                                my $key_name = $key_hashref->{Key_name};
201
 
                                my $column_name = $key_hashref->{Column_name};
202
 
 
203
 
                                foreach my $select_type ('*' , "`$column_name`") {
204
 
                                        my $main_predicate;
205
 
                                        if ($column_name =~ m{int}sio) {
206
 
                                                $main_predicate = "WHERE `$column_name` >= -9223372036854775808";
207
 
                                        } elsif ($column_name =~ m{char}sio) {
208
 
                                                $main_predicate = "WHERE `$column_name` = '' OR `$column_name` != ''";
209
 
                                        } elsif ($column_name =~ m{date}sio) {
210
 
                                                $main_predicate = "WHERE (`$column_name` >= '1900-01-01' OR `$column_name` = '0000-00-00') ";
211
 
                                        } elsif ($column_name =~ m{time}sio) {
212
 
                                                $main_predicate = "WHERE (`$column_name` >= '-838:59:59' OR `$column_name` = '00:00:00') ";
213
 
                                        } else {
214
 
                                                next;
215
 
                                        }
216
 
        
217
 
                                        if ($column_name =~m{not_null}) {}
218
 
                                        else
219
 
                                        {
220
 
                                                $main_predicate = $main_predicate." OR `$column_name` IS NULL OR `$column_name` IS NOT NULL";
221
 
                                        }
222
 
                        
223
 
                                        push @walk_queries, "SELECT $select_type FROM `$database`.`$table` FORCE INDEX ($key_name) ".$main_predicate;
224
 
                                }
225
 
                        };
226
 
 
227
 
                        my %rows;
228
 
                        my %data;
229
 
 
230
 
                        foreach my $walk_query (@walk_queries) {
231
 
                                my $sth_rows = $dbh->prepare($walk_query);
232
 
                                $sth_rows->execute();
233
 
 
234
 
                                if (defined $sth_rows->err()) {
235
 
                                        say("Failing query is $walk_query.");
236
 
                                        return STATUS_RECOVERY_FAILURE;
237
 
                                }
238
 
 
239
 
                                my $rows = $sth_rows->rows();
240
 
                                $sth_rows->finish();
241
 
 
242
 
                                push @{$rows{$rows}} , $walk_query;
243
 
                               
244
 
                        }
245
 
 
246
 
                        if (keys %rows > 1) {
247
 
                                say("Table `$database`.`$table` is inconsistent.");
248
 
                                print Dumper \%rows;
249
 
 
250
 
                                my @rows_sorted = grep { $_ > 0 } sort keys %rows;
251
 
                        
252
 
                                my $least_sql = $rows{$rows_sorted[0]}->[0];
253
 
                                my $most_sql  = $rows{$rows_sorted[$#rows_sorted]}->[0];
254
 
                        
255
 
                                say("Query that returned least rows: $least_sql\n");
256
 
                                say("Query that returned most rows: $most_sql\n");
257
 
        
258
 
                                my $least_result_obj = GenTest::Result->new(
259
 
                                        data => $dbh->selectall_arrayref($least_sql)
260
 
                                );
261
 
                                
262
 
                                my $most_result_obj = GenTest::Result->new(
263
 
                                        data => $dbh->selectall_arrayref($most_sql)
264
 
                                );
265
 
 
266
 
                                say(GenTest::Comparator::dumpDiff($least_result_obj, $most_result_obj));
267
 
 
268
 
                                $recovery_status = STATUS_DATABASE_CORRUPTION;
269
 
                        }
270
 
 
271
 
 
272
 
                                foreach my $sql (
273
 
                                        "CHECK TABLE `$database`.`$table` ",
274
 
                                        "ANALYZE TABLE `$database`.`$table`",
275
 
                                        #"OPTIMIZE TABLE `$database`.`$table`",
276
 
                                        #"REPAIR TABLE `$database`.`$table` "
277
 
                                ) 
278
 
                                {
279
 
                                        say("Executing $sql.");
280
 
                                        my $sth = $dbh->prepare($sql);
281
 
                                        if (defined $sth) 
282
 
                                        {
283
 
                                                $sth->execute();
284
 
 
285
 
                                                return STATUS_DATABASE_CORRUPTION if $dbh->err() > 0 && $dbh->err() != 1178;
286
 
                                                if ($sth->{NUM_OF_FIELDS} > 0) 
287
 
                                                {
288
 
                                                        my $result = Dumper($sth->fetchall_arrayref());
289
 
                                                        next if $result =~ m{is not BASE TABLE}sio;     # Do not process VIEWs
290
 
                                                        if ($result =~ m{error'|corrupt|repaired|invalid|crashed}sio) 
291
 
                                                        {
292
 
                                                                print $result;
293
 
                                                                return STATUS_DATABASE_CORRUPTION
294
 
                                                        }
295
 
                                                };
296
 
 
297
 
                                                $sth->finish();
298
 
                                        } 
299
 
                                        else 
300
 
                                        {
301
 
                                                say("Prepare failed: ".$dbh->errrstr());
302
 
                                                return STATUS_DATABASE_CORRUPTION;
303
 
                                        }
304
 
                                }
305
 
                }
306
 
        }
307
 
 
308
 
        close(MYSQLD);
309
 
 
310
 
        if ($recovery_status > STATUS_OK) {
311
 
                say("Recovery has failed.");
312
 
                return $recovery_status;
313
 
        } elsif ($reporter->serverVariable('falcon_error_inject') ne '') {
314
 
                return STATUS_SERVER_KILLED;
315
 
        } else {
316
 
                return STATUS_OK;
317
 
        }
318
 
}
319
 
 
320
 
sub eater {
321
 
        my $fh = shift;
322
 
        if (my $eater_pid = fork()) {
323
 
                # parent
324
 
                return $eater_pid;
325
 
        } else {
326
 
                # child
327
 
                $0 = 'Recovery log eater';
328
 
                while (<$fh>) {
329
 
                        $_ =~ s{[\r\n]}{}siog;
330
 
                        say($_);
331
 
                }
332
 
 
333
 
                exit(0);
334
 
        }
335
 
}
336
 
 
337
 
sub type {
338
 
        return REPORTER_TYPE_ALWAYS | REPORTER_TYPE_PERIODIC;
339
 
}
340
 
 
341
 
1;