~percona-toolkit-dev/percona-toolkit/fix-password-comma-bug-886077
» Revision
94.2.4
: bin/pt-table-checksum
« back to all changes in this revision
Viewing changes to bin/pt-table-checksum
- browse files at revision 94.2.4
- compare with another revision
- download diff
- download tarball
- view history from revision 94.2.4
- Committer: Daniel Nichter
- Date: 2012-02-07 20:10:11 UTC
- mfrom: (174 2.0)
- mto: This revision was merged to the branch mainline in revision 189.
- Revision ID: daniel@percona.com-20120207201011-sok2c1f2ay9qr3gm
Merge trunk r174.
- files added:
- bin/pt-align
- config/sphinx-build/percona-theme
- config/sphinx-build/percona-theme/static
- lib/bash
- t/lib/bash
- t/lib/samples/MasterSlave
- t/lib/samples/NibbleIterator
- t/lib/samples/bash
- t/pt-align
- t/pt-align/samples
- t/pt-diskstats/expected
- t/pt-ioprofile
- t/pt-ioprofile/samples
- t/pt-stalk/samples
- t/pt-table-checksum/samples/checksum_results
- t/pt-table-checksum/scripts
- t/pt-tcp-model/samples/in
- t/pt-trend
- t/pt-trend/samples
- files removed:
- bin/pt-collect
- t/pt-collect
- files renamed:
- t/pt-table-checksum/replicate.t => t/pt-table-checksum/basics.t
- t/pt-table-checksum/issue_81.t => t/pt-table-checksum/error_handling.t
- files modified:
- .bzrignore
added
removed
bin/pt-table-checksum
6
6
7
7
use strict;
8
8
use warnings FATAL => 'all';
9
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
9
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
10
10
11
11
# ###########################################################################
12
# TableParser package
12
# DSNParser package
13
13
# This package is a copy without comments from the original. The original
14
14
# with comments and its test file can be found in the Bazaar repository at,
15
# lib/TableParser.pm
16
# t/lib/TableParser.t
15
# lib/DSNParser.pm
16
# t/lib/DSNParser.t
17
17
# See https://launchpad.net/percona-toolkit for more information.
18
18
# ###########################################################################
19
19
{
20
package TableParser;
20
package DSNParser;
21
21
22
22
use strict;
23
23
use warnings FATAL => 'all';
24
24
use English qw(-no_match_vars);
25
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
25
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
26
26
27
27
use Data::Dumper;
28
$Data::Dumper::Indent = 1;
29
$Data::Dumper::Sortkeys = 1;
28
$Data::Dumper::Indent = 0;
30
29
$Data::Dumper::Quotekeys = 0;
31
30
31
eval {
32
require DBI;
33
};
34
my $have_dbi = $EVAL_ERROR ? 0 : 1;
35
32
36
sub new {
33
37
my ( $class, %args ) = @_;
34
my @required_args = qw(Quoter);
35
foreach my $arg ( @required_args ) {
38
foreach my $arg ( qw(opts) ) {
36
39
die "I need a $arg argument" unless $args{$arg};
37
40
}
38
my $self = { %args };
41
my $self = {
42
opts => {} # h, P, u, etc. Should come from DSN OPTIONS section in POD.
43
};
44
foreach my $opt ( @{$args{opts}} ) {
45
if ( !$opt->{key} || !$opt->{desc} ) {
46
die "Invalid DSN option: ", Dumper($opt);
47
}
48
PTDEBUG && _d('DSN option:',
49
join(', ',
50
map { "$_=" . (defined $opt->{$_} ? ($opt->{$_} || '') : 'undef') }
51
keys %$opt
52
)
53
);
54
$self->{opts}->{$opt->{key}} = {
55
dsn => $opt->{dsn},
56
desc => $opt->{desc},
57
copy => $opt->{copy} || 0,
58
};
59
}
39
60
return bless $self, $class;
40
61
}
41
62
63
sub prop {
64
my ( $self, $prop, $value ) = @_;
65
if ( @_ > 2 ) {
66
PTDEBUG && _d('Setting', $prop, 'property');
67
$self->{$prop} = $value;
68
}
69
return $self->{$prop};
70
}
71
42
72
sub parse {
43
my ( $self, $ddl, $opts ) = @_;
44
return unless $ddl;
45
if ( ref $ddl eq 'ARRAY' ) {
46
if ( lc $ddl->[0] eq 'table' ) {
47
$ddl = $ddl->[1];
48
}
49
else {
50
return {
51
engine => 'VIEW',
52
};
53
}
54
}
55
56
if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {
57
die "Cannot parse table definition; is ANSI quoting "
58
. "enabled or SQL_QUOTE_SHOW_CREATE disabled?";
59
}
60
61
my ($name) = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/;
62
(undef, $name) = $self->{Quoter}->split_unquote($name) if $name;
63
64
$ddl =~ s/(`[^`]+`)/\L$1/g;
65
66
my $engine = $self->get_engine($ddl);
67
68
my @defs = $ddl =~ m/^(\s+`.*?),?$/gm;
69
my @cols = map { $_ =~ m/`([^`]+)`/ } @defs;
70
MKDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols));
71
72
my %def_for;
73
@def_for{@cols} = @defs;
74
75
my (@nums, @null);
76
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
77
foreach my $col ( @cols ) {
78
my $def = $def_for{$col};
79
my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/;
80
die "Can't determine column type for $def" unless $type;
81
$type_for{$col} = $type;
82
if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) {
83
push @nums, $col;
84
$is_numeric{$col} = 1;
85
}
86
if ( $def !~ m/NOT NULL/ ) {
87
push @null, $col;
88
$is_nullable{$col} = 1;
89
}
90
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
91
}
92
93
my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable);
94
95
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
96
97
return {
98
name => $name,
99
cols => \@cols,
100
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
101
is_col => { map { $_ => 1 } @cols },
102
null_cols => \@null,
103
is_nullable => \%is_nullable,
104
is_autoinc => \%is_autoinc,
105
clustered_key => $clustered_key,
106
keys => $keys,
107
defs => \%def_for,
108
numeric_cols => \@nums,
109
is_numeric => \%is_numeric,
110
engine => $engine,
111
type_for => \%type_for,
112
charset => $charset,
113
};
114
}
115
116
sub sort_indexes {
117
my ( $self, $tbl ) = @_;
118
119
my @indexes
120
= sort {
121
(($a ne 'PRIMARY') <=> ($b ne 'PRIMARY'))
122
|| ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} )
123
|| ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} )
124
|| ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) )
125
}
126
grep {
127
$tbl->{keys}->{$_}->{type} eq 'BTREE'
128
}
129
sort keys %{$tbl->{keys}};
130
131
MKDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes));
132
return @indexes;
133
}
134
135
sub find_best_index {
136
my ( $self, $tbl, $index ) = @_;
137
my $best;
138
if ( $index ) {
139
($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}};
140
}
141
if ( !$best ) {
142
if ( $index ) {
143
die "Index '$index' does not exist in table";
144
}
145
else {
146
($best) = $self->sort_indexes($tbl);
147
}
148
}
149
MKDEBUG && _d('Best index found is', $best);
150
return $best;
151
}
152
153
sub find_possible_keys {
154
my ( $self, $dbh, $database, $table, $quoter, $where ) = @_;
155
return () unless $where;
156
my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table)
157
. ' WHERE ' . $where;
158
MKDEBUG && _d($sql);
159
my $expl = $dbh->selectrow_hashref($sql);
160
$expl = { map { lc($_) => $expl->{$_} } keys %$expl };
161
if ( $expl->{possible_keys} ) {
162
MKDEBUG && _d('possible_keys =', $expl->{possible_keys});
163
my @candidates = split(',', $expl->{possible_keys});
164
my %possible = map { $_ => 1 } @candidates;
165
if ( $expl->{key} ) {
166
MKDEBUG && _d('MySQL chose', $expl->{key});
167
unshift @candidates, grep { $possible{$_} } split(',', $expl->{key});
168
MKDEBUG && _d('Before deduping:', join(', ', @candidates));
169
my %seen;
170
@candidates = grep { !$seen{$_}++ } @candidates;
171
}
172
MKDEBUG && _d('Final list:', join(', ', @candidates));
173
return @candidates;
174
}
175
else {
176
MKDEBUG && _d('No keys in possible_keys');
177
return ();
178
}
179
}
180
181
sub check_table {
182
my ( $self, %args ) = @_;
183
my @required_args = qw(dbh db tbl);
184
foreach my $arg ( @required_args ) {
185
die "I need a $arg argument" unless $args{$arg};
186
}
187
my ($dbh, $db, $tbl) = @args{@required_args};
188
my $q = $self->{Quoter};
189
my $db_tbl = $q->quote($db, $tbl);
190
MKDEBUG && _d('Checking', $db_tbl);
191
192
my $sql = "SHOW TABLES FROM " . $q->quote($db)
193
. ' LIKE ' . $q->literal_like($tbl);
194
MKDEBUG && _d($sql);
195
my $row;
196
eval {
197
$row = $dbh->selectrow_arrayref($sql);
198
};
199
if ( $EVAL_ERROR ) {
200
MKDEBUG && _d($EVAL_ERROR);
201
return 0;
202
}
203
if ( !$row->[0] || $row->[0] ne $tbl ) {
204
MKDEBUG && _d('Table does not exist');
205
return 0;
206
}
207
208
MKDEBUG && _d('Table exists; no privs to check');
209
return 1 unless $args{all_privs};
210
211
$sql = "SHOW FULL COLUMNS FROM $db_tbl";
212
MKDEBUG && _d($sql);
213
eval {
214
$row = $dbh->selectrow_hashref($sql);
215
};
216
if ( $EVAL_ERROR ) {
217
MKDEBUG && _d($EVAL_ERROR);
218
return 0;
219
}
220
if ( !scalar keys %$row ) {
221
MKDEBUG && _d('Table has no columns:', Dumper($row));
222
return 0;
223
}
224
my $privs = $row->{privileges} || $row->{Privileges};
225
226
$sql = "DELETE FROM $db_tbl LIMIT 0";
227
MKDEBUG && _d($sql);
228
eval {
229
$dbh->do($sql);
230
};
231
my $can_delete = $EVAL_ERROR ? 0 : 1;
232
233
MKDEBUG && _d('User privs on', $db_tbl, ':', $privs,
234
($can_delete ? 'delete' : ''));
235
236
if ( !($privs =~ m/select/ && $privs =~ m/insert/ && $privs =~ m/update/
237
&& $can_delete) ) {
238
MKDEBUG && _d('User does not have all privs');
239
return 0;
240
}
241
242
MKDEBUG && _d('User has all privs');
243
return 1;
244
}
245
246
sub get_engine {
247
my ( $self, $ddl, $opts ) = @_;
248
my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/;
249
MKDEBUG && _d('Storage engine:', $engine);
250
return $engine || undef;
251
}
252
253
sub get_keys {
254
my ( $self, $ddl, $opts, $is_nullable ) = @_;
255
my $engine = $self->get_engine($ddl);
256
my $keys = {};
257
my $clustered_key = undef;
258
259
KEY:
260
foreach my $key ( $ddl =~ m/^ ((?:[A-Z]+ )?KEY .*)$/gm ) {
261
262
next KEY if $key =~ m/FOREIGN/;
263
264
my $key_ddl = $key;
265
MKDEBUG && _d('Parsed key:', $key_ddl);
266
267
if ( $engine !~ m/MEMORY|HEAP/ ) {
268
$key =~ s/USING HASH/USING BTREE/;
269
}
270
271
my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/;
272
my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/;
273
$type = $type || $special || 'BTREE';
274
if ( $opts->{mysql_version} && $opts->{mysql_version} lt '004001000'
275
&& $engine =~ m/HEAP|MEMORY/i )
73
my ( $self, $dsn, $prev, $defaults ) = @_;
74
if ( !$dsn ) {
75
PTDEBUG && _d('No DSN to parse');
76
return;
77
}
78
PTDEBUG && _d('Parsing', $dsn);
79
$prev ||= {};
80
$defaults ||= {};
81
my %given_props;
82
my %final_props;
83
my $opts = $self->{opts};
84
85
foreach my $dsn_part ( split(/,/, $dsn) ) {
86
if ( my ($prop_key, $prop_val) = $dsn_part =~ m/^(.)=(.*)$/ ) {
87
$given_props{$prop_key} = $prop_val;
88
}
89
else {
90
PTDEBUG && _d('Interpreting', $dsn_part, 'as h=', $dsn_part);
91
$given_props{h} = $dsn_part;
92
}
93
}
94
95
foreach my $key ( keys %$opts ) {
96
PTDEBUG && _d('Finding value for', $key);
97
$final_props{$key} = $given_props{$key};
98
if ( !defined $final_props{$key}
99
&& defined $prev->{$key} && $opts->{$key}->{copy} )
276
100
{
277
$type = 'HASH'; # MySQL pre-4.1 supports only HASH indexes on HEAP
278
}
279
280
my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/;
281
my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0;
282
my @cols;
283
my @col_prefixes;
284
foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) {
285
my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/;
286
push @cols, $name;
287
push @col_prefixes, $prefix;
288
}
289
$name =~ s/`//g;
290
291
MKDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols));
292
293
$keys->{$name} = {
294
name => $name,
295
type => $type,
296
colnames => $cols,
297
cols => \@cols,
298
col_prefixes => \@col_prefixes,
299
is_unique => $unique,
300
is_nullable => scalar(grep { $is_nullable->{$_} } @cols),
301
is_col => { map { $_ => 1 } @cols },
302
ddl => $key_ddl,
303
};
304
305
if ( $engine =~ m/InnoDB/i && !$clustered_key ) {
306
my $this_key = $keys->{$name};
307
if ( $this_key->{name} eq 'PRIMARY' ) {
308
$clustered_key = 'PRIMARY';
309
}
310
elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) {
311
$clustered_key = $this_key->{name};
312
}
313
MKDEBUG && $clustered_key && _d('This key is the clustered key');
314
}
315
}
316
317
return $keys, $clustered_key;
318
}
319
320
sub get_fks {
321
my ( $self, $ddl, $opts ) = @_;
322
my $q = $self->{Quoter};
323
my $fks = {};
324
325
foreach my $fk (
326
$ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg )
327
{
328
my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/;
329
my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/;
330
my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/;
331
332
my ($db, $tbl) = $q->split_unquote($parent, $opts->{database});
333
my %parent_tbl = (tbl => $tbl);
334
$parent_tbl{db} = $db if $db;
335
336
if ( $parent !~ m/\./ && $opts->{database} ) {
337
$parent = $q->quote($opts->{database}) . ".$parent";
338
}
339
340
$fks->{$name} = {
341
name => $name,
342
colnames => $cols,
343
cols => [ map { s/[ `]+//g; $_; } split(',', $cols) ],
344
parent_tbl => \%parent_tbl,
345
parent_tblname => $parent,
346
parent_cols => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ],
347
parent_colnames=> $parent_cols,
348
ddl => $fk,
349
};
350
}
351
352
return $fks;
353
}
354
355
sub remove_auto_increment {
356
my ( $self, $ddl ) = @_;
357
$ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m;
358
return $ddl;
359
}
360
361
sub remove_secondary_indexes {
362
my ( $self, $ddl ) = @_;
363
my $sec_indexes_ddl;
364
my $tbl_struct = $self->parse($ddl);
365
366
if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) {
367
my $clustered_key = $tbl_struct->{clustered_key};
368
$clustered_key ||= '';
369
370
my @sec_indexes = map {
371
my $key_def = $_->{ddl};
372
$key_def =~ s/([\(\)])/\\$1/g;
373
$ddl =~ s/\s+$key_def//i;
374
375
my $key_ddl = "ADD $_->{ddl}";
376
$key_ddl .= ',' unless $key_ddl =~ m/,$/;
377
$key_ddl;
378
}
379
grep { $_->{name} ne $clustered_key }
380
values %{$tbl_struct->{keys}};
381
MKDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes));
382
383
if ( @sec_indexes ) {
384
$sec_indexes_ddl = join(' ', @sec_indexes);
385
$sec_indexes_ddl =~ s/,$//;
386
}
387
388
$ddl =~ s/,(\n\) )/$1/s;
101
$final_props{$key} = $prev->{$key};
102
PTDEBUG && _d('Copying value for', $key, 'from previous DSN');
103
}
104
if ( !defined $final_props{$key} ) {
105
$final_props{$key} = $defaults->{$key};
106
PTDEBUG && _d('Copying value for', $key, 'from defaults');
107
}
108
}
109
110
foreach my $key ( keys %given_props ) {
111
die "Unknown DSN option '$key' in '$dsn'. For more details, "
112
. "please use the --help option, or try 'perldoc $PROGRAM_NAME' "
113
. "for complete documentation."
114
unless exists $opts->{$key};
115
}
116
if ( (my $required = $self->prop('required')) ) {
117
foreach my $key ( keys %$required ) {
118
die "Missing required DSN option '$key' in '$dsn'. For more details, "
119
. "please use the --help option, or try 'perldoc $PROGRAM_NAME' "
120
. "for complete documentation."
121
unless $final_props{$key};
122
}
123
}
124
125
return \%final_props;
126
}
127
128
sub parse_options {
129
my ( $self, $o ) = @_;
130
die 'I need an OptionParser object' unless ref $o eq 'OptionParser';
131
my $dsn_string
132
= join(',',
133
map { "$_=".$o->get($_); }
134
grep { $o->has($_) && $o->get($_) }
135
keys %{$self->{opts}}
136
);
137
PTDEBUG && _d('DSN string made from options:', $dsn_string);
138
return $self->parse($dsn_string);
139
}
140
141
sub as_string {
142
my ( $self, $dsn, $props ) = @_;
143
return $dsn unless ref $dsn;
144
my @keys = $props ? @$props : sort keys %$dsn;
145
return join(',',
146
map { "$_=" . ($_ eq 'p' ? '...' : $dsn->{$_}) }
147
grep {
148
exists $self->{opts}->{$_}
149
&& exists $dsn->{$_}
150
&& defined $dsn->{$_}
151
} @keys);
152
}
153
154
sub usage {
155
my ( $self ) = @_;
156
my $usage
157
= "DSN syntax is key=value[,key=value...] Allowable DSN keys:\n\n"
158
. " KEY COPY MEANING\n"
159
. " === ==== =============================================\n";
160
my %opts = %{$self->{opts}};
161
foreach my $key ( sort keys %opts ) {
162
$usage .= " $key "
163
. ($opts{$key}->{copy} ? 'yes ' : 'no ')
164
. ($opts{$key}->{desc} || '[No description]')
165
. "\n";
166
}
167
$usage .= "\n If the DSN is a bareword, the word is treated as the 'h' key.\n";
168
return $usage;
169
}
170
171
sub get_cxn_params {
172
my ( $self, $info ) = @_;
173
my $dsn;
174
my %opts = %{$self->{opts}};
175
my $driver = $self->prop('dbidriver') || '';
176
if ( $driver eq 'Pg' ) {
177
$dsn = 'DBI:Pg:dbname=' . ( $info->{D} || '' ) . ';'
178
. join(';', map { "$opts{$_}->{dsn}=$info->{$_}" }
179
grep { defined $info->{$_} }
180
qw(h P));
389
181
}
390
182
else {
391
MKDEBUG && _d('Not removing secondary indexes from',
392
$tbl_struct->{engine}, 'table');
393
}
394
395
return $ddl, $sec_indexes_ddl, $tbl_struct;
396
}
397
398
sub _d {
399
my ($package, undef, $line) = caller 0;
400
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
401
map { defined $_ ? $_ : 'undef' }
402
@_;
403
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
404
}
405
406
1;
407
}
408
# ###########################################################################
409
# End TableParser package
410
# ###########################################################################
411
412
# ###########################################################################
413
# TableChecksum package
414
# This package is a copy without comments from the original. The original
415
# with comments and its test file can be found in the Bazaar repository at,
416
# lib/TableChecksum.pm
417
# t/lib/TableChecksum.t
418
# See https://launchpad.net/percona-toolkit for more information.
419
# ###########################################################################
420
{
421
package TableChecksum;
422
423
use strict;
424
use warnings FATAL => 'all';
425
use English qw(-no_match_vars);
426
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
427
428
use List::Util qw(max);
429
430
our %ALGOS = (
431
CHECKSUM => { pref => 0, hash => 0 },
432
BIT_XOR => { pref => 2, hash => 1 },
433
ACCUM => { pref => 3, hash => 1 },
434
);
435
436
sub new {
437
my ( $class, %args ) = @_;
438
foreach my $arg ( qw(Quoter VersionParser) ) {
439
die "I need a $arg argument" unless defined $args{$arg};
440
}
441
my $self = { %args };
442
return bless $self, $class;
443
}
444
445
sub crc32 {
446
my ( $self, $string ) = @_;
447
my $poly = 0xEDB88320;
448
my $crc = 0xFFFFFFFF;
449
foreach my $char ( split(//, $string) ) {
450
my $comp = ($crc ^ ord($char)) & 0xFF;
451
for ( 1 .. 8 ) {
452
$comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1;
453
}
454
$crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp;
455
}
456
return $crc ^ 0xFFFFFFFF;
457
}
458
459
sub get_crc_wid {
460
my ( $self, $dbh, $func ) = @_;
461
my $crc_wid = 16;
462
if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
463
eval {
464
my ($val) = $dbh->selectrow_array("SELECT $func('a')");
465
$crc_wid = max(16, length($val));
466
};
467
}
468
return $crc_wid;
469
}
470
471
sub get_crc_type {
472
my ( $self, $dbh, $func ) = @_;
473
my $type = '';
474
my $length = 0;
475
my $sql = "SELECT $func('a')";
476
my $sth = $dbh->prepare($sql);
477
eval {
478
$sth->execute();
479
$type = $sth->{mysql_type_name}->[0];
480
$length = $sth->{mysql_length}->[0];
481
MKDEBUG && _d($sql, $type, $length);
482
if ( $type eq 'bigint' && $length < 20 ) {
483
$type = 'int';
484
}
183
$dsn = 'DBI:mysql:' . ( $info->{D} || '' ) . ';'
184
. join(';', map { "$opts{$_}->{dsn}=$info->{$_}" }
185
grep { defined $info->{$_} }
186
qw(F h P S A))
187
. ';mysql_read_default_group=client';
188
}
189
PTDEBUG && _d($dsn);
190
return ($dsn, $info->{u}, $info->{p});
191
}
192
193
sub fill_in_dsn {
194
my ( $self, $dbh, $dsn ) = @_;
195
my $vars = $dbh->selectall_hashref('SHOW VARIABLES', 'Variable_name');
196
my ($user, $db) = $dbh->selectrow_array('SELECT USER(), DATABASE()');
197
$user =~ s/@.*//;
198
$dsn->{h} ||= $vars->{hostname}->{Value};
199
$dsn->{S} ||= $vars->{'socket'}->{Value};
200
$dsn->{P} ||= $vars->{port}->{Value};
201
$dsn->{u} ||= $user;
202
$dsn->{D} ||= $db;
203
}
204
205
sub get_dbh {
206
my ( $self, $cxn_string, $user, $pass, $opts ) = @_;
207
$opts ||= {};
208
my $defaults = {
209
AutoCommit => 0,
210
RaiseError => 1,
211
PrintError => 0,
212
ShowErrorStatement => 1,
213
mysql_enable_utf8 => ($cxn_string =~ m/charset=utf8/i ? 1 : 0),
485
214
};
486
$sth->finish;
487
MKDEBUG && _d('crc_type:', $type, 'length:', $length);
488
return ($type, $length);
489
}
490
491
sub best_algorithm {
492
my ( $self, %args ) = @_;
493
my ( $alg, $dbh ) = @args{ qw(algorithm dbh) };
494
my $vp = $self->{VersionParser};
495
my @choices = sort { $ALGOS{$a}->{pref} <=> $ALGOS{$b}->{pref} } keys %ALGOS;
496
die "Invalid checksum algorithm $alg"
497
if $alg && !$ALGOS{$alg};
498
499
if (
500
$args{where} || $args{chunk} # CHECKSUM does whole table
501
|| $args{replicate} # CHECKSUM can't do INSERT.. SELECT
502
|| !$vp->version_ge($dbh, '4.1.1')) # CHECKSUM doesn't exist
503
{
504
MKDEBUG && _d('Cannot use CHECKSUM algorithm');
505
@choices = grep { $_ ne 'CHECKSUM' } @choices;
506
}
507
508
if ( !$vp->version_ge($dbh, '4.1.1') ) {
509
MKDEBUG && _d('Cannot use BIT_XOR algorithm because MySQL < 4.1.1');
510
@choices = grep { $_ ne 'BIT_XOR' } @choices;
511
}
512
513
if ( $alg && grep { $_ eq $alg } @choices ) {
514
MKDEBUG && _d('User requested', $alg, 'algorithm');
515
return $alg;
516
}
517
518
if ( $args{count} && grep { $_ ne 'CHECKSUM' } @choices ) {
519
MKDEBUG && _d('Not using CHECKSUM algorithm because COUNT desired');
520
@choices = grep { $_ ne 'CHECKSUM' } @choices;
521
}
522
523
MKDEBUG && _d('Algorithms, in order:', @choices);
524
return $choices[0];
525
}
526
527
sub is_hash_algorithm {
528
my ( $self, $algorithm ) = @_;
529
return $ALGOS{$algorithm} && $ALGOS{$algorithm}->{hash};
530
}
531
532
sub choose_hash_func {
533
my ( $self, %args ) = @_;
534
my @funcs = qw(CRC32 FNV1A_64 FNV_64 MD5 SHA1);
535
if ( $args{function} ) {
536
unshift @funcs, $args{function};
537
}
538
my ($result, $error);
539
do {
540
my $func;
215
@{$defaults}{ keys %$opts } = values %$opts;
216
217
if ( $opts->{mysql_use_result} ) {
218
$defaults->{mysql_use_result} = 1;
219
}
220
221
if ( !$have_dbi ) {
222
die "Cannot connect to MySQL because the Perl DBI module is not "
223
. "installed or not found. Run 'perl -MDBI' to see the directories "
224
. "that Perl searches for DBI. If DBI is not installed, try:\n"
225
. " Debian/Ubuntu apt-get install libdbi-perl\n"
226
. " RHEL/CentOS yum install perl-DBI\n"
227
. " OpenSolaris pgk install pkg:/SUNWpmdbi\n";
228
229
}
230
231
my $dbh;
232
my $tries = 2;
233
while ( !$dbh && $tries-- ) {
234
PTDEBUG && _d($cxn_string, ' ', $user, ' ', $pass,
235
join(', ', map { "$_=>$defaults->{$_}" } keys %$defaults ));
236
541
237
eval {
542
$func = shift(@funcs);
543
my $sql = "SELECT $func('test-string')";
544
MKDEBUG && _d($sql);
545
$args{dbh}->do($sql);
546
$result = $func;
238
$dbh = DBI->connect($cxn_string, $user, $pass, $defaults);
239
240
if ( $cxn_string =~ m/mysql/i ) {
241
my $sql;
242
243
$sql = 'SELECT @@SQL_MODE';
244
PTDEBUG && _d($dbh, $sql);
245
my ($sql_mode) = $dbh->selectrow_array($sql);
246
247
$sql = 'SET @@SQL_QUOTE_SHOW_CREATE = 1'
248
. '/*!40101, @@SQL_MODE=\'NO_AUTO_VALUE_ON_ZERO'
249
. ($sql_mode ? ",$sql_mode" : '')
250
. '\'*/';
251
PTDEBUG && _d($dbh, $sql);
252
$dbh->do($sql);
253
254
if ( my ($charset) = $cxn_string =~ m/charset=(\w+)/ ) {
255
$sql = "/*!40101 SET NAMES $charset*/";
256
PTDEBUG && _d($dbh, ':', $sql);
257
$dbh->do($sql);
258
PTDEBUG && _d('Enabling charset for STDOUT');
259
if ( $charset eq 'utf8' ) {
260
binmode(STDOUT, ':utf8')
261
or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR";
262
}
263
else {
264
binmode(STDOUT) or die "Can't binmode(STDOUT): $OS_ERROR";
265
}
266
}
267
268
if ( $self->prop('set-vars') ) {
269
$sql = "SET " . $self->prop('set-vars');
270
PTDEBUG && _d($dbh, ':', $sql);
271
$dbh->do($sql);
272
}
273
}
547
274
};
548
if ( $EVAL_ERROR && $EVAL_ERROR =~ m/failed: (.*?) at \S+ line/ ) {
549
$error .= qq{$func cannot be used because "$1"\n};
550
MKDEBUG && _d($func, 'cannot be used because', $1);
551
}
552
} while ( @funcs && !$result );
553
554
die $error unless $result;
555
MKDEBUG && _d('Chosen hash func:', $result);
556
return $result;
557
}
558
559
sub optimize_xor {
560
my ( $self, %args ) = @_;
561
my ($dbh, $func) = @args{qw(dbh function)};
562
563
die "$func never needs the BIT_XOR optimization"
564
if $func =~ m/^(?:FNV1A_64|FNV_64|CRC32)$/i;
565
566
my $opt_slice = 0;
567
my $unsliced = uc $dbh->selectall_arrayref("SELECT $func('a')")->[0]->[0];
568
my $sliced = '';
569
my $start = 1;
570
my $crc_wid = length($unsliced) < 16 ? 16 : length($unsliced);
571
572
do { # Try different positions till sliced result equals non-sliced.
573
MKDEBUG && _d('Trying slice', $opt_slice);
574
$dbh->do('SET @crc := "", @cnt := 0');
575
my $slices = $self->make_xor_slices(
576
query => "\@crc := $func('a')",
577
crc_wid => $crc_wid,
578
opt_slice => $opt_slice,
579
);
580
581
my $sql = "SELECT CONCAT($slices) AS TEST FROM (SELECT NULL) AS x";
582
$sliced = ($dbh->selectrow_array($sql))[0];
583
if ( $sliced ne $unsliced ) {
584
MKDEBUG && _d('Slice', $opt_slice, 'does not work');
585
$start += 16;
586
++$opt_slice;
587
}
588
} while ( $start < $crc_wid && $sliced ne $unsliced );
589
590
if ( $sliced eq $unsliced ) {
591
MKDEBUG && _d('Slice', $opt_slice, 'works');
592
return $opt_slice;
593
}
594
else {
595
MKDEBUG && _d('No slice works');
596
return undef;
597
}
598
}
599
600
sub make_xor_slices {
601
my ( $self, %args ) = @_;
602
foreach my $arg ( qw(query crc_wid) ) {
603
die "I need a $arg argument" unless defined $args{$arg};
604
}
605
my ( $query, $crc_wid, $opt_slice ) = @args{qw(query crc_wid opt_slice)};
606
607
my @slices;
608
for ( my $start = 1; $start <= $crc_wid; $start += 16 ) {
609
my $len = $crc_wid - $start + 1;
610
if ( $len > 16 ) {
611
$len = 16;
612
}
613
push @slices,
614
"LPAD(CONV(BIT_XOR("
615
. "CAST(CONV(SUBSTRING(\@crc, $start, $len), 16, 10) AS UNSIGNED))"
616
. ", 10, 16), $len, '0')";
617
}
618
619
if ( defined $opt_slice && $opt_slice < @slices ) {
620
$slices[$opt_slice] =~ s/\@crc/\@crc := $query/;
621
}
622
else {
623
map { s/\@crc/$query/ } @slices;
624
}
625
626
return join(', ', @slices);
627
}
628
629
sub make_row_checksum {
630
my ( $self, %args ) = @_;
631
my ( $tbl_struct, $func ) = @args{ qw(tbl_struct function) };
632
my $q = $self->{Quoter};
633
634
my $sep = $args{sep} || '#';
635
$sep =~ s/'//g;
636
$sep ||= '#';
637
638
my $ignorecols = $args{ignorecols} || {};
639
640
my %cols = map { lc($_) => 1 }
641
grep { !exists $ignorecols->{$_} }
642
($args{cols} ? @{$args{cols}} : @{$tbl_struct->{cols}});
643
my %seen;
644
my @cols =
645
map {
646
my $type = $tbl_struct->{type_for}->{$_};
647
my $result = $q->quote($_);
648
if ( $type eq 'timestamp' ) {
649
$result .= ' + 0';
650
}
651
elsif ( $args{float_precision} && $type =~ m/float|double/ ) {
652
$result = "ROUND($result, $args{float_precision})";
653
}
654
elsif ( $args{trim} && $type =~ m/varchar/ ) {
655
$result = "TRIM($result)";
656
}
657
$result;
658
}
659
grep {
660
$cols{$_} && !$seen{$_}++
661
}
662
@{$tbl_struct->{cols}};
663
664
my $query;
665
if ( !$args{no_cols} ) {
666
$query = join(', ',
667
map {
668
my $col = $_;
669
if ( $col =~ m/\+ 0/ ) {
670
my ($real_col) = /^(\S+)/;
671
$col .= " AS $real_col";
672
}
673
elsif ( $col =~ m/TRIM/ ) {
674
my ($real_col) = m/TRIM\(([^\)]+)\)/;
675
$col .= " AS $real_col";
676
}
677
$col;
678
} @cols)
679
. ', ';
680
}
681
682
if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
683
my @nulls = grep { $cols{$_} } @{$tbl_struct->{null_cols}};
684
if ( @nulls ) {
685
my $bitmap = "CONCAT("
686
. join(', ', map { 'ISNULL(' . $q->quote($_) . ')' } @nulls)
687
. ")";
688
push @cols, $bitmap;
689
}
690
691
$query .= @cols > 1
692
? "$func(CONCAT_WS('$sep', " . join(', ', @cols) . '))'
693
: "$func($cols[0])";
694
}
695
else {
696
my $fnv_func = uc $func;
697
$query .= "$fnv_func(" . join(', ', @cols) . ')';
698
}
699
700
return $query;
701
}
702
703
sub make_checksum_query {
704
my ( $self, %args ) = @_;
705
my @required_args = qw(db tbl tbl_struct algorithm crc_wid crc_type);
706
foreach my $arg( @required_args ) {
707
die "I need a $arg argument" unless $args{$arg};
708
}
709
my ( $db, $tbl, $tbl_struct, $algorithm,
710
$crc_wid, $crc_type) = @args{@required_args};
711
my $func = $args{function};
712
my $q = $self->{Quoter};
713
my $result;
714
715
die "Invalid or missing checksum algorithm"
716
unless $algorithm && $ALGOS{$algorithm};
717
718
if ( $algorithm eq 'CHECKSUM' ) {
719
return "CHECKSUM TABLE " . $q->quote($db, $tbl);
720
}
721
722
my $expr = $self->make_row_checksum(%args, no_cols=>1);
723
724
if ( $algorithm eq 'BIT_XOR' ) {
725
if ( $crc_type =~ m/int$/ ) {
726
$result = "COALESCE(LOWER(CONV(BIT_XOR(CAST($expr AS UNSIGNED)), 10, 16)), 0) AS crc ";
727
}
728
else {
729
my $slices = $self->make_xor_slices( query => $expr, %args );
730
$result = "COALESCE(LOWER(CONCAT($slices)), 0) AS crc ";
731
}
732
}
733
else {
734
if ( $crc_type =~ m/int$/ ) {
735
$result = "COALESCE(RIGHT(MAX("
736
. "\@crc := CONCAT(LPAD(\@cnt := \@cnt + 1, 16, '0'), "
737
. "CONV(CAST($func(CONCAT(\@crc, $expr)) AS UNSIGNED), 10, 16))"
738
. "), $crc_wid), 0) AS crc ";
739
}
740
else {
741
$result = "COALESCE(RIGHT(MAX("
742
. "\@crc := CONCAT(LPAD(\@cnt := \@cnt + 1, 16, '0'), "
743
. "$func(CONCAT(\@crc, $expr)))"
744
. "), $crc_wid), 0) AS crc ";
745
}
746
}
747
if ( $args{replicate} ) {
748
$result = "REPLACE /*PROGRESS_COMMENT*/ INTO $args{replicate} "
749
. "(db, tbl, chunk, boundaries, this_cnt, this_crc) "
750
. "SELECT ?, ?, /*CHUNK_NUM*/ ?, COUNT(*) AS cnt, $result";
751
}
752
else {
753
$result = "SELECT "
754
. ($args{buffer} ? 'SQL_BUFFER_RESULT ' : '')
755
. "/*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, $result";
756
}
757
return $result . "FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/";
758
}
759
760
sub find_replication_differences {
761
my ( $self, $dbh, $table ) = @_;
762
763
(my $sql = <<" EOF") =~ s/\s+/ /gm;
764
SELECT db, tbl, chunk, boundaries,
765
COALESCE(this_cnt-master_cnt, 0) AS cnt_diff,
766
COALESCE(
767
this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc),
768
0
769
) AS crc_diff,
770
this_cnt, master_cnt, this_crc, master_crc
771
FROM $table
772
WHERE master_cnt <> this_cnt OR master_crc <> this_crc
773
OR ISNULL(master_crc) <> ISNULL(this_crc)
774
EOF
775
776
MKDEBUG && _d($sql);
777
my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} });
778
return @$diffs;
275
if ( !$dbh && $EVAL_ERROR ) {
276
PTDEBUG && _d($EVAL_ERROR);
277
if ( $EVAL_ERROR =~ m/not a compiled character set|character set utf8/ ) {
278
PTDEBUG && _d('Going to try again without utf8 support');
279
delete $defaults->{mysql_enable_utf8};
280
}
281
elsif ( $EVAL_ERROR =~ m/locate DBD\/mysql/i ) {
282
die "Cannot connect to MySQL because the Perl DBD::mysql module is "
283
. "not installed or not found. Run 'perl -MDBD::mysql' to see "
284
. "the directories that Perl searches for DBD::mysql. If "
285
. "DBD::mysql is not installed, try:\n"
286
. " Debian/Ubuntu apt-get install libdbd-mysql-perl\n"
287
. " RHEL/CentOS yum install perl-DBD-MySQL\n"
288
. " OpenSolaris pgk install pkg:/SUNWapu13dbd-mysql\n";
289
}
290
if ( !$tries ) {
291
die $EVAL_ERROR;
292
}
293
}
294
}
295
296
PTDEBUG && _d('DBH info: ',
297
$dbh,
298
Dumper($dbh->selectrow_hashref(
299
'SELECT DATABASE(), CONNECTION_ID(), VERSION()/*!50038 , @@hostname*/')),
300
'Connection info:', $dbh->{mysql_hostinfo},
301
'Character set info:', Dumper($dbh->selectall_arrayref(
302
'SHOW VARIABLES LIKE "character_set%"', { Slice => {}})),
303
'$DBD::mysql::VERSION:', $DBD::mysql::VERSION,
304
'$DBI::VERSION:', $DBI::VERSION,
305
);
306
307
return $dbh;
308
}
309
310
sub get_hostname {
311
my ( $self, $dbh ) = @_;
312
if ( my ($host) = ($dbh->{mysql_hostinfo} || '') =~ m/^(\w+) via/ ) {
313
return $host;
314
}
315
my ( $hostname, $one ) = $dbh->selectrow_array(
316
'SELECT /*!50038 @@hostname, */ 1');
317
return $hostname;
318
}
319
320
sub disconnect {
321
my ( $self, $dbh ) = @_;
322
PTDEBUG && $self->print_active_handles($dbh);
323
$dbh->disconnect;
324
}
325
326
sub print_active_handles {
327
my ( $self, $thing, $level ) = @_;
328
$level ||= 0;
329
printf("# Active %sh: %s %s %s\n", ($thing->{Type} || 'undef'), "\t" x $level,
330
$thing, (($thing->{Type} || '') eq 'st' ? $thing->{Statement} || '' : ''))
331
or die "Cannot print: $OS_ERROR";
332
foreach my $handle ( grep {defined} @{ $thing->{ChildHandles} } ) {
333
$self->print_active_handles( $handle, $level + 1 );
334
}
335
}
336
337
sub copy {
338
my ( $self, $dsn_1, $dsn_2, %args ) = @_;
339
die 'I need a dsn_1 argument' unless $dsn_1;
340
die 'I need a dsn_2 argument' unless $dsn_2;
341
my %new_dsn = map {
342
my $key = $_;
343
my $val;
344
if ( $args{overwrite} ) {
345
$val = defined $dsn_1->{$key} ? $dsn_1->{$key} : $dsn_2->{$key};
346
}
347
else {
348
$val = defined $dsn_2->{$key} ? $dsn_2->{$key} : $dsn_1->{$key};
349
}
350
$key => $val;
351
} keys %{$self->{opts}};
352
return \%new_dsn;
779
353
}
780
354
781
355
sub _d {
789
363
1;
790
364
}
791
365
# ###########################################################################
792
# End TableChecksum package
366
# End DSNParser package
793
367
# ###########################################################################
794
368
795
369
# ###########################################################################
806
380
use strict;
807
381
use warnings FATAL => 'all';
808
382
use English qw(-no_match_vars);
809
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
383
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
810
384
811
385
use List::Util qw(max);
812
386
use Getopt::Long;
890
464
my $contents = do { local $/ = undef; <$fh> };
891
465
close $fh;
892
466
if ( $contents =~ m/^=head1 DSN OPTIONS/m ) {
893
MKDEBUG && _d('Parsing DSN OPTIONS');
467
PTDEBUG && _d('Parsing DSN OPTIONS');
894
468
my $dsn_attribs = {
895
469
dsn => 1,
896
470
copy => 1,
932
506
$self->{DSNParser} = DSNParser->new(opts => \@dsn_opts);
933
507
}
934
508
935
if ( $contents =~ m/^(Percona Toolkit v.+)$/m ) {
509
if ( $contents =~ m/^=head1 VERSION\n\n^(.+)$/m ) {
936
510
$self->{version} = $1;
937
MKDEBUG && _d($self->{version});
511
PTDEBUG && _d($self->{version});
938
512
}
939
513
940
514
return;
971
545
chomp $para;
972
546
$para =~ s/\s+/ /g;
973
547
$para =~ s/$POD_link_re/$1/go;
974
MKDEBUG && _d('Option rule:', $para);
548
PTDEBUG && _d('Option rule:', $para);
975
549
push @rules, $para;
976
550
}
977
551
980
554
do {
981
555
if ( my ($option) = $para =~ m/^=item $self->{item}/ ) {
982
556
chomp $para;
983
MKDEBUG && _d($para);
557
PTDEBUG && _d($para);
984
558
my %attribs;
985
559
986
560
$para = <$fh>; # read next paragraph, possibly attributes
999
573
$para = <$fh>; # read next paragraph, probably short help desc
1000
574
}
1001
575
else {
1002
MKDEBUG && _d('Option has no attributes');
576
PTDEBUG && _d('Option has no attributes');
1003
577
}
1004
578
1005
579
$para =~ s/\s+\Z//g;
1007
581
$para =~ s/$POD_link_re/$1/go;
1008
582
1009
583
$para =~ s/\.(?:\n.*| [A-Z].*|\Z)//s;
1010
MKDEBUG && _d('Short help:', $para);
584
PTDEBUG && _d('Short help:', $para);
1011
585
1012
586
die "No description after option spec $option" if $para =~ m/^=item/;
1013
587
1045
619
1046
620
foreach my $opt ( @specs ) {
1047
621
if ( ref $opt ) { # It's an option spec, not a rule.
1048
MKDEBUG && _d('Parsing opt spec:',
622
PTDEBUG && _d('Parsing opt spec:',
1049
623
map { ($_, '=>', $opt->{$_}) } keys %$opt);
1050
624
1051
625
my ( $long, $short ) = $opt->{spec} =~ m/^([\w-]+)(?:\|([^!+=]*))?/;
1058
632
$self->{opts}->{$long} = $opt;
1059
633
1060
634
if ( length $long == 1 ) {
1061
MKDEBUG && _d('Long opt', $long, 'looks like short opt');
635
PTDEBUG && _d('Long opt', $long, 'looks like short opt');
1062
636
$self->{short_opts}->{$long} = $long;
1063
637
}
1064
638
1084
658
1085
659
my ( $type ) = $opt->{spec} =~ m/=(.)/;
1086
660
$opt->{type} = $type;
1087
MKDEBUG && _d($long, 'type:', $type);
661
PTDEBUG && _d($long, 'type:', $type);
1088
662
1089
663
1090
664
$opt->{spec} =~ s/=./=s/ if ( $type && $type =~ m/[HhAadzm]/ );
1091
665
1092
666
if ( (my ($def) = $opt->{desc} =~ m/default\b(?: ([^)]+))?/) ) {
1093
667
$self->{defaults}->{$long} = defined $def ? $def : 1;
1094
MKDEBUG && _d($long, 'default:', $def);
668
PTDEBUG && _d($long, 'default:', $def);
1095
669
}
1096
670
1097
671
if ( $long eq 'config' ) {
1100
674
1101
675
if ( (my ($dis) = $opt->{desc} =~ m/(disables .*)/) ) {
1102
676
$disables{$long} = $dis;
1103
MKDEBUG && _d('Deferring check of disables rule for', $opt, $dis);
677
PTDEBUG && _d('Deferring check of disables rule for', $opt, $dis);
1104
678
}
1105
679
1106
680
$self->{opts}->{$long} = $opt;
1107
681
}
1108
682
else { # It's an option rule, not a spec.
1109
MKDEBUG && _d('Parsing rule:', $opt);
683
PTDEBUG && _d('Parsing rule:', $opt);
1110
684
push @{$self->{rules}}, $opt;
1111
685
my @participants = $self->_get_participants($opt);
1112
686
my $rule_ok = 0;
1114
688
if ( $opt =~ m/mutually exclusive|one and only one/ ) {
1115
689
$rule_ok = 1;
1116
690
push @{$self->{mutex}}, \@participants;
1117
MKDEBUG && _d(@participants, 'are mutually exclusive');
691
PTDEBUG && _d(@participants, 'are mutually exclusive');
1118
692
}
1119
693
if ( $opt =~ m/at least one|one and only one/ ) {
1120
694
$rule_ok = 1;
1121
695
push @{$self->{atleast1}}, \@participants;
1122
MKDEBUG && _d(@participants, 'require at least one');
696
PTDEBUG && _d(@participants, 'require at least one');
1123
697
}
1124
698
if ( $opt =~ m/default to/ ) {
1125
699
$rule_ok = 1;
1126
700
$self->{defaults_to}->{$participants[0]} = $participants[1];
1127
MKDEBUG && _d($participants[0], 'defaults to', $participants[1]);
701
PTDEBUG && _d($participants[0], 'defaults to', $participants[1]);
1128
702
}
1129
703
if ( $opt =~ m/restricted to option groups/ ) {
1130
704
$rule_ok = 1;
1138
712
if( $opt =~ m/accepts additional command-line arguments/ ) {
1139
713
$rule_ok = 1;
1140
714
$self->{strict} = 0;
1141
MKDEBUG && _d("Strict mode disabled by rule");
715
PTDEBUG && _d("Strict mode disabled by rule");
1142
716
}
1143
717
1144
718
die "Unrecognized option rule: $opt" unless $rule_ok;
1148
722
foreach my $long ( keys %disables ) {
1149
723
my @participants = $self->_get_participants($disables{$long});
1150
724
$self->{disables}->{$long} = \@participants;
1151
MKDEBUG && _d('Option', $long, 'disables', @participants);
725
PTDEBUG && _d('Option', $long, 'disables', @participants);
1152
726
}
1153
727
1154
728
return;
1162
736
unless exists $self->{opts}->{$long};
1163
737
push @participants, $long;
1164
738
}
1165
MKDEBUG && _d('Participants for', $str, ':', @participants);
739
PTDEBUG && _d('Participants for', $str, ':', @participants);
1166
740
return @participants;
1167
741
}
1168
742
1185
759
die "Cannot set default for nonexistent option $long"
1186
760
unless exists $self->{opts}->{$long};
1187
761
$self->{defaults}->{$long} = $defaults{$long};
1188
MKDEBUG && _d('Default val for', $long, ':', $defaults{$long});
762
PTDEBUG && _d('Default val for', $long, ':', $defaults{$long});
1189
763
}
1190
764
return;
1191
765
}
1214
788
$opt->{value} = $val;
1215
789
}
1216
790
$opt->{got} = 1;
1217
MKDEBUG && _d('Got option', $long, '=', $val);
791
PTDEBUG && _d('Got option', $long, '=', $val);
1218
792
}
1219
793
1220
794
sub get_opts {
1245
819
if ( $self->got('config') ) {
1246
820
die $EVAL_ERROR;
1247
821
}
1248
elsif ( MKDEBUG ) {
822
elsif ( PTDEBUG ) {
1249
823
_d($EVAL_ERROR);
1250
824
}
1251
825
}
1312
886
if ( exists $self->{disables}->{$long} ) {
1313
887
my @disable_opts = @{$self->{disables}->{$long}};
1314
888
map { $self->{opts}->{$_}->{value} = undef; } @disable_opts;
1315
MKDEBUG && _d('Unset options', @disable_opts,
889
PTDEBUG && _d('Unset options', @disable_opts,
1316
890
'because', $long,'disables them');
1317
891
}
1318
892
1361
935
delete $long[$i];
1362
936
}
1363
937
else {
1364
MKDEBUG && _d('Temporarily failed to parse', $long);
938
PTDEBUG && _d('Temporarily failed to parse', $long);
1365
939
}
1366
940
}
1367
941
1385
959
my $val = $opt->{value};
1386
960
1387
961
if ( $val && $opt->{type} eq 'm' ) { # type time
1388
MKDEBUG && _d('Parsing option', $opt->{long}, 'as a time value');
962
PTDEBUG && _d('Parsing option', $opt->{long}, 'as a time value');
1389
963
my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/;
1390
964
if ( !$suffix ) {
1391
965
my ( $s ) = $opt->{desc} =~ m/\(suffix (.)\)/;
1392
966
$suffix = $s || 's';
1393
MKDEBUG && _d('No suffix given; using', $suffix, 'for',
967
PTDEBUG && _d('No suffix given; using', $suffix, 'for',
1394
968
$opt->{long}, '(value:', $val, ')');
1395
969
}
1396
970
if ( $suffix =~ m/[smhd]/ ) {
1399
973
: $suffix eq 'h' ? $num * 3600 # Hours
1400
974
: $num * 86400; # Days
1401
975
$opt->{value} = ($prefix || '') . $val;
1402
MKDEBUG && _d('Setting option', $opt->{long}, 'to', $val);
976
PTDEBUG && _d('Setting option', $opt->{long}, 'to', $val);
1403
977
}
1404
978
else {
1405
979
$self->save_error("Invalid time suffix for --$opt->{long}");
1406
980
}
1407
981
}
1408
982
elsif ( $val && $opt->{type} eq 'd' ) { # type DSN
1409
MKDEBUG && _d('Parsing option', $opt->{long}, 'as a DSN');
983
PTDEBUG && _d('Parsing option', $opt->{long}, 'as a DSN');
1410
984
my $prev = {};
1411
985
my $from_key = $self->{defaults_to}->{ $opt->{long} };
1412
986
if ( $from_key ) {
1413
MKDEBUG && _d($opt->{long}, 'DSN copies from', $from_key, 'DSN');
987
PTDEBUG && _d($opt->{long}, 'DSN copies from', $from_key, 'DSN');
1414
988
if ( $self->{opts}->{$from_key}->{parsed} ) {
1415
989
$prev = $self->{opts}->{$from_key}->{value};
1416
990
}
1417
991
else {
1418
MKDEBUG && _d('Cannot parse', $opt->{long}, 'until',
992
PTDEBUG && _d('Cannot parse', $opt->{long}, 'until',
1419
993
$from_key, 'parsed');
1420
994
return;
1421
995
}
1424
998
$opt->{value} = $self->{DSNParser}->parse($val, $prev, $defaults);
1425
999
}
1426
1000
elsif ( $val && $opt->{type} eq 'z' ) { # type size
1427
MKDEBUG && _d('Parsing option', $opt->{long}, 'as a size value');
1001
PTDEBUG && _d('Parsing option', $opt->{long}, 'as a size value');
1428
1002
$self->_parse_size($opt, $val);
1429
1003
}
1430
1004
elsif ( $opt->{type} eq 'H' || (defined $val && $opt->{type} eq 'h') ) {
1434
1008
$opt->{value} = [ split(/(?<!\\),\s*/, ($val || '')) ];
1435
1009
}
1436
1010
else {
1437
MKDEBUG && _d('Nothing to validate for option',
1011
PTDEBUG && _d('Nothing to validate for option',
1438
1012
$opt->{long}, 'type', $opt->{type}, 'value', $val);
1439
1013
}
1440
1014
1508
1082
$file ||= $self->{file} || __FILE__;
1509
1083
1510
1084
if ( !$self->{description} || !$self->{usage} ) {
1511
MKDEBUG && _d("Getting description and usage from SYNOPSIS in", $file);
1085
PTDEBUG && _d("Getting description and usage from SYNOPSIS in", $file);
1512
1086
my %synop = $self->_parse_synopsis($file);
1513
1087
$self->{description} ||= $synop{description};
1514
1088
$self->{usage} ||= $synop{usage};
1515
MKDEBUG && _d("Description:", $self->{description},
1089
PTDEBUG && _d("Description:", $self->{description},
1516
1090
"\nUsage:", $self->{usage});
1517
1091
}
1518
1092
1727
1301
my ( $self, $opt, $val ) = @_;
1728
1302
1729
1303
if ( lc($val || '') eq 'null' ) {
1730
MKDEBUG && _d('NULL size for', $opt->{long});
1304
PTDEBUG && _d('NULL size for', $opt->{long});
1731
1305
$opt->{value} = 'null';
1732
1306
return;
1733
1307
}
1737
1311
if ( defined $num ) {
1738
1312
if ( $factor ) {
1739
1313
$num *= $factor_for{$factor};
1740
MKDEBUG && _d('Setting option', $opt->{y},
1314
PTDEBUG && _d('Setting option', $opt->{y},
1741
1315
'to num', $num, '* factor', $factor);
1742
1316
}
1743
1317
$opt->{value} = ($pre || '') . $num;
1744
1318
}
1745
1319
else {
1746
$self->save_error("Invalid size for --$opt->{long}");
1320
$self->save_error("Invalid size for --$opt->{long}: $val");
1747
1321
}
1748
1322
return;
1749
1323
}
1761
1335
sub _parse_synopsis {
1762
1336
my ( $self, $file ) = @_;
1763
1337
$file ||= $self->{file} || __FILE__;
1764
MKDEBUG && _d("Parsing SYNOPSIS in", $file);
1338
PTDEBUG && _d("Parsing SYNOPSIS in", $file);
1765
1339
1766
1340
local $INPUT_RECORD_SEPARATOR = ''; # read paragraphs
1767
1341
open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
1774
1348
push @synop, $para;
1775
1349
}
1776
1350
close $fh;
1777
MKDEBUG && _d("Raw SYNOPSIS text:", @synop);
1351
PTDEBUG && _d("Raw SYNOPSIS text:", @synop);
1778
1352
my ($usage, $desc) = @synop;
1779
1353
die "The SYNOPSIS section in $file is not formatted properly"
1780
1354
unless $usage && $desc;
1801
1375
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1802
1376
}
1803
1377
1804
if ( MKDEBUG ) {
1378
if ( PTDEBUG ) {
1805
1379
print '# ', $^X, ' ', $], "\n";
1806
1380
if ( my $uname = `uname -a` ) {
1807
1381
$uname =~ s/\s+/ /g;
1818
1392
# ###########################################################################
1819
1393
1820
1394
# ###########################################################################
1821
# DSNParser package
1395
# Cxn package
1822
1396
# This package is a copy without comments from the original. The original
1823
1397
# with comments and its test file can be found in the Bazaar repository at,
1824
# lib/DSNParser.pm
1825
# t/lib/DSNParser.t
1398
# lib/Cxn.pm
1399
# t/lib/Cxn.t
1826
1400
# See https://launchpad.net/percona-toolkit for more information.
1827
1401
# ###########################################################################
1828
1402
{
1829
package DSNParser;
1403
package Cxn;
1830
1404
1831
1405
use strict;
1832
1406
use warnings FATAL => 'all';
1833
1407
use English qw(-no_match_vars);
1834
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
1835
1836
use Data::Dumper;
1837
$Data::Dumper::Indent = 0;
1838
$Data::Dumper::Quotekeys = 0;
1839
1840
eval {
1841
require DBI;
1842
};
1843
my $have_dbi = $EVAL_ERROR ? 0 : 1;
1408
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
1409
1410
use constant PERCONA_TOOLKIT_TEST_USE_DSN_NAMES => $ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} || 0;
1844
1411
1845
1412
sub new {
1846
1413
my ( $class, %args ) = @_;
1847
foreach my $arg ( qw(opts) ) {
1414
my @required_args = qw(DSNParser OptionParser);
1415
foreach my $arg ( @required_args ) {
1848
1416
die "I need a $arg argument" unless $args{$arg};
1849
}
1850
my $self = {
1851
opts => {} # h, P, u, etc. Should come from DSN OPTIONS section in POD.
1852
1417
};
1853
foreach my $opt ( @{$args{opts}} ) {
1854
if ( !$opt->{key} || !$opt->{desc} ) {
1855
die "Invalid DSN option: ", Dumper($opt);
1856
}
1857
MKDEBUG && _d('DSN option:',
1858
join(', ',
1859
map { "$_=" . (defined $opt->{$_} ? ($opt->{$_} || '') : 'undef') }
1860
keys %$opt
1861
)
1862
);
1863
$self->{opts}->{$opt->{key}} = {
1864
dsn => $opt->{dsn},
1865
desc => $opt->{desc},
1866
copy => $opt->{copy} || 0,
1867
};
1868
}
1869
return bless $self, $class;
1870
}
1871
1872
sub prop {
1873
my ( $self, $prop, $value ) = @_;
1874
if ( @_ > 2 ) {
1875
MKDEBUG && _d('Setting', $prop, 'property');
1876
$self->{$prop} = $value;
1877
}
1878
return $self->{$prop};
1879
}
1880
1881
sub parse {
1882
my ( $self, $dsn, $prev, $defaults ) = @_;
1418
my ($dp, $o) = @args{@required_args};
1419
1420
my $dsn_defaults = $dp->parse_options($o);
1421
my $prev_dsn = $args{prev_dsn};
1422
my $dsn = $args{dsn};
1883
1423
if ( !$dsn ) {
1884
MKDEBUG && _d('No DSN to parse');
1885
return;
1886
}
1887
MKDEBUG && _d('Parsing', $dsn);
1888
$prev ||= {};
1889
$defaults ||= {};
1890
my %given_props;
1891
my %final_props;
1892
my $opts = $self->{opts};
1893
1894
foreach my $dsn_part ( split(/,/, $dsn) ) {
1895
if ( my ($prop_key, $prop_val) = $dsn_part =~ m/^(.)=(.*)$/ ) {
1896
$given_props{$prop_key} = $prop_val;
1897
}
1898
else {
1899
MKDEBUG && _d('Interpreting', $dsn_part, 'as h=', $dsn_part);
1900
$given_props{h} = $dsn_part;
1901
}
1902
}
1903
1904
foreach my $key ( keys %$opts ) {
1905
MKDEBUG && _d('Finding value for', $key);
1906
$final_props{$key} = $given_props{$key};
1907
if ( !defined $final_props{$key}
1908
&& defined $prev->{$key} && $opts->{$key}->{copy} )
1909
{
1910
$final_props{$key} = $prev->{$key};
1911
MKDEBUG && _d('Copying value for', $key, 'from previous DSN');
1912
}
1913
if ( !defined $final_props{$key} ) {
1914
$final_props{$key} = $defaults->{$key};
1915
MKDEBUG && _d('Copying value for', $key, 'from defaults');
1916
}
1917
}
1918
1919
foreach my $key ( keys %given_props ) {
1920
die "Unknown DSN option '$key' in '$dsn'. For more details, "
1921
. "please use the --help option, or try 'perldoc $PROGRAM_NAME' "
1922
. "for complete documentation."
1923
unless exists $opts->{$key};
1924
}
1925
if ( (my $required = $self->prop('required')) ) {
1926
foreach my $key ( keys %$required ) {
1927
die "Missing required DSN option '$key' in '$dsn'. For more details, "
1928
. "please use the --help option, or try 'perldoc $PROGRAM_NAME' "
1929
. "for complete documentation."
1930
unless $final_props{$key};
1931
}
1932
}
1933
1934
return \%final_props;
1935
}
1936
1937
sub parse_options {
1938
my ( $self, $o ) = @_;
1939
die 'I need an OptionParser object' unless ref $o eq 'OptionParser';
1940
my $dsn_string
1941
= join(',',
1942
map { "$_=".$o->get($_); }
1943
grep { $o->has($_) && $o->get($_) }
1944
keys %{$self->{opts}}
1945
);
1946
MKDEBUG && _d('DSN string made from options:', $dsn_string);
1947
return $self->parse($dsn_string);
1948
}
1949
1950
sub as_string {
1951
my ( $self, $dsn, $props ) = @_;
1952
return $dsn unless ref $dsn;
1953
my %allowed = $props ? map { $_=>1 } @$props : ();
1954
return join(',',
1955
map { "$_=" . ($_ eq 'p' ? '...' : $dsn->{$_}) }
1956
grep { defined $dsn->{$_} && $self->{opts}->{$_} }
1957
grep { !$props || $allowed{$_} }
1958
sort keys %$dsn );
1959
}
1960
1961
sub usage {
1424
$args{dsn_string} ||= 'h=' . ($dsn_defaults->{h} || 'localhost');
1425
1426
$dsn = $dp->parse(
1427
$args{dsn_string}, $prev_dsn, $dsn_defaults);
1428
}
1429
elsif ( $prev_dsn ) {
1430
$dsn = $dp->copy($prev_dsn, $dsn);
1431
}
1432
1433
my $self = {
1434
dsn => $dsn,
1435
dbh => $args{dbh},
1436
dsn_name => $dp->as_string($dsn, [qw(h P S)]),
1437
hostname => '',
1438
set => $args{set},
1439
dbh_set => 0,
1440
OptionParser => $o,
1441
DSNParser => $dp,
1442
};
1443
1444
return bless $self, $class;
1445
}
1446
1447
sub connect {
1962
1448
my ( $self ) = @_;
1963
my $usage
1964
= "DSN syntax is key=value[,key=value...] Allowable DSN keys:\n\n"
1965
. " KEY COPY MEANING\n"
1966
. " === ==== =============================================\n";
1967
my %opts = %{$self->{opts}};
1968
foreach my $key ( sort keys %opts ) {
1969
$usage .= " $key "
1970
. ($opts{$key}->{copy} ? 'yes ' : 'no ')
1971
. ($opts{$key}->{desc} || '[No description]')
1972
. "\n";
1973
}
1974
$usage .= "\n If the DSN is a bareword, the word is treated as the 'h' key.\n";
1975
return $usage;
1976
}
1977
1978
sub get_cxn_params {
1979
my ( $self, $info ) = @_;
1980
my $dsn;
1981
my %opts = %{$self->{opts}};
1982
my $driver = $self->prop('dbidriver') || '';
1983
if ( $driver eq 'Pg' ) {
1984
$dsn = 'DBI:Pg:dbname=' . ( $info->{D} || '' ) . ';'
1985
. join(';', map { "$opts{$_}->{dsn}=$info->{$_}" }
1986
grep { defined $info->{$_} }
1987
qw(h P));
1988
}
1989
else {
1990
$dsn = 'DBI:mysql:' . ( $info->{D} || '' ) . ';'
1991
. join(';', map { "$opts{$_}->{dsn}=$info->{$_}" }
1992
grep { defined $info->{$_} }
1993
qw(F h P S A))
1994
. ';mysql_read_default_group=client';
1995
}
1996
MKDEBUG && _d($dsn);
1997
return ($dsn, $info->{u}, $info->{p});
1998
}
1999
2000
sub fill_in_dsn {
2001
my ( $self, $dbh, $dsn ) = @_;
2002
my $vars = $dbh->selectall_hashref('SHOW VARIABLES', 'Variable_name');
2003
my ($user, $db) = $dbh->selectrow_array('SELECT USER(), DATABASE()');
2004
$user =~ s/@.*//;
2005
$dsn->{h} ||= $vars->{hostname}->{Value};
2006
$dsn->{S} ||= $vars->{'socket'}->{Value};
2007
$dsn->{P} ||= $vars->{port}->{Value};
2008
$dsn->{u} ||= $user;
2009
$dsn->{D} ||= $db;
2010
}
2011
2012
sub get_dbh {
2013
my ( $self, $cxn_string, $user, $pass, $opts ) = @_;
2014
$opts ||= {};
2015
my $defaults = {
2016
AutoCommit => 0,
2017
RaiseError => 1,
2018
PrintError => 0,
2019
ShowErrorStatement => 1,
2020
mysql_enable_utf8 => ($cxn_string =~ m/charset=utf8/i ? 1 : 0),
2021
};
2022
@{$defaults}{ keys %$opts } = values %$opts;
2023
2024
if ( $opts->{mysql_use_result} ) {
2025
$defaults->{mysql_use_result} = 1;
2026
}
2027
2028
if ( !$have_dbi ) {
2029
die "Cannot connect to MySQL because the Perl DBI module is not "
2030
. "installed or not found. Run 'perl -MDBI' to see the directories "
2031
. "that Perl searches for DBI. If DBI is not installed, try:\n"
2032
. " Debian/Ubuntu apt-get install libdbi-perl\n"
2033
. " RHEL/CentOS yum install perl-DBI\n"
2034
. " OpenSolaris pgk install pkg:/SUNWpmdbi\n";
2035
2036
}
2037
2038
my $dbh;
2039
my $tries = 2;
2040
while ( !$dbh && $tries-- ) {
2041
MKDEBUG && _d($cxn_string, ' ', $user, ' ', $pass,
2042
join(', ', map { "$_=>$defaults->{$_}" } keys %$defaults ));
2043
2044
eval {
2045
$dbh = DBI->connect($cxn_string, $user, $pass, $defaults);
2046
2047
if ( $cxn_string =~ m/mysql/i ) {
2048
my $sql;
2049
2050
$sql = 'SELECT @@SQL_MODE';
2051
MKDEBUG && _d($dbh, $sql);
2052
my ($sql_mode) = $dbh->selectrow_array($sql);
2053
2054
$sql = 'SET @@SQL_QUOTE_SHOW_CREATE = 1'
2055
. '/*!40101, @@SQL_MODE=\'NO_AUTO_VALUE_ON_ZERO'
2056
. ($sql_mode ? ",$sql_mode" : '')
2057
. '\'*/';
2058
MKDEBUG && _d($dbh, $sql);
2059
$dbh->do($sql);
2060
2061
if ( my ($charset) = $cxn_string =~ m/charset=(\w+)/ ) {
2062
$sql = "/*!40101 SET NAMES $charset*/";
2063
MKDEBUG && _d($dbh, ':', $sql);
2064
$dbh->do($sql);
2065
MKDEBUG && _d('Enabling charset for STDOUT');
2066
if ( $charset eq 'utf8' ) {
2067
binmode(STDOUT, ':utf8')
2068
or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR";
2069
}
2070
else {
2071
binmode(STDOUT) or die "Can't binmode(STDOUT): $OS_ERROR";
2072
}
2073
}
2074
2075
if ( $self->prop('set-vars') ) {
2076
$sql = "SET " . $self->prop('set-vars');
2077
MKDEBUG && _d($dbh, ':', $sql);
2078
$dbh->do($sql);
2079
}
2080
}
2081
};
2082
if ( !$dbh && $EVAL_ERROR ) {
2083
MKDEBUG && _d($EVAL_ERROR);
2084
if ( $EVAL_ERROR =~ m/not a compiled character set|character set utf8/ ) {
2085
MKDEBUG && _d('Going to try again without utf8 support');
2086
delete $defaults->{mysql_enable_utf8};
2087
}
2088
elsif ( $EVAL_ERROR =~ m/locate DBD\/mysql/i ) {
2089
die "Cannot connect to MySQL because the Perl DBD::mysql module is "
2090
. "not installed or not found. Run 'perl -MDBD::mysql' to see "
2091
. "the directories that Perl searches for DBD::mysql. If "
2092
. "DBD::mysql is not installed, try:\n"
2093
. " Debian/Ubuntu apt-get install libdbd-mysql-perl\n"
2094
. " RHEL/CentOS yum install perl-DBD-MySQL\n"
2095
. " OpenSolaris pgk install pkg:/SUNWapu13dbd-mysql\n";
2096
}
2097
if ( !$tries ) {
2098
die $EVAL_ERROR;
2099
}
1449
my $dsn = $self->{dsn};
1450
my $dp = $self->{DSNParser};
1451
my $o = $self->{OptionParser};
1452
1453
my $dbh = $self->{dbh};
1454
if ( !$dbh || !$dbh->ping() ) {
1455
if ( $o->get('ask-pass') && !$self->{asked_for_pass} ) {
1456
$dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: ");
1457
$self->{asked_for_pass} = 1;
2100
1458
}
2101
}
2102
2103
MKDEBUG && _d('DBH info: ',
2104
$dbh,
2105
Dumper($dbh->selectrow_hashref(
2106
'SELECT DATABASE(), CONNECTION_ID(), VERSION()/*!50038 , @@hostname*/')),
2107
'Connection info:', $dbh->{mysql_hostinfo},
2108
'Character set info:', Dumper($dbh->selectall_arrayref(
2109
'SHOW VARIABLES LIKE "character_set%"', { Slice => {}})),
2110
'$DBD::mysql::VERSION:', $DBD::mysql::VERSION,
2111
'$DBI::VERSION:', $DBI::VERSION,
2112
);
2113
1459
$dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 });
1460
}
1461
PTDEBUG && _d($dbh, 'Connected dbh to', $self->{name});
1462
1463
return $self->set_dbh($dbh);
1464
}
1465
1466
sub set_dbh {
1467
my ($self, $dbh) = @_;
1468
1469
if ( $self->{dbh} && $self->{dbh} == $dbh && $self->{dbh_set} ) {
1470
PTDEBUG && _d($dbh, 'Already set dbh');
1471
return $dbh;
1472
}
1473
1474
PTDEBUG && _d($dbh, 'Setting dbh');
1475
1476
$dbh->{FetchHashKeyName} = 'NAME_lc';
1477
1478
my $sql = 'SELECT @@hostname, @@server_id';
1479
PTDEBUG && _d($dbh, $sql);
1480
my ($hostname, $server_id) = $dbh->selectrow_array($sql);
1481
PTDEBUG && _d($dbh, 'hostname:', $hostname, $server_id);
1482
if ( $hostname ) {
1483
$self->{hostname} = $hostname;
1484
}
1485
1486
if ( my $set = $self->{set}) {
1487
$set->($dbh);
1488
}
1489
1490
$self->{dbh} = $dbh;
1491
$self->{dbh_set} = 1;
2114
1492
return $dbh;
2115
1493
}
2116
1494
2117
sub get_hostname {
2118
my ( $self, $dbh ) = @_;
2119
if ( my ($host) = ($dbh->{mysql_hostinfo} || '') =~ m/^(\w+) via/ ) {
2120
return $host;
2121
}
2122
my ( $hostname, $one ) = $dbh->selectrow_array(
2123
'SELECT /*!50038 @@hostname, */ 1');
2124
return $hostname;
2125
}
2126
2127
sub disconnect {
2128
my ( $self, $dbh ) = @_;
2129
MKDEBUG && $self->print_active_handles($dbh);
2130
$dbh->disconnect;
2131
}
2132
2133
sub print_active_handles {
2134
my ( $self, $thing, $level ) = @_;
2135
$level ||= 0;
2136
printf("# Active %sh: %s %s %s\n", ($thing->{Type} || 'undef'), "\t" x $level,
2137
$thing, (($thing->{Type} || '') eq 'st' ? $thing->{Statement} || '' : ''))
2138
or die "Cannot print: $OS_ERROR";
2139
foreach my $handle ( grep {defined} @{ $thing->{ChildHandles} } ) {
2140
$self->print_active_handles( $handle, $level + 1 );
2141
}
2142
}
2143
2144
sub copy {
2145
my ( $self, $dsn_1, $dsn_2, %args ) = @_;
2146
die 'I need a dsn_1 argument' unless $dsn_1;
2147
die 'I need a dsn_2 argument' unless $dsn_2;
2148
my %new_dsn = map {
2149
my $key = $_;
2150
my $val;
2151
if ( $args{overwrite} ) {
2152
$val = defined $dsn_1->{$key} ? $dsn_1->{$key} : $dsn_2->{$key};
2153
}
2154
else {
2155
$val = defined $dsn_2->{$key} ? $dsn_2->{$key} : $dsn_1->{$key};
2156
}
2157
$key => $val;
2158
} keys %{$self->{opts}};
2159
return \%new_dsn;
2160
}
2161
2162
sub _d {
2163
my ($package, undef, $line) = caller 0;
2164
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
2165
map { defined $_ ? $_ : 'undef' }
2166
@_;
2167
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
2168
}
2169
2170
1;
2171
}
2172
# ###########################################################################
2173
# End DSNParser package
2174
# ###########################################################################
2175
2176
# ###########################################################################
2177
# VersionParser package
2178
# This package is a copy without comments from the original. The original
2179
# with comments and its test file can be found in the Bazaar repository at,
2180
# lib/VersionParser.pm
2181
# t/lib/VersionParser.t
2182
# See https://launchpad.net/percona-toolkit for more information.
2183
# ###########################################################################
2184
{
2185
package VersionParser;
2186
2187
use strict;
2188
use warnings FATAL => 'all';
2189
use English qw(-no_match_vars);
2190
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
2191
2192
sub new {
2193
my ( $class ) = @_;
2194
bless {}, $class;
2195
}
2196
2197
sub parse {
2198
my ( $self, $str ) = @_;
2199
my $result = sprintf('%03d%03d%03d', $str =~ m/(\d+)/g);
2200
MKDEBUG && _d($str, 'parses to', $result);
2201
return $result;
2202
}
2203
2204
sub version_ge {
2205
my ( $self, $dbh, $target ) = @_;
2206
if ( !$self->{$dbh} ) {
2207
$self->{$dbh} = $self->parse(
2208
$dbh->selectrow_array('SELECT VERSION()'));
2209
}
2210
my $result = $self->{$dbh} ge $self->parse($target) ? 1 : 0;
2211
MKDEBUG && _d($self->{$dbh}, 'ge', $target, ':', $result);
2212
return $result;
2213
}
2214
2215
sub innodb_version {
2216
my ( $self, $dbh ) = @_;
2217
return unless $dbh;
2218
my $innodb_version = "NO";
2219
2220
my ($innodb) =
2221
grep { $_->{engine} =~ m/InnoDB/i }
2222
map {
2223
my %hash;
2224
@hash{ map { lc $_ } keys %$_ } = values %$_;
2225
\%hash;
2226
}
2227
@{ $dbh->selectall_arrayref("SHOW ENGINES", {Slice=>{}}) };
2228
if ( $innodb ) {
2229
MKDEBUG && _d("InnoDB support:", $innodb->{support});
2230
if ( $innodb->{support} =~ m/YES|DEFAULT/i ) {
2231
my $vars = $dbh->selectrow_hashref(
2232
"SHOW VARIABLES LIKE 'innodb_version'");
2233
$innodb_version = !$vars ? "BUILTIN"
2234
: ($vars->{Value} || $vars->{value});
2235
}
2236
else {
2237
$innodb_version = $innodb->{support}; # probably DISABLED or NO
2238
}
2239
}
2240
2241
MKDEBUG && _d("InnoDB version:", $innodb_version);
2242
return $innodb_version;
2243
}
2244
2245
sub _d {
2246
my ($package, undef, $line) = caller 0;
2247
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
2248
map { defined $_ ? $_ : 'undef' }
2249
@_;
2250
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
2251
}
2252
2253
1;
2254
}
2255
# ###########################################################################
2256
# End VersionParser package
2257
# ###########################################################################
2258
2259
# ###########################################################################
2260
# MySQLDump package
2261
# This package is a copy without comments from the original. The original
2262
# with comments and its test file can be found in the Bazaar repository at,
2263
# lib/MySQLDump.pm
2264
# t/lib/MySQLDump.t
2265
# See https://launchpad.net/percona-toolkit for more information.
2266
# ###########################################################################
2267
{
2268
package MySQLDump;
2269
2270
use strict;
2271
use warnings FATAL => 'all';
2272
use English qw(-no_match_vars);
2273
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
2274
2275
( our $before = <<'EOF') =~ s/^ //gm;
2276
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
2277
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
2278
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
2279
/*!40101 SET NAMES utf8 */;
2280
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
2281
/*!40103 SET TIME_ZONE='+00:00' */;
2282
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
2283
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
2284
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
2285
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
2286
EOF
2287
2288
( our $after = <<'EOF') =~ s/^ //gm;
2289
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
2290
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
2291
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
2292
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
2293
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
2294
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
2295
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
2296
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
2297
EOF
2298
2299
sub new {
2300
my ( $class, %args ) = @_;
2301
my $self = {
2302
cache => 0, # Afaik no script uses this cache any longer because
2303
};
2304
return bless $self, $class;
2305
}
2306
2307
sub dump {
2308
my ( $self, $dbh, $quoter, $db, $tbl, $what ) = @_;
2309
2310
if ( $what eq 'table' ) {
2311
my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl);
2312
return unless $ddl;
2313
if ( $ddl->[0] eq 'table' ) {
2314
return $before
2315
. 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n"
2316
. $ddl->[1] . ";\n";
2317
}
2318
else {
2319
return 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n"
2320
. '/*!50001 DROP VIEW IF EXISTS '
2321
. $quoter->quote($tbl) . "*/;\n/*!50001 "
2322
. $self->get_tmp_table($dbh, $quoter, $db, $tbl) . "*/;\n";
2323
}
2324
}
2325
elsif ( $what eq 'triggers' ) {
2326
my $trgs = $self->get_triggers($dbh, $quoter, $db, $tbl);
2327
if ( $trgs && @$trgs ) {
2328
my $result = $before . "\nDELIMITER ;;\n";
2329
foreach my $trg ( @$trgs ) {
2330
if ( $trg->{sql_mode} ) {
2331
$result .= qq{/*!50003 SET SESSION SQL_MODE='$trg->{sql_mode}' */;;\n};
2332
}
2333
$result .= "/*!50003 CREATE */ ";
2334
if ( $trg->{definer} ) {
2335
my ( $user, $host )
2336
= map { s/'/''/g; "'$_'"; }
2337
split('@', $trg->{definer}, 2);
2338
$result .= "/*!50017 DEFINER=$user\@$host */ ";
2339
}
2340
$result .= sprintf("/*!50003 TRIGGER %s %s %s ON %s\nFOR EACH ROW %s */;;\n\n",
2341
$quoter->quote($trg->{trigger}),
2342
@{$trg}{qw(timing event)},
2343
$quoter->quote($trg->{table}),
2344
$trg->{statement});
2345
}
2346
$result .= "DELIMITER ;\n\n/*!50003 SET SESSION SQL_MODE=\@OLD_SQL_MODE */;\n\n";
2347
return $result;
2348
}
2349
else {
2350
return undef;
2351
}
2352
}
2353
elsif ( $what eq 'view' ) {
2354
my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl);
2355
return '/*!50001 DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . "*/;\n"
2356
. '/*!50001 DROP VIEW IF EXISTS ' . $quoter->quote($tbl) . "*/;\n"
2357
. '/*!50001 ' . $ddl->[1] . "*/;\n";
2358
}
2359
else {
2360
die "You didn't say what to dump.";
2361
}
2362
}
2363
2364
sub _use_db {
2365
my ( $self, $dbh, $quoter, $new ) = @_;
2366
if ( !$new ) {
2367
MKDEBUG && _d('No new DB to use');
2368
return;
2369
}
2370
my $sql = 'USE ' . $quoter->quote($new);
2371
MKDEBUG && _d($dbh, $sql);
2372
$dbh->do($sql);
1495
sub dbh {
1496
my ($self) = @_;
1497
return $self->{dbh};
1498
}
1499
1500
sub dsn {
1501
my ($self) = @_;
1502
return $self->{dsn};
1503
}
1504
1505
sub name {
1506
my ($self) = @_;
1507
return $self->{dsn_name} if PERCONA_TOOLKIT_TEST_USE_DSN_NAMES;
1508
return $self->{hostname} || $self->{dsn_name} || 'unknown host';
1509
}
1510
1511
sub DESTROY {
1512
my ($self) = @_;
1513
if ( $self->{dbh} ) {
1514
PTDEBUG && _d('Disconnecting dbh', $self->{dbh}, $self->{name});
1515
$self->{dbh}->disconnect();
1516
}
2373
1517
return;
2374
1518
}
2375
1519
2376
sub get_create_table {
2377
my ( $self, $dbh, $quoter, $db, $tbl ) = @_;
2378
if ( !$self->{cache} || !$self->{tables}->{$db}->{$tbl} ) {
2379
my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, '
2380
. q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), }
2381
. '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, '
2382
. '@@SQL_QUOTE_SHOW_CREATE := 1 */';
2383
MKDEBUG && _d($sql);
2384
eval { $dbh->do($sql); };
2385
MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
2386
$self->_use_db($dbh, $quoter, $db);
2387
$sql = "SHOW CREATE TABLE " . $quoter->quote($db, $tbl);
2388
MKDEBUG && _d($sql);
2389
my $href;
2390
eval { $href = $dbh->selectrow_hashref($sql); };
2391
if ( $EVAL_ERROR ) {
2392
warn "Failed to $sql. The table may be damaged.\nError: $EVAL_ERROR";
2393
return;
2394
}
2395
2396
$sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, '
2397
. '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */';
2398
MKDEBUG && _d($sql);
2399
$dbh->do($sql);
2400
my ($key) = grep { m/create table/i } keys %$href;
2401
if ( $key ) {
2402
MKDEBUG && _d('This table is a base table');
2403
$self->{tables}->{$db}->{$tbl} = [ 'table', $href->{$key} ];
2404
}
2405
else {
2406
MKDEBUG && _d('This table is a view');
2407
($key) = grep { m/create view/i } keys %$href;
2408
$self->{tables}->{$db}->{$tbl} = [ 'view', $href->{$key} ];
2409
}
2410
}
2411
return $self->{tables}->{$db}->{$tbl};
2412
}
2413
2414
sub get_columns {
2415
my ( $self, $dbh, $quoter, $db, $tbl ) = @_;
2416
MKDEBUG && _d('Get columns for', $db, $tbl);
2417
if ( !$self->{cache} || !$self->{columns}->{$db}->{$tbl} ) {
2418
$self->_use_db($dbh, $quoter, $db);
2419
my $sql = "SHOW COLUMNS FROM " . $quoter->quote($db, $tbl);
2420
MKDEBUG && _d($sql);
2421
my $cols = $dbh->selectall_arrayref($sql, { Slice => {} });
2422
2423
$self->{columns}->{$db}->{$tbl} = [
2424
map {
2425
my %row;
2426
@row{ map { lc $_ } keys %$_ } = values %$_;
2427
\%row;
2428
} @$cols
2429
];
2430
}
2431
return $self->{columns}->{$db}->{$tbl};
2432
}
2433
2434
sub get_tmp_table {
2435
my ( $self, $dbh, $quoter, $db, $tbl ) = @_;
2436
my $result = 'CREATE TABLE ' . $quoter->quote($tbl) . " (\n";
2437
$result .= join(",\n",
2438
map { ' ' . $quoter->quote($_->{field}) . ' ' . $_->{type} }
2439
@{$self->get_columns($dbh, $quoter, $db, $tbl)});
2440
$result .= "\n)";
2441
MKDEBUG && _d($result);
2442
return $result;
2443
}
2444
2445
sub get_triggers {
2446
my ( $self, $dbh, $quoter, $db, $tbl ) = @_;
2447
if ( !$self->{cache} || !$self->{triggers}->{$db} ) {
2448
$self->{triggers}->{$db} = {};
2449
my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, '
2450
. q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), }
2451
. '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, '
2452
. '@@SQL_QUOTE_SHOW_CREATE := 1 */';
2453
MKDEBUG && _d($sql);
2454
eval { $dbh->do($sql); };
2455
MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
2456
$sql = "SHOW TRIGGERS FROM " . $quoter->quote($db);
2457
MKDEBUG && _d($sql);
2458
my $sth = $dbh->prepare($sql);
2459
$sth->execute();
2460
if ( $sth->rows ) {
2461
my $trgs = $sth->fetchall_arrayref({});
2462
foreach my $trg (@$trgs) {
2463
my %trg;
2464
@trg{ map { lc $_ } keys %$trg } = values %$trg;
2465
push @{ $self->{triggers}->{$db}->{ $trg{table} } }, \%trg;
2466
}
2467
}
2468
$sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, '
2469
. '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */';
2470
MKDEBUG && _d($sql);
2471
$dbh->do($sql);
2472
}
2473
if ( $tbl ) {
2474
return $self->{triggers}->{$db}->{$tbl};
2475
}
2476
return values %{$self->{triggers}->{$db}};
2477
}
2478
2479
sub get_databases {
2480
my ( $self, $dbh, $quoter, $like ) = @_;
2481
if ( !$self->{cache} || !$self->{databases} || $like ) {
2482
my $sql = 'SHOW DATABASES';
2483
my @params;
2484
if ( $like ) {
2485
$sql .= ' LIKE ?';
2486
push @params, $like;
2487
}
2488
my $sth = $dbh->prepare($sql);
2489
MKDEBUG && _d($sql, @params);
2490
$sth->execute( @params );
2491
my @dbs = map { $_->[0] } @{$sth->fetchall_arrayref()};
2492
$self->{databases} = \@dbs unless $like;
2493
return @dbs;
2494
}
2495
return @{$self->{databases}};
2496
}
2497
2498
sub get_table_status {
2499
my ( $self, $dbh, $quoter, $db, $like ) = @_;
2500
if ( !$self->{cache} || !$self->{table_status}->{$db} || $like ) {
2501
my $sql = "SHOW TABLE STATUS FROM " . $quoter->quote($db);
2502
my @params;
2503
if ( $like ) {
2504
$sql .= ' LIKE ?';
2505
push @params, $like;
2506
}
2507
MKDEBUG && _d($sql, @params);
2508
my $sth = $dbh->prepare($sql);
2509
$sth->execute(@params);
2510
my @tables = @{$sth->fetchall_arrayref({})};
2511
@tables = map {
2512
my %tbl; # Make a copy with lowercased keys
2513
@tbl{ map { lc $_ } keys %$_ } = values %$_;
2514
$tbl{engine} ||= $tbl{type} || $tbl{comment};
2515
delete $tbl{type};
2516
\%tbl;
2517
} @tables;
2518
$self->{table_status}->{$db} = \@tables unless $like;
2519
return @tables;
2520
}
2521
return @{$self->{table_status}->{$db}};
2522
}
2523
2524
sub get_table_list {
2525
my ( $self, $dbh, $quoter, $db, $like ) = @_;
2526
if ( !$self->{cache} || !$self->{table_list}->{$db} || $like ) {
2527
my $sql = "SHOW /*!50002 FULL*/ TABLES FROM " . $quoter->quote($db);
2528
my @params;
2529
if ( $like ) {
2530
$sql .= ' LIKE ?';
2531
push @params, $like;
2532
}
2533
MKDEBUG && _d($sql, @params);
2534
my $sth = $dbh->prepare($sql);
2535
$sth->execute(@params);
2536
my @tables = @{$sth->fetchall_arrayref()};
2537
@tables = map {
2538
my %tbl = (
2539
name => $_->[0],
2540
engine => ($_->[1] || '') eq 'VIEW' ? 'VIEW' : '',
2541
);
2542
\%tbl;
2543
} @tables;
2544
$self->{table_list}->{$db} = \@tables unless $like;
2545
return @tables;
2546
}
2547
return @{$self->{table_list}->{$db}};
2548
}
2549
2550
sub _d {
2551
my ($package, undef, $line) = caller 0;
2552
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
2553
map { defined $_ ? $_ : 'undef' }
2554
@_;
2555
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
2556
}
2557
2558
1;
2559
}
2560
# ###########################################################################
2561
# End MySQLDump package
2562
# ###########################################################################
2563
2564
# ###########################################################################
2565
# TableChunker package
2566
# This package is a copy without comments from the original. The original
2567
# with comments and its test file can be found in the Bazaar repository at,
2568
# lib/TableChunker.pm
2569
# t/lib/TableChunker.t
2570
# See https://launchpad.net/percona-toolkit for more information.
2571
# ###########################################################################
2572
{
2573
package TableChunker;
2574
2575
use strict;
2576
use warnings FATAL => 'all';
2577
use English qw(-no_match_vars);
2578
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
2579
2580
use POSIX qw(floor ceil);
2581
use List::Util qw(min max);
2582
use Data::Dumper;
2583
$Data::Dumper::Indent = 1;
2584
$Data::Dumper::Sortkeys = 1;
2585
$Data::Dumper::Quotekeys = 0;
2586
2587
sub new {
2588
my ( $class, %args ) = @_;
2589
foreach my $arg ( qw(Quoter MySQLDump) ) {
2590
die "I need a $arg argument" unless $args{$arg};
2591
}
2592
2593
my %int_types = map { $_ => 1 } qw(bigint date datetime int mediumint smallint time timestamp tinyint year);
2594
my %real_types = map { $_ => 1 } qw(decimal double float);
2595
2596
my $self = {
2597
%args,
2598
int_types => \%int_types,
2599
real_types => \%real_types,
2600
EPOCH => '1970-01-01',
2601
};
2602
2603
return bless $self, $class;
2604
}
2605
2606
sub find_chunk_columns {
2607
my ( $self, %args ) = @_;
2608
foreach my $arg ( qw(tbl_struct) ) {
2609
die "I need a $arg argument" unless $args{$arg};
2610
}
2611
my $tbl_struct = $args{tbl_struct};
2612
2613
my @possible_indexes;
2614
foreach my $index ( values %{ $tbl_struct->{keys} } ) {
2615
2616
next unless $index->{type} eq 'BTREE';
2617
2618
next if grep { defined } @{$index->{col_prefixes}};
2619
2620
if ( $args{exact} ) {
2621
next unless $index->{is_unique} && @{$index->{cols}} == 1;
2622
}
2623
2624
push @possible_indexes, $index;
2625
}
2626
MKDEBUG && _d('Possible chunk indexes in order:',
2627
join(', ', map { $_->{name} } @possible_indexes));
2628
2629
my $can_chunk_exact = 0;
2630
my @candidate_cols;
2631
foreach my $index ( @possible_indexes ) {
2632
my $col = $index->{cols}->[0];
2633
2634
my $col_type = $tbl_struct->{type_for}->{$col};
2635
next unless $self->{int_types}->{$col_type}
2636
|| $self->{real_types}->{$col_type}
2637
|| $col_type =~ m/char/;
2638
2639
push @candidate_cols, { column => $col, index => $index->{name} };
2640
}
2641
2642
$can_chunk_exact = 1 if $args{exact} && scalar @candidate_cols;
2643
2644
if ( MKDEBUG ) {
2645
my $chunk_type = $args{exact} ? 'Exact' : 'Inexact';
2646
_d($chunk_type, 'chunkable:',
2647
join(', ', map { "$_->{column} on $_->{index}" } @candidate_cols));
2648
}
2649
2650
my @result;
2651
MKDEBUG && _d('Ordering columns by order in tbl, PK first');
2652
if ( $tbl_struct->{keys}->{PRIMARY} ) {
2653
my $pk_first_col = $tbl_struct->{keys}->{PRIMARY}->{cols}->[0];
2654
@result = grep { $_->{column} eq $pk_first_col } @candidate_cols;
2655
@candidate_cols = grep { $_->{column} ne $pk_first_col } @candidate_cols;
2656
}
2657
my $i = 0;
2658
my %col_pos = map { $_ => $i++ } @{$tbl_struct->{cols}};
2659
push @result, sort { $col_pos{$a->{column}} <=> $col_pos{$b->{column}} }
2660
@candidate_cols;
2661
2662
if ( MKDEBUG ) {
2663
_d('Chunkable columns:',
2664
join(', ', map { "$_->{column} on $_->{index}" } @result));
2665
_d('Can chunk exactly:', $can_chunk_exact);
2666
}
2667
2668
return ($can_chunk_exact, @result);
2669
}
2670
2671
sub calculate_chunks {
2672
my ( $self, %args ) = @_;
2673
my @required_args = qw(dbh db tbl tbl_struct chunk_col rows_in_range chunk_size);
2674
foreach my $arg ( @required_args ) {
2675
die "I need a $arg argument" unless defined $args{$arg};
2676
}
2677
MKDEBUG && _d('Calculate chunks for',
2678
join(", ", map {"$_=".(defined $args{$_} ? $args{$_} : "undef")}
2679
qw(db tbl chunk_col min max rows_in_range chunk_size zero_chunk exact)
2680
));
2681
2682
if ( !$args{rows_in_range} ) {
2683
MKDEBUG && _d("Empty table");
2684
return '1=1';
2685
}
2686
2687
if ( $args{rows_in_range} < $args{chunk_size} ) {
2688
MKDEBUG && _d("Chunk size larger than rows in range");
2689
return '1=1';
2690
}
2691
2692
my $q = $self->{Quoter};
2693
my $dbh = $args{dbh};
2694
my $chunk_col = $args{chunk_col};
2695
my $tbl_struct = $args{tbl_struct};
2696
my $col_type = $tbl_struct->{type_for}->{$chunk_col};
2697
MKDEBUG && _d('chunk col type:', $col_type);
2698
2699
my %chunker;
2700
if ( $tbl_struct->{is_numeric}->{$chunk_col} || $col_type =~ /date|time/ ) {
2701
%chunker = $self->_chunk_numeric(%args);
2702
}
2703
elsif ( $col_type =~ m/char/ ) {
2704
%chunker = $self->_chunk_char(%args);
2705
}
2706
else {
2707
die "Cannot chunk $col_type columns";
2708
}
2709
MKDEBUG && _d("Chunker:", Dumper(\%chunker));
2710
my ($col, $start_point, $end_point, $interval, $range_func)
2711
= @chunker{qw(col start_point end_point interval range_func)};
2712
2713
my @chunks;
2714
if ( $start_point < $end_point ) {
2715
2716
push @chunks, "$col = 0" if $chunker{have_zero_chunk};
2717
2718
my ($beg, $end);
2719
my $iter = 0;
2720
for ( my $i = $start_point; $i < $end_point; $i += $interval ) {
2721
($beg, $end) = $self->$range_func($dbh, $i, $interval, $end_point);
2722
2723
if ( $iter++ == 0 ) {
2724
push @chunks,
2725
($chunker{have_zero_chunk} ? "$col > 0 AND " : "")
2726
."$col < " . $q->quote_val($end);
2727
}
2728
else {
2729
push @chunks, "$col >= " . $q->quote_val($beg) . " AND $col < " . $q->quote_val($end);
2730
}
2731
}
2732
2733
my $chunk_range = lc $args{chunk_range} || 'open';
2734
my $nullable = $args{tbl_struct}->{is_nullable}->{$args{chunk_col}};
2735
pop @chunks;
2736
if ( @chunks ) {
2737
push @chunks, "$col >= " . $q->quote_val($beg)
2738
. ($chunk_range eq 'openclosed'
2739
? " AND $col <= " . $q->quote_val($args{max}) : "");
2740
}
2741
else {
2742
push @chunks, $nullable ? "$col IS NOT NULL" : '1=1';
2743
}
2744
if ( $nullable ) {
2745
push @chunks, "$col IS NULL";
2746
}
2747
}
2748
else {
2749
MKDEBUG && _d('No chunks; using single chunk 1=1');
2750
push @chunks, '1=1';
2751
}
2752
2753
return @chunks;
2754
}
2755
2756
sub _chunk_numeric {
2757
my ( $self, %args ) = @_;
2758
my @required_args = qw(dbh db tbl tbl_struct chunk_col rows_in_range chunk_size);
2759
foreach my $arg ( @required_args ) {
2760
die "I need a $arg argument" unless defined $args{$arg};
2761
}
2762
my $q = $self->{Quoter};
2763
my $db_tbl = $q->quote($args{db}, $args{tbl});
2764
my $col_type = $args{tbl_struct}->{type_for}->{$args{chunk_col}};
2765
2766
my $range_func;
2767
if ( $col_type =~ m/(?:int|year|float|double|decimal)$/ ) {
2768
$range_func = 'range_num';
2769
}
2770
elsif ( $col_type =~ m/^(?:timestamp|date|time)$/ ) {
2771
$range_func = "range_$col_type";
2772
}
2773
elsif ( $col_type eq 'datetime' ) {
2774
$range_func = 'range_datetime';
2775
}
2776
2777
my ($start_point, $end_point);
2778
eval {
2779
$start_point = $self->value_to_number(
2780
value => $args{min},
2781
column_type => $col_type,
2782
dbh => $args{dbh},
2783
);
2784
$end_point = $self->value_to_number(
2785
value => $args{max},
2786
column_type => $col_type,
2787
dbh => $args{dbh},
2788
);
2789
};
2790
if ( $EVAL_ERROR ) {
2791
if ( $EVAL_ERROR =~ m/don't know how to chunk/ ) {
2792
die $EVAL_ERROR;
2793
}
2794
else {
2795
die "Error calculating chunk start and end points for table "
2796
. "`$args{tbl_struct}->{name}` on column `$args{chunk_col}` "
2797
. "with min/max values "
2798
. join('/',
2799
map { defined $args{$_} ? $args{$_} : 'undef' } qw(min max))
2800
. ":\n\n"
2801
. $EVAL_ERROR
2802
. "\nVerify that the min and max values are valid for the column. "
2803
. "If they are valid, this error could be caused by a bug in the "
2804
. "tool.";
2805
}
2806
}
2807
2808
if ( !defined $start_point ) {
2809
MKDEBUG && _d('Start point is undefined');
2810
$start_point = 0;
2811
}
2812
if ( !defined $end_point || $end_point < $start_point ) {
2813
MKDEBUG && _d('End point is undefined or before start point');
2814
$end_point = 0;
2815
}
2816
MKDEBUG && _d("Actual chunk range:", $start_point, "to", $end_point);
2817
2818
my $have_zero_chunk = 0;
2819
if ( $args{zero_chunk} ) {
2820
if ( $start_point != $end_point && $start_point >= 0 ) {
2821
MKDEBUG && _d('Zero chunking');
2822
my $nonzero_val = $self->get_nonzero_value(
2823
%args,
2824
db_tbl => $db_tbl,
2825
col => $args{chunk_col},
2826
col_type => $col_type,
2827
val => $args{min}
2828
);
2829
$start_point = $self->value_to_number(
2830
value => $nonzero_val,
2831
column_type => $col_type,
2832
dbh => $args{dbh},
2833
);
2834
$have_zero_chunk = 1;
2835
}
2836
else {
2837
MKDEBUG && _d("Cannot zero chunk");
2838
}
2839
}
2840
MKDEBUG && _d("Using chunk range:", $start_point, "to", $end_point);
2841
2842
my $interval = $args{chunk_size}
2843
* ($end_point - $start_point)
2844
/ $args{rows_in_range};
2845
if ( $self->{int_types}->{$col_type} ) {
2846
$interval = ceil($interval);
2847
}
2848
$interval ||= $args{chunk_size};
2849
if ( $args{exact} ) {
2850
$interval = $args{chunk_size};
2851
}
2852
MKDEBUG && _d('Chunk interval:', $interval, 'units');
2853
2854
return (
2855
col => $q->quote($args{chunk_col}),
2856
start_point => $start_point,
2857
end_point => $end_point,
2858
interval => $interval,
2859
range_func => $range_func,
2860
have_zero_chunk => $have_zero_chunk,
2861
);
2862
}
2863
2864
sub _chunk_char {
2865
my ( $self, %args ) = @_;
2866
my @required_args = qw(dbh db tbl tbl_struct chunk_col rows_in_range chunk_size);
2867
foreach my $arg ( @required_args ) {
2868
die "I need a $arg argument" unless defined $args{$arg};
2869
}
2870
my $q = $self->{Quoter};
2871
my $db_tbl = $q->quote($args{db}, $args{tbl});
2872
my $dbh = $args{dbh};
2873
my $chunk_col = $args{chunk_col};
2874
my $row;
2875
my $sql;
2876
2877
$sql = "SELECT MIN($chunk_col), MAX($chunk_col) FROM $db_tbl "
2878
. "ORDER BY `$chunk_col`";
2879
MKDEBUG && _d($dbh, $sql);
2880
$row = $dbh->selectrow_arrayref($sql);
2881
my ($min_col, $max_col) = ($row->[0], $row->[1]);
2882
2883
$sql = "SELECT ORD(?) AS min_col_ord, ORD(?) AS max_col_ord";
2884
MKDEBUG && _d($dbh, $sql);
2885
my $ord_sth = $dbh->prepare($sql); # avoid quoting issues
2886
$ord_sth->execute($min_col, $max_col);
2887
$row = $ord_sth->fetchrow_arrayref();
2888
my ($min_col_ord, $max_col_ord) = ($row->[0], $row->[1]);
2889
MKDEBUG && _d("Min/max col char code:", $min_col_ord, $max_col_ord);
2890
2891
my $base;
2892
my @chars;
2893
MKDEBUG && _d("Table charset:", $args{tbl_struct}->{charset});
2894
if ( ($args{tbl_struct}->{charset} || "") eq "latin1" ) {
2895
my @sorted_latin1_chars = (
2896
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45,
2897
46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
2898
60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
2899
74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
2900
88, 89, 90, 91, 92, 93, 94, 95, 96, 123, 124, 125, 126, 161,
2901
162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
2902
176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
2903
190, 191, 215, 216, 222, 223, 247, 255);
2904
2905
my ($first_char, $last_char);
2906
for my $i ( 0..$#sorted_latin1_chars ) {
2907
$first_char = $i and last if $sorted_latin1_chars[$i] >= $min_col_ord;
2908
}
2909
for my $i ( $first_char..$#sorted_latin1_chars ) {
2910
$last_char = $i and last if $sorted_latin1_chars[$i] >= $max_col_ord;
2911
};
2912
2913
@chars = map { chr $_; } @sorted_latin1_chars[$first_char..$last_char];
2914
$base = scalar @chars;
2915
}
2916
else {
2917
2918
my $tmp_tbl = '__maatkit_char_chunking_map';
2919
my $tmp_db_tbl = $q->quote($args{db}, $tmp_tbl);
2920
$sql = "DROP TABLE IF EXISTS $tmp_db_tbl";
2921
MKDEBUG && _d($dbh, $sql);
2922
$dbh->do($sql);
2923
my $col_def = $args{tbl_struct}->{defs}->{$chunk_col};
2924
$sql = "CREATE TEMPORARY TABLE $tmp_db_tbl ($col_def) "
2925
. "ENGINE=MEMORY";
2926
MKDEBUG && _d($dbh, $sql);
2927
$dbh->do($sql);
2928
2929
$sql = "INSERT INTO $tmp_db_tbl VALUE (CHAR(?))";
2930
MKDEBUG && _d($dbh, $sql);
2931
my $ins_char_sth = $dbh->prepare($sql); # avoid quoting issues
2932
for my $char_code ( $min_col_ord..$max_col_ord ) {
2933
$ins_char_sth->execute($char_code);
2934
}
2935
2936
$sql = "SELECT `$chunk_col` FROM $tmp_db_tbl "
2937
. "WHERE `$chunk_col` BETWEEN ? AND ? "
2938
. "ORDER BY `$chunk_col`";
2939
MKDEBUG && _d($dbh, $sql);
2940
my $sel_char_sth = $dbh->prepare($sql);
2941
$sel_char_sth->execute($min_col, $max_col);
2942
2943
@chars = map { $_->[0] } @{ $sel_char_sth->fetchall_arrayref() };
2944
$base = scalar @chars;
2945
2946
$sql = "DROP TABLE $tmp_db_tbl";
2947
MKDEBUG && _d($dbh, $sql);
2948
$dbh->do($sql);
2949
}
2950
MKDEBUG && _d("Base", $base, "chars:", @chars);
2951
2952
2953
$sql = "SELECT MAX(LENGTH($chunk_col)) FROM $db_tbl ORDER BY `$chunk_col`";
2954
MKDEBUG && _d($dbh, $sql);
2955
$row = $dbh->selectrow_arrayref($sql);
2956
my $max_col_len = $row->[0];
2957
MKDEBUG && _d("Max column value:", $max_col, $max_col_len);
2958
my $n_values;
2959
for my $n_chars ( 1..$max_col_len ) {
2960
$n_values = $base**$n_chars;
2961
if ( $n_values >= $args{chunk_size} ) {
2962
MKDEBUG && _d($n_chars, "chars in base", $base, "expresses",
2963
$n_values, "values");
2964
last;
2965
}
2966
}
2967
2968
my $n_chunks = $args{rows_in_range} / $args{chunk_size};
2969
my $interval = floor($n_values / $n_chunks) || 1;
2970
2971
my $range_func = sub {
2972
my ( $self, $dbh, $start, $interval, $max ) = @_;
2973
my $start_char = $self->base_count(
2974
count_to => $start,
2975
base => $base,
2976
symbols => \@chars,
2977
);
2978
my $end_char = $self->base_count(
2979
count_to => min($max, $start + $interval),
2980
base => $base,
2981
symbols => \@chars,
2982
);
2983
return $start_char, $end_char;
2984
};
2985
2986
return (
2987
col => $q->quote($chunk_col),
2988
start_point => 0,
2989
end_point => $n_values,
2990
interval => $interval,
2991
range_func => $range_func,
2992
);
2993
}
2994
2995
sub get_first_chunkable_column {
2996
my ( $self, %args ) = @_;
2997
foreach my $arg ( qw(tbl_struct) ) {
2998
die "I need a $arg argument" unless $args{$arg};
2999
}
3000
3001
my ($exact, @cols) = $self->find_chunk_columns(%args);
3002
my $col = $cols[0]->{column};
3003
my $idx = $cols[0]->{index};
3004
3005
my $wanted_col = $args{chunk_column};
3006
my $wanted_idx = $args{chunk_index};
3007
MKDEBUG && _d("Preferred chunk col/idx:", $wanted_col, $wanted_idx);
3008
3009
if ( $wanted_col && $wanted_idx ) {
3010
foreach my $chunkable_col ( @cols ) {
3011
if ( $wanted_col eq $chunkable_col->{column}
3012
&& $wanted_idx eq $chunkable_col->{index} ) {
3013
$col = $wanted_col;
3014
$idx = $wanted_idx;
3015
last;
3016
}
3017
}
3018
}
3019
elsif ( $wanted_col ) {
3020
foreach my $chunkable_col ( @cols ) {
3021
if ( $wanted_col eq $chunkable_col->{column} ) {
3022
$col = $wanted_col;
3023
$idx = $chunkable_col->{index};
3024
last;
3025
}
3026
}
3027
}
3028
elsif ( $wanted_idx ) {
3029
foreach my $chunkable_col ( @cols ) {
3030
if ( $wanted_idx eq $chunkable_col->{index} ) {
3031
$col = $chunkable_col->{column};
3032
$idx = $wanted_idx;
3033
last;
3034
}
3035
}
3036
}
3037
3038
MKDEBUG && _d('First chunkable col/index:', $col, $idx);
3039
return $col, $idx;
3040
}
3041
3042
sub size_to_rows {
3043
my ( $self, %args ) = @_;
3044
my @required_args = qw(dbh db tbl chunk_size);
3045
foreach my $arg ( @required_args ) {
3046
die "I need a $arg argument" unless $args{$arg};
3047
}
3048
my ($dbh, $db, $tbl, $chunk_size) = @args{@required_args};
3049
my $q = $self->{Quoter};
3050
my $du = $self->{MySQLDump};
3051
3052
my ($n_rows, $avg_row_length);
3053
3054
my ( $num, $suffix ) = $chunk_size =~ m/^(\d+)([MGk])?$/;
3055
if ( $suffix ) { # Convert to bytes.
3056
$chunk_size = $suffix eq 'k' ? $num * 1_024
3057
: $suffix eq 'M' ? $num * 1_024 * 1_024
3058
: $num * 1_024 * 1_024 * 1_024;
3059
}
3060
elsif ( $num ) {
3061
$n_rows = $num;
3062
}
3063
else {
3064
die "Invalid chunk size $chunk_size; must be an integer "
3065
. "with optional suffix kMG";
3066
}
3067
3068
if ( $suffix || $args{avg_row_length} ) {
3069
my ($status) = $du->get_table_status($dbh, $q, $db, $tbl);
3070
$avg_row_length = $status->{avg_row_length};
3071
if ( !defined $n_rows ) {
3072
$n_rows = $avg_row_length ? ceil($chunk_size / $avg_row_length) : undef;
3073
}
3074
}
3075
3076
return $n_rows, $avg_row_length;
3077
}
3078
3079
sub get_range_statistics {
3080
my ( $self, %args ) = @_;
3081
my @required_args = qw(dbh db tbl chunk_col tbl_struct);
3082
foreach my $arg ( @required_args ) {
3083
die "I need a $arg argument" unless $args{$arg};
3084
}
3085
my ($dbh, $db, $tbl, $col) = @args{@required_args};
3086
my $where = $args{where};
3087
my $q = $self->{Quoter};
3088
3089
my $col_type = $args{tbl_struct}->{type_for}->{$col};
3090
my $col_is_numeric = $args{tbl_struct}->{is_numeric}->{$col};
3091
3092
my $db_tbl = $q->quote($db, $tbl);
3093
$col = $q->quote($col);
3094
3095
my ($min, $max);
3096
eval {
3097
my $sql = "SELECT MIN($col), MAX($col) FROM $db_tbl"
3098
. ($args{index_hint} ? " $args{index_hint}" : "")
3099
. ($where ? " WHERE ($where)" : '');
3100
MKDEBUG && _d($dbh, $sql);
3101
($min, $max) = $dbh->selectrow_array($sql);
3102
MKDEBUG && _d("Actual end points:", $min, $max);
3103
3104
($min, $max) = $self->get_valid_end_points(
3105
%args,
3106
dbh => $dbh,
3107
db_tbl => $db_tbl,
3108
col => $col,
3109
col_type => $col_type,
3110
min => $min,
3111
max => $max,
3112
);
3113
MKDEBUG && _d("Valid end points:", $min, $max);
3114
};
3115
if ( $EVAL_ERROR ) {
3116
die "Error getting min and max values for table $db_tbl "
3117
. "on column $col: $EVAL_ERROR";
3118
}
3119
3120
my $sql = "EXPLAIN SELECT * FROM $db_tbl"
3121
. ($args{index_hint} ? " $args{index_hint}" : "")
3122
. ($where ? " WHERE $where" : '');
3123
MKDEBUG && _d($sql);
3124
my $expl = $dbh->selectrow_hashref($sql);
3125
3126
return (
3127
min => $min,
3128
max => $max,
3129
rows_in_range => $expl->{rows},
3130
);
3131
}
3132
3133
sub inject_chunks {
3134
my ( $self, %args ) = @_;
3135
foreach my $arg ( qw(database table chunks chunk_num query) ) {
3136
die "I need a $arg argument" unless defined $args{$arg};
3137
}
3138
MKDEBUG && _d('Injecting chunk', $args{chunk_num});
3139
my $query = $args{query};
3140
my $comment = sprintf("/*%s.%s:%d/%d*/",
3141
$args{database}, $args{table},
3142
$args{chunk_num} + 1, scalar @{$args{chunks}});
3143
$query =~ s!/\*PROGRESS_COMMENT\*/!$comment!;
3144
my $where = "WHERE (" . $args{chunks}->[$args{chunk_num}] . ')';
3145
if ( $args{where} && grep { $_ } @{$args{where}} ) {
3146
$where .= " AND ("
3147
. join(" AND ", map { "($_)" } grep { $_ } @{$args{where}} )
3148
. ")";
3149
}
3150
my $db_tbl = $self->{Quoter}->quote(@args{qw(database table)});
3151
my $index_hint = $args{index_hint} || '';
3152
3153
MKDEBUG && _d('Parameters:',
3154
Dumper({WHERE => $where, DB_TBL => $db_tbl, INDEX_HINT => $index_hint}));
3155
$query =~ s!/\*WHERE\*/! $where!;
3156
$query =~ s!/\*DB_TBL\*/!$db_tbl!;
3157
$query =~ s!/\*INDEX_HINT\*/! $index_hint!;
3158
$query =~ s!/\*CHUNK_NUM\*/! $args{chunk_num} AS chunk_num,!;
3159
3160
return $query;
3161
}
3162
3163
3164
sub value_to_number {
3165
my ( $self, %args ) = @_;
3166
my @required_args = qw(column_type dbh);
3167
foreach my $arg ( @required_args ) {
3168
die "I need a $arg argument" unless defined $args{$arg};
3169
}
3170
my $val = $args{value};
3171
my ($col_type, $dbh) = @args{@required_args};
3172
MKDEBUG && _d('Converting MySQL', $col_type, $val);
3173
3174
return unless defined $val; # value is NULL
3175
3176
my %mysql_conv_func_for = (
3177
timestamp => 'UNIX_TIMESTAMP',
3178
date => 'TO_DAYS',
3179
time => 'TIME_TO_SEC',
3180
datetime => 'TO_DAYS',
3181
);
3182
3183
my $num;
3184
if ( $col_type =~ m/(?:int|year|float|double|decimal)$/ ) {
3185
$num = $val;
3186
}
3187
elsif ( $col_type =~ m/^(?:timestamp|date|time)$/ ) {
3188
my $func = $mysql_conv_func_for{$col_type};
3189
my $sql = "SELECT $func(?)";
3190
MKDEBUG && _d($dbh, $sql, $val);
3191
my $sth = $dbh->prepare($sql);
3192
$sth->execute($val);
3193
($num) = $sth->fetchrow_array();
3194
}
3195
elsif ( $col_type eq 'datetime' ) {
3196
$num = $self->timestampdiff($dbh, $val);
3197
}
3198
else {
3199
die "I don't know how to chunk $col_type\n";
3200
}
3201
MKDEBUG && _d('Converts to', $num);
3202
return $num;
3203
}
3204
3205
sub range_num {
3206
my ( $self, $dbh, $start, $interval, $max ) = @_;
3207
my $end = min($max, $start + $interval);
3208
3209
3210
$start = sprintf('%.17f', $start) if $start =~ /e/;
3211
$end = sprintf('%.17f', $end) if $end =~ /e/;
3212
3213
$start =~ s/\.(\d{5}).*$/.$1/;
3214
$end =~ s/\.(\d{5}).*$/.$1/;
3215
3216
if ( $end > $start ) {
3217
return ( $start, $end );
3218
}
3219
else {
3220
die "Chunk size is too small: $end !> $start\n";
3221
}
3222
}
3223
3224
sub range_time {
3225
my ( $self, $dbh, $start, $interval, $max ) = @_;
3226
my $sql = "SELECT SEC_TO_TIME($start), SEC_TO_TIME(LEAST($max, $start + $interval))";
3227
MKDEBUG && _d($sql);
3228
return $dbh->selectrow_array($sql);
3229
}
3230
3231
sub range_date {
3232
my ( $self, $dbh, $start, $interval, $max ) = @_;
3233
my $sql = "SELECT FROM_DAYS($start), FROM_DAYS(LEAST($max, $start + $interval))";
3234
MKDEBUG && _d($sql);
3235
return $dbh->selectrow_array($sql);
3236
}
3237
3238
sub range_datetime {
3239
my ( $self, $dbh, $start, $interval, $max ) = @_;
3240
my $sql = "SELECT DATE_ADD('$self->{EPOCH}', INTERVAL $start SECOND), "
3241
. "DATE_ADD('$self->{EPOCH}', INTERVAL LEAST($max, $start + $interval) SECOND)";
3242
MKDEBUG && _d($sql);
3243
return $dbh->selectrow_array($sql);
3244
}
3245
3246
sub range_timestamp {
3247
my ( $self, $dbh, $start, $interval, $max ) = @_;
3248
my $sql = "SELECT FROM_UNIXTIME($start), FROM_UNIXTIME(LEAST($max, $start + $interval))";
3249
MKDEBUG && _d($sql);
3250
return $dbh->selectrow_array($sql);
3251
}
3252
3253
sub timestampdiff {
3254
my ( $self, $dbh, $time ) = @_;
3255
my $sql = "SELECT (COALESCE(TO_DAYS('$time'), 0) * 86400 + TIME_TO_SEC('$time')) "
3256
. "- TO_DAYS('$self->{EPOCH} 00:00:00') * 86400";
3257
MKDEBUG && _d($sql);
3258
my ( $diff ) = $dbh->selectrow_array($sql);
3259
$sql = "SELECT DATE_ADD('$self->{EPOCH}', INTERVAL $diff SECOND)";
3260
MKDEBUG && _d($sql);
3261
my ( $check ) = $dbh->selectrow_array($sql);
3262
die <<" EOF"
3263
Incorrect datetime math: given $time, calculated $diff but checked to $check.
3264
This could be due to a version of MySQL that overflows on large interval
3265
values to DATE_ADD(), or the given datetime is not a valid date. If not,
3266
please report this as a bug.
3267
EOF
3268
unless $check eq $time;
3269
return $diff;
3270
}
3271
3272
3273
3274
3275
sub get_valid_end_points {
3276
my ( $self, %args ) = @_;
3277
my @required_args = qw(dbh db_tbl col col_type);
3278
foreach my $arg ( @required_args ) {
3279
die "I need a $arg argument" unless $args{$arg};
3280
}
3281
my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
3282
my ($real_min, $real_max) = @args{qw(min max)};
3283
3284
my $err_fmt = "Error finding a valid %s value for table $db_tbl on "
3285
. "column $col. The real %s value %s is invalid and "
3286
. "no other valid values were found. Verify that the table "
3287
. "has at least one valid value for this column"
3288
. ($args{where} ? " where $args{where}." : ".");
3289
3290
my $valid_min = $real_min;
3291
if ( defined $valid_min ) {
3292
MKDEBUG && _d("Validating min end point:", $real_min);
3293
$valid_min = $self->_get_valid_end_point(
3294
%args,
3295
val => $real_min,
3296
endpoint => 'min',
3297
);
3298
die sprintf($err_fmt, 'minimum', 'minimum',
3299
(defined $real_min ? $real_min : "NULL"))
3300
unless defined $valid_min;
3301
}
3302
3303
my $valid_max = $real_max;
3304
if ( defined $valid_max ) {
3305
MKDEBUG && _d("Validating max end point:", $real_min);
3306
$valid_max = $self->_get_valid_end_point(
3307
%args,
3308
val => $real_max,
3309
endpoint => 'max',
3310
);
3311
die sprintf($err_fmt, 'maximum', 'maximum',
3312
(defined $real_max ? $real_max : "NULL"))
3313
unless defined $valid_max;
3314
}
3315
3316
return $valid_min, $valid_max;
3317
}
3318
3319
sub _get_valid_end_point {
3320
my ( $self, %args ) = @_;
3321
my @required_args = qw(dbh db_tbl col col_type);
3322
foreach my $arg ( @required_args ) {
3323
die "I need a $arg argument" unless $args{$arg};
3324
}
3325
my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
3326
my $val = $args{val};
3327
3328
return $val unless defined $val;
3329
3330
my $validate = $col_type =~ m/time|date/ ? \&_validate_temporal_value
3331
: undef;
3332
3333
if ( !$validate ) {
3334
MKDEBUG && _d("No validator for", $col_type, "values");
3335
return $val;
3336
}
3337
3338
return $val if defined $validate->($dbh, $val);
3339
3340
MKDEBUG && _d("Value is invalid, getting first valid value");
3341
$val = $self->get_first_valid_value(
3342
%args,
3343
val => $val,
3344
validate => $validate,
3345
);
3346
3347
return $val;
3348
}
3349
3350
sub get_first_valid_value {
3351
my ( $self, %args ) = @_;
3352
my @required_args = qw(dbh db_tbl col validate endpoint);
3353
foreach my $arg ( @required_args ) {
3354
die "I need a $arg argument" unless $args{$arg};
3355
}
3356
my ($dbh, $db_tbl, $col, $validate, $endpoint) = @args{@required_args};
3357
my $tries = defined $args{tries} ? $args{tries} : 5;
3358
my $val = $args{val};
3359
3360
return unless defined $val;
3361
3362
my $cmp = $endpoint =~ m/min/i ? '>'
3363
: $endpoint =~ m/max/i ? '<'
3364
: die "Invalid endpoint arg: $endpoint";
3365
my $sql = "SELECT $col FROM $db_tbl "
3366
. ($args{index_hint} ? "$args{index_hint} " : "")
3367
. "WHERE $col $cmp ? AND $col IS NOT NULL "
3368
. ($args{where} ? "AND ($args{where}) " : "")
3369
. "ORDER BY $col LIMIT 1";
3370
MKDEBUG && _d($dbh, $sql);
3371
my $sth = $dbh->prepare($sql);
3372
3373
my $last_val = $val;
3374
while ( $tries-- ) {
3375
$sth->execute($last_val);
3376
my ($next_val) = $sth->fetchrow_array();
3377
MKDEBUG && _d('Next value:', $next_val, '; tries left:', $tries);
3378
if ( !defined $next_val ) {
3379
MKDEBUG && _d('No more rows in table');
3380
last;
3381
}
3382
if ( defined $validate->($dbh, $next_val) ) {
3383
MKDEBUG && _d('First valid value:', $next_val);
3384
$sth->finish();
3385
return $next_val;
3386
}
3387
$last_val = $next_val;
3388
}
3389
$sth->finish();
3390
$val = undef; # no valid value found
3391
3392
return $val;
3393
}
3394
3395
sub _validate_temporal_value {
3396
my ( $dbh, $val ) = @_;
3397
my $sql = "SELECT IF(TIME_FORMAT(?,'%H:%i:%s')=?, TIME_TO_SEC(?), TO_DAYS(?))";
3398
my $res;
3399
eval {
3400
MKDEBUG && _d($dbh, $sql, $val);
3401
my $sth = $dbh->prepare($sql);
3402
$sth->execute($val, $val, $val, $val);
3403
($res) = $sth->fetchrow_array();
3404
$sth->finish();
3405
};
3406
if ( $EVAL_ERROR ) {
3407
MKDEBUG && _d($EVAL_ERROR);
3408
}
3409
return $res;
3410
}
3411
3412
sub get_nonzero_value {
3413
my ( $self, %args ) = @_;
3414
my @required_args = qw(dbh db_tbl col col_type);
3415
foreach my $arg ( @required_args ) {
3416
die "I need a $arg argument" unless $args{$arg};
3417
}
3418
my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
3419
my $tries = defined $args{tries} ? $args{tries} : 5;
3420
my $val = $args{val};
3421
3422
my $is_nonzero = $col_type =~ m/time|date/ ? \&_validate_temporal_value
3423
: sub { return $_[1]; };
3424
3425
if ( !$is_nonzero->($dbh, $val) ) { # quasi-double-negative, sorry
3426
MKDEBUG && _d('Discarding zero value:', $val);
3427
my $sql = "SELECT $col FROM $db_tbl "
3428
. ($args{index_hint} ? "$args{index_hint} " : "")
3429
. "WHERE $col > ? AND $col IS NOT NULL "
3430
. ($args{where} ? "AND ($args{where}) " : '')
3431
. "ORDER BY $col LIMIT 1";
3432
MKDEBUG && _d($sql);
3433
my $sth = $dbh->prepare($sql);
3434
3435
my $last_val = $val;
3436
while ( $tries-- ) {
3437
$sth->execute($last_val);
3438
my ($next_val) = $sth->fetchrow_array();
3439
if ( $is_nonzero->($dbh, $next_val) ) {
3440
MKDEBUG && _d('First non-zero value:', $next_val);
3441
$sth->finish();
3442
return $next_val;
3443
}
3444
$last_val = $next_val;
3445
}
3446
$sth->finish();
3447
$val = undef; # no non-zero value found
3448
}
3449
3450
return $val;
3451
}
3452
3453
sub base_count {
3454
my ( $self, %args ) = @_;
3455
my @required_args = qw(count_to base symbols);
3456
foreach my $arg ( @required_args ) {
3457
die "I need a $arg argument" unless defined $args{$arg};
3458
}
3459
my ($n, $base, $symbols) = @args{@required_args};
3460
3461
return $symbols->[0] if $n == 0;
3462
3463
my $highest_power = floor(log($n)/log($base));
3464
if ( $highest_power == 0 ){
3465
return $symbols->[$n];
3466
}
3467
3468
my @base_powers;
3469
for my $power ( 0..$highest_power ) {
3470
push @base_powers, ($base**$power) || 1;
3471
}
3472
3473
my @base_multiples;
3474
foreach my $base_power ( reverse @base_powers ) {
3475
my $multiples = floor($n / $base_power);
3476
push @base_multiples, $multiples;
3477
$n -= $multiples * $base_power;
3478
}
3479
3480
return join('', map { $symbols->[$_] } @base_multiples);
3481
}
3482
3483
sub _d {
3484
my ($package, undef, $line) = caller 0;
3485
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
3486
map { defined $_ ? $_ : 'undef' }
3487
@_;
3488
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
3489
}
3490
3491
1;
3492
}
3493
# ###########################################################################
3494
# End TableChunker package
1520
sub _d {
1521
my ($package, undef, $line) = caller 0;
1522
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
1523
map { defined $_ ? $_ : 'undef' }
1524
@_;
1525
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1526
}
1527
1528
1;
1529
}
1530
# ###########################################################################
1531
# End Cxn package
3495
1532
# ###########################################################################
3496
1533
3497
1534
# ###########################################################################
3508
1545
use strict;
3509
1546
use warnings FATAL => 'all';
3510
1547
use English qw(-no_match_vars);
3511
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
1548
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
3512
1549
3513
1550
sub new {
3514
1551
my ( $class, %args ) = @_;
3565
1602
return $db ? "$db.$tbl" : $tbl;
3566
1603
}
3567
1604
1605
sub serialize_list {
1606
my ( $self, @args ) = @_;
1607
return unless @args;
1608
1609
return $args[0] if @args == 1 && !defined $args[0];
1610
1611
die "Cannot serialize multiple values with undef/NULL"
1612
if grep { !defined $_ } @args;
1613
1614
return join ',', map { quotemeta } @args;
1615
}
1616
1617
sub deserialize_list {
1618
my ( $self, $string ) = @_;
1619
return $string unless defined $string;
1620
my @escaped_parts = $string =~ /
1621
\G # Start of string, or end of previous match.
1622
( # Each of these is an element in the original list.
1623
[^\\,]* # Anything not a backslash or a comma
1624
(?: # When we get here, we found one of the above.
1625
\\. # A backslash followed by something so we can continue
1626
[^\\,]* # Same as above.
1627
)* # Repeat zero of more times.
1628
)
1629
, # Comma dividing elements
1630
/sxgc;
1631
1632
push @escaped_parts, pos($string) ? substr( $string, pos($string) ) : $string;
1633
1634
my @unescaped_parts = map {
1635
my $part = $_;
1636
1637
my $char_class = utf8::is_utf8($part) # If it's a UTF-8 string,
1638
? qr/(?=\p{ASCII})\W/ # We only care about non-word
1639
: qr/(?=\p{ASCII})\W|[\x{80}-\x{FF}]/; # Otherwise,
1640
$part =~ s/\\($char_class)/$1/g;
1641
$part;
1642
} @escaped_parts;
1643
1644
return @unescaped_parts;
1645
}
1646
3568
1647
1;
3569
1648
}
3570
1649
# ###########################################################################
3572
1651
# ###########################################################################
3573
1652
3574
1653
# ###########################################################################
1654
# VersionParser package
1655
# This package is a copy without comments from the original. The original
1656
# with comments and its test file can be found in the Bazaar repository at,
1657
# lib/VersionParser.pm
1658
# t/lib/VersionParser.t
1659
# See https://launchpad.net/percona-toolkit for more information.
1660
# ###########################################################################
1661
{
1662
package VersionParser;
1663
1664
use strict;
1665
use warnings FATAL => 'all';
1666
use English qw(-no_match_vars);
1667
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
1668
1669
sub new {
1670
my ( $class ) = @_;
1671
bless {}, $class;
1672
}
1673
1674
sub parse {
1675
my ( $self, $str ) = @_;
1676
my $result = sprintf('%03d%03d%03d', $str =~ m/(\d+)/g);
1677
PTDEBUG && _d($str, 'parses to', $result);
1678
return $result;
1679
}
1680
1681
sub version_ge {
1682
my ( $self, $dbh, $target ) = @_;
1683
if ( !$self->{$dbh} ) {
1684
$self->{$dbh} = $self->parse(
1685
$dbh->selectrow_array('SELECT VERSION()'));
1686
}
1687
my $result = $self->{$dbh} ge $self->parse($target) ? 1 : 0;
1688
PTDEBUG && _d($self->{$dbh}, 'ge', $target, ':', $result);
1689
return $result;
1690
}
1691
1692
sub innodb_version {
1693
my ( $self, $dbh ) = @_;
1694
return unless $dbh;
1695
my $innodb_version = "NO";
1696
1697
my ($innodb) =
1698
grep { $_->{engine} =~ m/InnoDB/i }
1699
map {
1700
my %hash;
1701
@hash{ map { lc $_ } keys %$_ } = values %$_;
1702
\%hash;
1703
}
1704
@{ $dbh->selectall_arrayref("SHOW ENGINES", {Slice=>{}}) };
1705
if ( $innodb ) {
1706
PTDEBUG && _d("InnoDB support:", $innodb->{support});
1707
if ( $innodb->{support} =~ m/YES|DEFAULT/i ) {
1708
my $vars = $dbh->selectrow_hashref(
1709
"SHOW VARIABLES LIKE 'innodb_version'");
1710
$innodb_version = !$vars ? "BUILTIN"
1711
: ($vars->{Value} || $vars->{value});
1712
}
1713
else {
1714
$innodb_version = $innodb->{support}; # probably DISABLED or NO
1715
}
1716
}
1717
1718
PTDEBUG && _d("InnoDB version:", $innodb_version);
1719
return $innodb_version;
1720
}
1721
1722
sub _d {
1723
my ($package, undef, $line) = caller 0;
1724
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
1725
map { defined $_ ? $_ : 'undef' }
1726
@_;
1727
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1728
}
1729
1730
1;
1731
}
1732
# ###########################################################################
1733
# End VersionParser package
1734
# ###########################################################################
1735
1736
# ###########################################################################
1737
# TableParser package
1738
# This package is a copy without comments from the original. The original
1739
# with comments and its test file can be found in the Bazaar repository at,
1740
# lib/TableParser.pm
1741
# t/lib/TableParser.t
1742
# See https://launchpad.net/percona-toolkit for more information.
1743
# ###########################################################################
1744
{
1745
package TableParser;
1746
1747
use strict;
1748
use warnings FATAL => 'all';
1749
use English qw(-no_match_vars);
1750
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
1751
1752
use Data::Dumper;
1753
$Data::Dumper::Indent = 1;
1754
$Data::Dumper::Sortkeys = 1;
1755
$Data::Dumper::Quotekeys = 0;
1756
1757
sub new {
1758
my ( $class, %args ) = @_;
1759
my @required_args = qw(Quoter);
1760
foreach my $arg ( @required_args ) {
1761
die "I need a $arg argument" unless $args{$arg};
1762
}
1763
my $self = { %args };
1764
return bless $self, $class;
1765
}
1766
1767
sub get_create_table {
1768
my ( $self, $dbh, $db, $tbl ) = @_;
1769
die "I need a dbh parameter" unless $dbh;
1770
die "I need a db parameter" unless $db;
1771
die "I need a tbl parameter" unless $tbl;
1772
my $q = $self->{Quoter};
1773
1774
my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, '
1775
. q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), }
1776
. '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, '
1777
. '@@SQL_QUOTE_SHOW_CREATE := 1 */';
1778
PTDEBUG && _d($sql);
1779
eval { $dbh->do($sql); };
1780
PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
1781
1782
$sql = 'USE ' . $q->quote($db);
1783
PTDEBUG && _d($dbh, $sql);
1784
$dbh->do($sql);
1785
1786
$sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl);
1787
PTDEBUG && _d($sql);
1788
my $href;
1789
eval { $href = $dbh->selectrow_hashref($sql); };
1790
if ( $EVAL_ERROR ) {
1791
PTDEBUG && _d($EVAL_ERROR);
1792
return;
1793
}
1794
1795
$sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, '
1796
. '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */';
1797
PTDEBUG && _d($sql);
1798
$dbh->do($sql);
1799
1800
my ($key) = grep { m/create table/i } keys %$href;
1801
if ( $key ) {
1802
PTDEBUG && _d('This table is a base table');
1803
$href->{$key} =~ s/\b[ ]{2,}/ /g;
1804
$href->{$key} .= "\n";
1805
}
1806
else {
1807
PTDEBUG && _d('This table is a view');
1808
($key) = grep { m/create view/i } keys %$href;
1809
}
1810
1811
return $href->{$key};
1812
}
1813
1814
sub parse {
1815
my ( $self, $ddl, $opts ) = @_;
1816
return unless $ddl;
1817
1818
if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {
1819
die "Cannot parse table definition; is ANSI quoting "
1820
. "enabled or SQL_QUOTE_SHOW_CREATE disabled?";
1821
}
1822
1823
my ($name) = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/;
1824
(undef, $name) = $self->{Quoter}->split_unquote($name) if $name;
1825
1826
$ddl =~ s/(`[^`]+`)/\L$1/g;
1827
1828
my $engine = $self->get_engine($ddl);
1829
1830
my @defs = $ddl =~ m/^(\s+`.*?),?$/gm;
1831
my @cols = map { $_ =~ m/`([^`]+)`/ } @defs;
1832
PTDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols));
1833
1834
my %def_for;
1835
@def_for{@cols} = @defs;
1836
1837
my (@nums, @null);
1838
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
1839
foreach my $col ( @cols ) {
1840
my $def = $def_for{$col};
1841
my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/;
1842
die "Can't determine column type for $def" unless $type;
1843
$type_for{$col} = $type;
1844
if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) {
1845
push @nums, $col;
1846
$is_numeric{$col} = 1;
1847
}
1848
if ( $def !~ m/NOT NULL/ ) {
1849
push @null, $col;
1850
$is_nullable{$col} = 1;
1851
}
1852
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
1853
}
1854
1855
my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable);
1856
1857
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
1858
1859
return {
1860
name => $name,
1861
cols => \@cols,
1862
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
1863
is_col => { map { $_ => 1 } @cols },
1864
null_cols => \@null,
1865
is_nullable => \%is_nullable,
1866
is_autoinc => \%is_autoinc,
1867
clustered_key => $clustered_key,
1868
keys => $keys,
1869
defs => \%def_for,
1870
numeric_cols => \@nums,
1871
is_numeric => \%is_numeric,
1872
engine => $engine,
1873
type_for => \%type_for,
1874
charset => $charset,
1875
};
1876
}
1877
1878
sub sort_indexes {
1879
my ( $self, $tbl ) = @_;
1880
1881
my @indexes
1882
= sort {
1883
(($a ne 'PRIMARY') <=> ($b ne 'PRIMARY'))
1884
|| ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} )
1885
|| ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} )
1886
|| ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) )
1887
}
1888
grep {
1889
$tbl->{keys}->{$_}->{type} eq 'BTREE'
1890
}
1891
sort keys %{$tbl->{keys}};
1892
1893
PTDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes));
1894
return @indexes;
1895
}
1896
1897
sub find_best_index {
1898
my ( $self, $tbl, $index ) = @_;
1899
my $best;
1900
if ( $index ) {
1901
($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}};
1902
}
1903
if ( !$best ) {
1904
if ( $index ) {
1905
die "Index '$index' does not exist in table";
1906
}
1907
else {
1908
($best) = $self->sort_indexes($tbl);
1909
}
1910
}
1911
PTDEBUG && _d('Best index found is', $best);
1912
return $best;
1913
}
1914
1915
sub find_possible_keys {
1916
my ( $self, $dbh, $database, $table, $quoter, $where ) = @_;
1917
return () unless $where;
1918
my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table)
1919
. ' WHERE ' . $where;
1920
PTDEBUG && _d($sql);
1921
my $expl = $dbh->selectrow_hashref($sql);
1922
$expl = { map { lc($_) => $expl->{$_} } keys %$expl };
1923
if ( $expl->{possible_keys} ) {
1924
PTDEBUG && _d('possible_keys =', $expl->{possible_keys});
1925
my @candidates = split(',', $expl->{possible_keys});
1926
my %possible = map { $_ => 1 } @candidates;
1927
if ( $expl->{key} ) {
1928
PTDEBUG && _d('MySQL chose', $expl->{key});
1929
unshift @candidates, grep { $possible{$_} } split(',', $expl->{key});
1930
PTDEBUG && _d('Before deduping:', join(', ', @candidates));
1931
my %seen;
1932
@candidates = grep { !$seen{$_}++ } @candidates;
1933
}
1934
PTDEBUG && _d('Final list:', join(', ', @candidates));
1935
return @candidates;
1936
}
1937
else {
1938
PTDEBUG && _d('No keys in possible_keys');
1939
return ();
1940
}
1941
}
1942
1943
sub check_table {
1944
my ( $self, %args ) = @_;
1945
my @required_args = qw(dbh db tbl);
1946
foreach my $arg ( @required_args ) {
1947
die "I need a $arg argument" unless $args{$arg};
1948
}
1949
my ($dbh, $db, $tbl) = @args{@required_args};
1950
my $q = $self->{Quoter};
1951
my $db_tbl = $q->quote($db, $tbl);
1952
PTDEBUG && _d('Checking', $db_tbl);
1953
1954
my $sql = "SHOW TABLES FROM " . $q->quote($db)
1955
. ' LIKE ' . $q->literal_like($tbl);
1956
PTDEBUG && _d($sql);
1957
my $row;
1958
eval {
1959
$row = $dbh->selectrow_arrayref($sql);
1960
};
1961
if ( $EVAL_ERROR ) {
1962
PTDEBUG && _d($EVAL_ERROR);
1963
return 0;
1964
}
1965
if ( !$row->[0] || $row->[0] ne $tbl ) {
1966
PTDEBUG && _d('Table does not exist');
1967
return 0;
1968
}
1969
1970
PTDEBUG && _d('Table exists; no privs to check');
1971
return 1 unless $args{all_privs};
1972
1973
$sql = "SHOW FULL COLUMNS FROM $db_tbl";
1974
PTDEBUG && _d($sql);
1975
eval {
1976
$row = $dbh->selectrow_hashref($sql);
1977
};
1978
if ( $EVAL_ERROR ) {
1979
PTDEBUG && _d($EVAL_ERROR);
1980
return 0;
1981
}
1982
if ( !scalar keys %$row ) {
1983
PTDEBUG && _d('Table has no columns:', Dumper($row));
1984
return 0;
1985
}
1986
my $privs = $row->{privileges} || $row->{Privileges};
1987
1988
$sql = "DELETE FROM $db_tbl LIMIT 0";
1989
PTDEBUG && _d($sql);
1990
eval {
1991
$dbh->do($sql);
1992
};
1993
my $can_delete = $EVAL_ERROR ? 0 : 1;
1994
1995
PTDEBUG && _d('User privs on', $db_tbl, ':', $privs,
1996
($can_delete ? 'delete' : ''));
1997
1998
if ( !($privs =~ m/select/ && $privs =~ m/insert/ && $privs =~ m/update/
1999
&& $can_delete) ) {
2000
PTDEBUG && _d('User does not have all privs');
2001
return 0;
2002
}
2003
2004
PTDEBUG && _d('User has all privs');
2005
return 1;
2006
}
2007
2008
sub get_engine {
2009
my ( $self, $ddl, $opts ) = @_;
2010
my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/;
2011
PTDEBUG && _d('Storage engine:', $engine);
2012
return $engine || undef;
2013
}
2014
2015
sub get_keys {
2016
my ( $self, $ddl, $opts, $is_nullable ) = @_;
2017
my $engine = $self->get_engine($ddl);
2018
my $keys = {};
2019
my $clustered_key = undef;
2020
2021
KEY:
2022
foreach my $key ( $ddl =~ m/^ ((?:[A-Z]+ )?KEY .*)$/gm ) {
2023
2024
next KEY if $key =~ m/FOREIGN/;
2025
2026
my $key_ddl = $key;
2027
PTDEBUG && _d('Parsed key:', $key_ddl);
2028
2029
if ( $engine !~ m/MEMORY|HEAP/ ) {
2030
$key =~ s/USING HASH/USING BTREE/;
2031
}
2032
2033
my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/;
2034
my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/;
2035
$type = $type || $special || 'BTREE';
2036
if ( $opts->{mysql_version} && $opts->{mysql_version} lt '004001000'
2037
&& $engine =~ m/HEAP|MEMORY/i )
2038
{
2039
$type = 'HASH'; # MySQL pre-4.1 supports only HASH indexes on HEAP
2040
}
2041
2042
my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/;
2043
my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0;
2044
my @cols;
2045
my @col_prefixes;
2046
foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) {
2047
my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/;
2048
push @cols, $name;
2049
push @col_prefixes, $prefix;
2050
}
2051
$name =~ s/`//g;
2052
2053
PTDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols));
2054
2055
$keys->{$name} = {
2056
name => $name,
2057
type => $type,
2058
colnames => $cols,
2059
cols => \@cols,
2060
col_prefixes => \@col_prefixes,
2061
is_unique => $unique,
2062
is_nullable => scalar(grep { $is_nullable->{$_} } @cols),
2063
is_col => { map { $_ => 1 } @cols },
2064
ddl => $key_ddl,
2065
};
2066
2067
if ( $engine =~ m/InnoDB/i && !$clustered_key ) {
2068
my $this_key = $keys->{$name};
2069
if ( $this_key->{name} eq 'PRIMARY' ) {
2070
$clustered_key = 'PRIMARY';
2071
}
2072
elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) {
2073
$clustered_key = $this_key->{name};
2074
}
2075
PTDEBUG && $clustered_key && _d('This key is the clustered key');
2076
}
2077
}
2078
2079
return $keys, $clustered_key;
2080
}
2081
2082
sub get_fks {
2083
my ( $self, $ddl, $opts ) = @_;
2084
my $q = $self->{Quoter};
2085
my $fks = {};
2086
2087
foreach my $fk (
2088
$ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg )
2089
{
2090
my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/;
2091
my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/;
2092
my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/;
2093
2094
my ($db, $tbl) = $q->split_unquote($parent, $opts->{database});
2095
my %parent_tbl = (tbl => $tbl);
2096
$parent_tbl{db} = $db if $db;
2097
2098
if ( $parent !~ m/\./ && $opts->{database} ) {
2099
$parent = $q->quote($opts->{database}) . ".$parent";
2100
}
2101
2102
$fks->{$name} = {
2103
name => $name,
2104
colnames => $cols,
2105
cols => [ map { s/[ `]+//g; $_; } split(',', $cols) ],
2106
parent_tbl => \%parent_tbl,
2107
parent_tblname => $parent,
2108
parent_cols => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ],
2109
parent_colnames=> $parent_cols,
2110
ddl => $fk,
2111
};
2112
}
2113
2114
return $fks;
2115
}
2116
2117
sub remove_auto_increment {
2118
my ( $self, $ddl ) = @_;
2119
$ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m;
2120
return $ddl;
2121
}
2122
2123
sub get_table_status {
2124
my ( $self, $dbh, $db, $like ) = @_;
2125
my $q = $self->{Quoter};
2126
my $sql = "SHOW TABLE STATUS FROM " . $q->quote($db);
2127
my @params;
2128
if ( $like ) {
2129
$sql .= ' LIKE ?';
2130
push @params, $like;
2131
}
2132
PTDEBUG && _d($sql, @params);
2133
my $sth = $dbh->prepare($sql);
2134
eval { $sth->execute(@params); };
2135
if ($EVAL_ERROR) {
2136
PTDEBUG && _d($EVAL_ERROR);
2137
return;
2138
}
2139
my @tables = @{$sth->fetchall_arrayref({})};
2140
@tables = map {
2141
my %tbl; # Make a copy with lowercased keys
2142
@tbl{ map { lc $_ } keys %$_ } = values %$_;
2143
$tbl{engine} ||= $tbl{type} || $tbl{comment};
2144
delete $tbl{type};
2145
\%tbl;
2146
} @tables;
2147
return @tables;
2148
}
2149
2150
sub _d {
2151
my ($package, undef, $line) = caller 0;
2152
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
2153
map { defined $_ ? $_ : 'undef' }
2154
@_;
2155
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
2156
}
2157
2158
1;
2159
}
2160
# ###########################################################################
2161
# End TableParser package
2162
# ###########################################################################
2163
2164
# ###########################################################################
2165
# TableNibbler package
2166
# This package is a copy without comments from the original. The original
2167
# with comments and its test file can be found in the Bazaar repository at,
2168
# lib/TableNibbler.pm
2169
# t/lib/TableNibbler.t
2170
# See https://launchpad.net/percona-toolkit for more information.
2171
# ###########################################################################
2172
{
2173
package TableNibbler;
2174
2175
use strict;
2176
use warnings FATAL => 'all';
2177
use English qw(-no_match_vars);
2178
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
2179
2180
sub new {
2181
my ( $class, %args ) = @_;
2182
my @required_args = qw(TableParser Quoter);
2183
foreach my $arg ( @required_args ) {
2184
die "I need a $arg argument" unless $args{$arg};
2185
}
2186
my $self = { %args };
2187
return bless $self, $class;
2188
}
2189
2190
sub generate_asc_stmt {
2191
my ( $self, %args ) = @_;
2192
my @required_args = qw(tbl_struct index);
2193
foreach my $arg ( @required_args ) {
2194
die "I need a $arg argument" unless defined $args{$arg};
2195
}
2196
my ($tbl_struct, $index) = @args{@required_args};
2197
my @cols = $args{cols} ? @{$args{cols}} : @{$tbl_struct->{cols}};
2198
my $q = $self->{Quoter};
2199
2200
die "Index '$index' does not exist in table"
2201
unless exists $tbl_struct->{keys}->{$index};
2202
PTDEBUG && _d('Will ascend index', $index);
2203
2204
my @asc_cols = @{$tbl_struct->{keys}->{$index}->{cols}};
2205
if ( $args{asc_first} ) {
2206
@asc_cols = $asc_cols[0];
2207
PTDEBUG && _d('Ascending only first column');
2208
}
2209
PTDEBUG && _d('Will ascend columns', join(', ', @asc_cols));
2210
2211
my @asc_slice;
2212
my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols };
2213
foreach my $col ( @asc_cols ) {
2214
if ( !exists $col_posn{$col} ) {
2215
push @cols, $col;
2216
$col_posn{$col} = $#cols;
2217
}
2218
push @asc_slice, $col_posn{$col};
2219
}
2220
PTDEBUG && _d('Will ascend, in ordinal position:', join(', ', @asc_slice));
2221
2222
my $asc_stmt = {
2223
cols => \@cols,
2224
index => $index,
2225
where => '',
2226
slice => [],
2227
scols => [],
2228
};
2229
2230
if ( @asc_slice ) {
2231
my $cmp_where;
2232
foreach my $cmp ( qw(< <= >= >) ) {
2233
$cmp_where = $self->generate_cmp_where(
2234
type => $cmp,
2235
slice => \@asc_slice,
2236
cols => \@cols,
2237
quoter => $q,
2238
is_nullable => $tbl_struct->{is_nullable},
2239
);
2240
$asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
2241
}
2242
my $cmp = $args{asc_only} ? '>' : '>=';
2243
$asc_stmt->{where} = $asc_stmt->{boundaries}->{$cmp};
2244
$asc_stmt->{slice} = $cmp_where->{slice};
2245
$asc_stmt->{scols} = $cmp_where->{scols};
2246
}
2247
2248
return $asc_stmt;
2249
}
2250
2251
sub generate_cmp_where {
2252
my ( $self, %args ) = @_;
2253
foreach my $arg ( qw(type slice cols is_nullable) ) {
2254
die "I need a $arg arg" unless defined $args{$arg};
2255
}
2256
my @slice = @{$args{slice}};
2257
my @cols = @{$args{cols}};
2258
my $is_nullable = $args{is_nullable};
2259
my $type = $args{type};
2260
my $q = $self->{Quoter};
2261
2262
(my $cmp = $type) =~ s/=//;
2263
2264
my @r_slice; # Resulting slice columns, by ordinal
2265
my @r_scols; # Ditto, by name
2266
2267
my @clauses;
2268
foreach my $i ( 0 .. $#slice ) {
2269
my @clause;
2270
2271
foreach my $j ( 0 .. $i - 1 ) {
2272
my $ord = $slice[$j];
2273
my $col = $cols[$ord];
2274
my $quo = $q->quote($col);
2275
if ( $is_nullable->{$col} ) {
2276
push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))";
2277
push @r_slice, $ord, $ord;
2278
push @r_scols, $col, $col;
2279
}
2280
else {
2281
push @clause, "$quo = ?";
2282
push @r_slice, $ord;
2283
push @r_scols, $col;
2284
}
2285
}
2286
2287
my $ord = $slice[$i];
2288
my $col = $cols[$ord];
2289
my $quo = $q->quote($col);
2290
my $end = $i == $#slice; # Last clause of the whole group.
2291
if ( $is_nullable->{$col} ) {
2292
if ( $type =~ m/=/ && $end ) {
2293
push @clause, "(? IS NULL OR $quo $type ?)";
2294
}
2295
elsif ( $type =~ m/>/ ) {
2296
push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))";
2297
}
2298
else { # If $type =~ m/</ ) {
2299
push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))";
2300
}
2301
push @r_slice, $ord, $ord;
2302
push @r_scols, $col, $col;
2303
}
2304
else {
2305
push @r_slice, $ord;
2306
push @r_scols, $col;
2307
push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?");
2308
}
2309
2310
push @clauses, '(' . join(' AND ', @clause) . ')';
2311
}
2312
my $result = '(' . join(' OR ', @clauses) . ')';
2313
my $where = {
2314
slice => \@r_slice,
2315
scols => \@r_scols,
2316
where => $result,
2317
};
2318
return $where;
2319
}
2320
2321
sub generate_del_stmt {
2322
my ( $self, %args ) = @_;
2323
2324
my $tbl = $args{tbl_struct};
2325
my @cols = $args{cols} ? @{$args{cols}} : ();
2326
my $tp = $self->{TableParser};
2327
my $q = $self->{Quoter};
2328
2329
my @del_cols;
2330
my @del_slice;
2331
2332
my $index = $tp->find_best_index($tbl, $args{index});
2333
die "Cannot find an ascendable index in table" unless $index;
2334
2335
if ( $index ) {
2336
@del_cols = @{$tbl->{keys}->{$index}->{cols}};
2337
}
2338
else {
2339
@del_cols = @{$tbl->{cols}};
2340
}
2341
PTDEBUG && _d('Columns needed for DELETE:', join(', ', @del_cols));
2342
2343
my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols };
2344
foreach my $col ( @del_cols ) {
2345
if ( !exists $col_posn{$col} ) {
2346
push @cols, $col;
2347
$col_posn{$col} = $#cols;
2348
}
2349
push @del_slice, $col_posn{$col};
2350
}
2351
PTDEBUG && _d('Ordinals needed for DELETE:', join(', ', @del_slice));
2352
2353
my $del_stmt = {
2354
cols => \@cols,
2355
index => $index,
2356
where => '',
2357
slice => [],
2358
scols => [],
2359
};
2360
2361
my @clauses;
2362
foreach my $i ( 0 .. $#del_slice ) {
2363
my $ord = $del_slice[$i];
2364
my $col = $cols[$ord];
2365
my $quo = $q->quote($col);
2366
if ( $tbl->{is_nullable}->{$col} ) {
2367
push @clauses, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))";
2368
push @{$del_stmt->{slice}}, $ord, $ord;
2369
push @{$del_stmt->{scols}}, $col, $col;
2370
}
2371
else {
2372
push @clauses, "$quo = ?";
2373
push @{$del_stmt->{slice}}, $ord;
2374
push @{$del_stmt->{scols}}, $col;
2375
}
2376
}
2377
2378
$del_stmt->{where} = '(' . join(' AND ', @clauses) . ')';
2379
2380
return $del_stmt;
2381
}
2382
2383
sub generate_ins_stmt {
2384
my ( $self, %args ) = @_;
2385
foreach my $arg ( qw(ins_tbl sel_cols) ) {
2386
die "I need a $arg argument" unless $args{$arg};
2387
}
2388
my $ins_tbl = $args{ins_tbl};
2389
my @sel_cols = @{$args{sel_cols}};
2390
2391
die "You didn't specify any SELECT columns" unless @sel_cols;
2392
2393
my @ins_cols;
2394
my @ins_slice;
2395
for my $i ( 0..$#sel_cols ) {
2396
next unless $ins_tbl->{is_col}->{$sel_cols[$i]};
2397
push @ins_cols, $sel_cols[$i];
2398
push @ins_slice, $i;
2399
}
2400
2401
return {
2402
cols => \@ins_cols,
2403
slice => \@ins_slice,
2404
};
2405
}
2406
2407
sub _d {
2408
my ($package, undef, $line) = caller 0;
2409
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
2410
map { defined $_ ? $_ : 'undef' }
2411
@_;
2412
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
2413
}
2414
2415
1;
2416
}
2417
# ###########################################################################
2418
# End TableNibbler package
2419
# ###########################################################################
2420
2421
# ###########################################################################
3575
2422
# MasterSlave package
3576
2423
# This package is a copy without comments from the original. The original
3577
2424
# with comments and its test file can be found in the Bazaar repository at,
3585
2432
use strict;
3586
2433
use warnings FATAL => 'all';
3587
2434
use English qw(-no_match_vars);
3588
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
2435
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
3589
2436
3590
2437
sub new {
3591
2438
my ( $class, %args ) = @_;
3596
2443
return bless $self, $class;
3597
2444
}
3598
2445
2446
sub get_slaves {
2447
my ($self, %args) = @_;
2448
my @required_args = qw(make_cxn OptionParser DSNParser Quoter);
2449
foreach my $arg ( @required_args ) {
2450
die "I need a $arg argument" unless $args{$arg};
2451
}
2452
my ($make_cxn, $o, $dp) = @args{@required_args};
2453
2454
my $slaves = [];
2455
my $method = $o->get('recursion-method');
2456
PTDEBUG && _d('Slave recursion method:', $method);
2457
if ( !$method || $method =~ m/processlist|hosts/i ) {
2458
my @required_args = qw(dbh dsn);
2459
foreach my $arg ( @required_args ) {
2460
die "I need a $arg argument" unless $args{$arg};
2461
}
2462
my ($dbh, $dsn) = @args{@required_args};
2463
$self->recurse_to_slaves(
2464
{ dbh => $dbh,
2465
dsn => $dsn,
2466
dsn_parser => $dp,
2467
recurse => $o->get('recurse'),
2468
method => $o->get('recursion-method'),
2469
callback => sub {
2470
my ( $dsn, $dbh, $level, $parent ) = @_;
2471
return unless $level;
2472
PTDEBUG && _d('Found slave:', $dp->as_string($dsn));
2473
push @$slaves, $make_cxn->(dsn => $dsn, dbh => $dbh);
2474
return;
2475
},
2476
}
2477
);
2478
}
2479
elsif ( $method =~ m/^dsn=/i ) {
2480
my ($dsn_table_dsn) = $method =~ m/^dsn=(.+)/i;
2481
$slaves = $self->get_cxn_from_dsn_table(
2482
%args,
2483
dsn_table_dsn => $dsn_table_dsn,
2484
);
2485
}
2486
else {
2487
die "Invalid --recursion-method: $method. Valid values are: "
2488
. "dsn=DSN, hosts, or processlist.\n";
2489
}
2490
2491
return $slaves;
2492
}
2493
3599
2494
sub recurse_to_slaves {
3600
2495
my ( $self, $args, $level ) = @_;
3601
2496
$level ||= 0;
3606
2501
eval {
3607
2502
$dbh = $args->{dbh} || $dp->get_dbh(
3608
2503
$dp->get_cxn_params($dsn), { AutoCommit => 1 });
3609
MKDEBUG && _d('Connected to', $dp->as_string($dsn));
2504
PTDEBUG && _d('Connected to', $dp->as_string($dsn));
3610
2505
};
3611
2506
if ( $EVAL_ERROR ) {
3612
2507
print STDERR "Cannot connect to ", $dp->as_string($dsn), "\n"
3615
2510
}
3616
2511
3617
2512
my $sql = 'SELECT @@SERVER_ID';
3618
MKDEBUG && _d($sql);
2513
PTDEBUG && _d($sql);
3619
2514
my ($id) = $dbh->selectrow_array($sql);
3620
MKDEBUG && _d('Working on server ID', $id);
2515
PTDEBUG && _d('Working on server ID', $id);
3621
2516
my $master_thinks_i_am = $dsn->{server_id};
3622
2517
if ( !defined $id
3623
2518
|| ( defined $master_thinks_i_am && $master_thinks_i_am != $id )
3624
2519
|| $args->{server_ids_seen}->{$id}++
3625
2520
) {
3626
MKDEBUG && _d('Server ID seen, or not what master said');
2521
PTDEBUG && _d('Server ID seen, or not what master said');
3627
2522
if ( $args->{skip_callback} ) {
3628
2523
$args->{skip_callback}->($dsn, $dbh, $level, $args->{parent});
3629
2524
}
3639
2534
$self->find_slave_hosts($dp, $dbh, $dsn, $args->{method});
3640
2535
3641
2536
foreach my $slave ( @slaves ) {
3642
MKDEBUG && _d('Recursing from',
2537
PTDEBUG && _d('Recursing from',
3643
2538
$dp->as_string($dsn), 'to', $dp->as_string($slave));
3644
2539
$self->recurse_to_slaves(
3645
2540
{ %$args, dsn => $slave, dbh => undef, parent => $dsn }, $level + 1 );
3657
2552
}
3658
2553
else {
3659
2554
if ( ($dsn->{P} || 3306) != 3306 ) {
3660
MKDEBUG && _d('Port number is non-standard; using only hosts method');
2555
PTDEBUG && _d('Port number is non-standard; using only hosts method');
3661
2556
@methods = qw(hosts);
3662
2557
}
3663
2558
}
3664
MKDEBUG && _d('Looking for slaves on', $dsn_parser->as_string($dsn),
2559
PTDEBUG && _d('Looking for slaves on', $dsn_parser->as_string($dsn),
3665
2560
'using methods', @methods);
3666
2561
3667
2562
my @slaves;
3668
2563
METHOD:
3669
2564
foreach my $method ( @methods ) {
3670
2565
my $find_slaves = "_find_slaves_by_$method";
3671
MKDEBUG && _d('Finding slaves with', $find_slaves);
2566
PTDEBUG && _d('Finding slaves with', $find_slaves);
3672
2567
@slaves = $self->$find_slaves($dsn_parser, $dbh, $dsn);
3673
2568
last METHOD if @slaves;
3674
2569
}
3675
2570
3676
MKDEBUG && _d('Found', scalar(@slaves), 'slaves');
2571
PTDEBUG && _d('Found', scalar(@slaves), 'slaves');
3677
2572
return @slaves;
3678
2573
}
3679
2574
3702
2597
3703
2598
my @slaves;
3704
2599
my $sql = 'SHOW SLAVE HOSTS';
3705
MKDEBUG && _d($dbh, $sql);
2600
PTDEBUG && _d($dbh, $sql);
3706
2601
@slaves = @{$dbh->selectall_arrayref($sql, { Slice => {} })};
3707
2602
3708
2603
if ( @slaves ) {
3709
MKDEBUG && _d('Found some SHOW SLAVE HOSTS info');
2604
PTDEBUG && _d('Found some SHOW SLAVE HOSTS info');
3710
2605
@slaves = map {
3711
2606
my %hash;
3712
2607
@hash{ map { lc $_ } keys %$_ } = values %$_;
3735
2630
$user =~ s/([^@]+)@(.+)/'$1'\@'$2'/;
3736
2631
}
3737
2632
my $sql = $show . $user;
3738
MKDEBUG && _d($dbh, $sql);
2633
PTDEBUG && _d($dbh, $sql);
3739
2634
3740
2635
my $proc;
3741
2636
eval {
3746
2641
if ( $EVAL_ERROR ) {
3747
2642
3748
2643
if ( $EVAL_ERROR =~ m/no such grant defined for user/ ) {
3749
MKDEBUG && _d('Retrying SHOW GRANTS without host; error:',
2644
PTDEBUG && _d('Retrying SHOW GRANTS without host; error:',
3750
2645
$EVAL_ERROR);
3751
2646
($user) = split('@', $user);
3752
2647
$sql = $show . $user;
3753
MKDEBUG && _d($sql);
2648
PTDEBUG && _d($sql);
3754
2649
eval {
3755
2650
$proc = grep {
3756
2651
m/ALL PRIVILEGES.*?\*\.\*|PROCESS/
3765
2660
}
3766
2661
3767
2662
$sql = 'SHOW PROCESSLIST';
3768
MKDEBUG && _d($dbh, $sql);
2663
PTDEBUG && _d($dbh, $sql);
3769
2664
grep { $_->{command} =~ m/Binlog Dump/i }
3770
2665
map { # Lowercase the column names
3771
2666
my %hash;
3825
2720
if ( !$self->{not_a_slave}->{$dbh} ) {
3826
2721
my $sth = $self->{sths}->{$dbh}->{SLAVE_STATUS}
3827
2722
||= $dbh->prepare('SHOW SLAVE STATUS');
3828
MKDEBUG && _d($dbh, 'SHOW SLAVE STATUS');
2723
PTDEBUG && _d($dbh, 'SHOW SLAVE STATUS');
3829
2724
$sth->execute();
3830
2725
my ($ss) = @{$sth->fetchall_arrayref({})};
3831
2726
3834
2729
return $ss;
3835
2730
}
3836
2731
3837
MKDEBUG && _d('This server returns nothing for SHOW SLAVE STATUS');
2732
PTDEBUG && _d('This server returns nothing for SHOW SLAVE STATUS');
3838
2733
$self->{not_a_slave}->{$dbh}++;
3839
2734
}
3840
2735
}
3843
2738
my ( $self, $dbh ) = @_;
3844
2739
3845
2740
if ( $self->{not_a_master}->{$dbh} ) {
3846
MKDEBUG && _d('Server on dbh', $dbh, 'is not a master');
2741
PTDEBUG && _d('Server on dbh', $dbh, 'is not a master');
3847
2742
return;
3848
2743
}
3849
2744
3850
2745
my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS}
3851
2746
||= $dbh->prepare('SHOW MASTER STATUS');
3852
MKDEBUG && _d($dbh, 'SHOW MASTER STATUS');
2747
PTDEBUG && _d($dbh, 'SHOW MASTER STATUS');
3853
2748
$sth->execute();
3854
2749
my ($ms) = @{$sth->fetchall_arrayref({})};
3855
MKDEBUG && _d(
2750
PTDEBUG && _d(
3856
2751
$ms ? map { "$_=" . (defined $ms->{$_} ? $ms->{$_} : '') } keys %$ms
3857
2752
: '');
3858
2753
3859
2754
if ( !$ms || scalar keys %$ms < 2 ) {
3860
MKDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master');
2755
PTDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master');
3861
2756
$self->{not_a_master}->{$dbh}++;
3862
2757
}
3863
2758
3878
2773
if ( $master_status ) {
3879
2774
my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', "
3880
2775
. "$master_status->{position}, $timeout)";
3881
MKDEBUG && _d($slave_dbh, $sql);
2776
PTDEBUG && _d($slave_dbh, $sql);
3882
2777
my $start = time;
3883
2778
($result) = $slave_dbh->selectrow_array($sql);
3884
2779
3885
2780
$waited = time - $start;
3886
2781
3887
MKDEBUG && _d('Result of waiting:', $result);
3888
MKDEBUG && _d("Waited", $waited, "seconds");
2782
PTDEBUG && _d('Result of waiting:', $result);
2783
PTDEBUG && _d("Waited", $waited, "seconds");
3889
2784
}
3890
2785
else {
3891
MKDEBUG && _d('Not waiting: this server is not a master');
2786
PTDEBUG && _d('Not waiting: this server is not a master');
3892
2787
}
3893
2788
3894
2789
return {
3901
2796
my ( $self, $dbh ) = @_;
3902
2797
my $sth = $self->{sths}->{$dbh}->{STOP_SLAVE}
3903
2798
||= $dbh->prepare('STOP SLAVE');
3904
MKDEBUG && _d($dbh, $sth->{Statement});
2799
PTDEBUG && _d($dbh, $sth->{Statement});
3905
2800
$sth->execute();
3906
2801
}
3907
2802
3910
2805
if ( $pos ) {
3911
2806
my $sql = "START SLAVE UNTIL MASTER_LOG_FILE='$pos->{file}', "
3912
2807
. "MASTER_LOG_POS=$pos->{position}";
3913
MKDEBUG && _d($dbh, $sql);
2808
PTDEBUG && _d($dbh, $sql);
3914
2809
$dbh->do($sql);
3915
2810
}
3916
2811
else {
3917
2812
my $sth = $self->{sths}->{$dbh}->{START_SLAVE}
3918
2813
||= $dbh->prepare('START SLAVE');
3919
MKDEBUG && _d($dbh, $sth->{Statement});
2814
PTDEBUG && _d($dbh, $sth->{Statement});
3920
2815
$sth->execute();
3921
2816
}
3922
2817
}
3929
2824
my $slave_pos = $self->repl_posn($slave_status);
3930
2825
my $master_status = $self->get_master_status($master);
3931
2826
my $master_pos = $self->repl_posn($master_status);
3932
MKDEBUG && _d('Master position:', $self->pos_to_string($master_pos),
2827
PTDEBUG && _d('Master position:', $self->pos_to_string($master_pos),
3933
2828
'Slave position:', $self->pos_to_string($slave_pos));
3934
2829
3935
2830
my $result;
3936
2831
if ( $self->pos_cmp($slave_pos, $master_pos) < 0 ) {
3937
MKDEBUG && _d('Waiting for slave to catch up to master');
2832
PTDEBUG && _d('Waiting for slave to catch up to master');
3938
2833
$self->start_slave($slave, $master_pos);
3939
2834
3940
2835
$result = $self->wait_for_master(
3946
2841
if ( !defined $result->{result} ) {
3947
2842
$slave_status = $self->get_slave_status($slave);
3948
2843
if ( !$self->slave_is_running($slave_status) ) {
3949
MKDEBUG && _d('Master position:',
2844
PTDEBUG && _d('Master position:',
3950
2845
$self->pos_to_string($master_pos),
3951
2846
'Slave position:', $self->pos_to_string($slave_pos));
3952
2847
$slave_pos = $self->repl_posn($slave_status);
3954
2849
die "MASTER_POS_WAIT() returned NULL but slave has not "
3955
2850
. "caught up to master";
3956
2851
}
3957
MKDEBUG && _d('Slave is caught up to master and stopped');
2852
PTDEBUG && _d('Slave is caught up to master and stopped');
3958
2853
}
3959
2854
else {
3960
2855
die "Slave has not caught up to master and it is still running";
3962
2857
}
3963
2858
}
3964
2859
else {
3965
MKDEBUG && _d("Slave is already caught up to master");
2860
PTDEBUG && _d("Slave is already caught up to master");
3966
2861
}
3967
2862
3968
2863
return $result;
4005
2900
sub has_slave_updates {
4006
2901
my ( $self, $dbh ) = @_;
4007
2902
my $sql = q{SHOW VARIABLES LIKE 'log_slave_updates'};
4008
MKDEBUG && _d($dbh, $sql);
2903
PTDEBUG && _d($dbh, $sql);
4009
2904
my ($name, $value) = $dbh->selectrow_array($sql);
4010
2905
return $value && $value =~ m/^(1|ON)$/;
4011
2906
}
4067
2962
}
4068
2963
if ( !$match ) {
4069
2964
if ( ($query->{User} || $query->{user} || '') eq "system user" ) {
4070
MKDEBUG && _d("Slave replication thread");
2965
PTDEBUG && _d("Slave replication thread");
4071
2966
if ( $type ne 'all' ) {
4072
2967
my $state = $query->{State} || $query->{state} || '';
4073
2968
4074
2969
if ( $state =~ m/^init|end$/ ) {
4075
MKDEBUG && _d("Special state:", $state);
2970
PTDEBUG && _d("Special state:", $state);
4076
2971
$match = 1;
4077
2972
}
4078
2973
else {
4093
2988
}
4094
2989
}
4095
2990
else {
4096
MKDEBUG && _d('Not system user');
2991
PTDEBUG && _d('Not system user');
4097
2992
}
4098
2993
4099
2994
if ( !defined $args{check_known_ids} || $args{check_known_ids} ) {
4103
2998
}
4104
2999
else {
4105
3000
if ( $self->{replication_thread}->{$id} ) {
4106
MKDEBUG && _d("Thread ID is a known replication thread ID");
3001
PTDEBUG && _d("Thread ID is a known replication thread ID");
4107
3002
$match = 1;
4108
3003
}
4109
3004
}
4110
3005
}
4111
3006
}
4112
3007
4113
MKDEBUG && _d('Matches', $type, 'replication thread:',
3008
PTDEBUG && _d('Matches', $type, 'replication thread:',
4114
3009
($match ? 'yes' : 'no'), '; match:', $match);
4115
3010
4116
3011
return $match;
4151
3046
);
4152
3047
4153
3048
my $sql = "SHOW VARIABLES LIKE 'slave_skip_errors'";
4154
MKDEBUG && _d($dbh, $sql);
3049
PTDEBUG && _d($dbh, $sql);
4155
3050
my $row = $dbh->selectrow_arrayref($sql);
4156
3051
$filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF';
4157
3052
}
4172
3067
return;
4173
3068
}
4174
3069
3070
sub get_cxn_from_dsn_table {
3071
my ($self, %args) = @_;
3072
my @required_args = qw(dsn_table_dsn make_cxn DSNParser Quoter);
3073
foreach my $arg ( @required_args ) {
3074
die "I need a $arg argument" unless $args{$arg};
3075
}
3076
my ($dsn_table_dsn, $make_cxn, $dp, $q) = @args{@required_args};
3077
PTDEBUG && _d('DSN table DSN:', $dsn_table_dsn);
3078
3079
my $dsn = $dp->parse($dsn_table_dsn);
3080
my $dsn_table;
3081
if ( $dsn->{D} && $dsn->{t} ) {
3082
$dsn_table = $q->quote($dsn->{D}, $dsn->{t});
3083
}
3084
elsif ( $dsn->{t} && $dsn->{t} =~ m/\./ ) {
3085
$dsn_table = $q->quote($q->split_unquote($dsn->{t}));
3086
}
3087
else {
3088
die "DSN table DSN does not specify a database (D) "
3089
. "or a database-qualified table (t)";
3090
}
3091
3092
my $dsn_tbl_cxn = $make_cxn->(dsn => $dsn);
3093
my $dbh = $dsn_tbl_cxn->connect();
3094
my $sql = "SELECT dsn FROM $dsn_table ORDER BY id";
3095
PTDEBUG && _d($sql);
3096
my $dsn_strings = $dbh->selectcol_arrayref($sql);
3097
my @cxn;
3098
if ( $dsn_strings ) {
3099
foreach my $dsn_string ( @$dsn_strings ) {
3100
PTDEBUG && _d('DSN from DSN table:', $dsn_string);
3101
push @cxn, $make_cxn->(dsn_string => $dsn_string);
3102
}
3103
}
3104
return \@cxn;
3105
}
3106
4175
3107
sub _d {
4176
3108
my ($package, undef, $line) = caller 0;
4177
3109
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
4187
3119
# ###########################################################################
4188
3120
4189
3121
# ###########################################################################
3122
# RowChecksum package
3123
# This package is a copy without comments from the original. The original
3124
# with comments and its test file can be found in the Bazaar repository at,
3125
# lib/RowChecksum.pm
3126
# t/lib/RowChecksum.t
3127
# See https://launchpad.net/percona-toolkit for more information.
3128
# ###########################################################################
3129
{
3130
package RowChecksum;
3131
3132
use strict;
3133
use warnings FATAL => 'all';
3134
use English qw(-no_match_vars);
3135
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
3136
3137
use List::Util qw(max);
3138
use Data::Dumper;
3139
$Data::Dumper::Indent = 1;
3140
$Data::Dumper::Sortkeys = 1;
3141
$Data::Dumper::Quotekeys = 0;
3142
3143
sub new {
3144
my ( $class, %args ) = @_;
3145
foreach my $arg ( qw(OptionParser Quoter) ) {
3146
die "I need a $arg argument" unless defined $args{$arg};
3147
}
3148
my $self = { %args };
3149
return bless $self, $class;
3150
}
3151
3152
sub make_row_checksum {
3153
my ( $self, %args ) = @_;
3154
my @required_args = qw(tbl);
3155
foreach my $arg( @required_args ) {
3156
die "I need a $arg argument" unless $args{$arg};
3157
}
3158
my ($tbl) = @args{@required_args};
3159
3160
my $o = $self->{OptionParser};
3161
my $q = $self->{Quoter};
3162
my $tbl_struct = $tbl->{tbl_struct};
3163
my $func = $args{func} || uc($o->get('function'));
3164
my $cols = $self->get_checksum_columns(%args);
3165
3166
my $query;
3167
if ( !$args{no_cols} ) {
3168
$query = join(', ',
3169
map {
3170
my $col = $_;
3171
if ( $col =~ m/\+ 0/ ) {
3172
my ($real_col) = /^(\S+)/;
3173
$col .= " AS $real_col";
3174
}
3175
elsif ( $col =~ m/TRIM/ ) {
3176
my ($real_col) = m/TRIM\(([^\)]+)\)/;
3177
$col .= " AS $real_col";
3178
}
3179
$col;
3180
} @{$cols->{select}})
3181
. ', ';
3182
}
3183
3184
if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
3185
my $sep = $o->get('separator') || '#';
3186
$sep =~ s/'//g;
3187
$sep ||= '#';
3188
3189
my @nulls = grep { $cols->{allowed}->{$_} } @{$tbl_struct->{null_cols}};
3190
if ( @nulls ) {
3191
my $bitmap = "CONCAT("
3192
. join(', ', map { 'ISNULL(' . $q->quote($_) . ')' } @nulls)
3193
. ")";
3194
push @{$cols->{select}}, $bitmap;
3195
}
3196
3197
$query .= @{$cols->{select}} > 1
3198
? "$func(CONCAT_WS('$sep', " . join(', ', @{$cols->{select}}) . '))'
3199
: "$func($cols->{select}->[0])";
3200
}
3201
else {
3202
my $fnv_func = uc $func;
3203
$query .= "$fnv_func(" . join(', ', @{$cols->{select}}) . ')';
3204
}
3205
3206
PTDEBUG && _d('Row checksum:', $query);
3207
return $query;
3208
}
3209
3210
sub make_chunk_checksum {
3211
my ( $self, %args ) = @_;
3212
my @required_args = qw(tbl);
3213
foreach my $arg( @required_args ) {
3214
die "I need a $arg argument" unless $args{$arg};
3215
}
3216
if ( !$args{dbh} && !($args{func} && $args{crc_width} && $args{crc_type}) ) {
3217
die "I need a dbh argument"
3218
}
3219
my ($tbl) = @args{@required_args};
3220
my $o = $self->{OptionParser};
3221
my $q = $self->{Quoter};
3222
3223
my %crc_args = $self->get_crc_args(%args);
3224
PTDEBUG && _d("Checksum strat:", Dumper(\%crc_args));
3225
3226
my $row_checksum = $self->make_row_checksum(
3227
%args,
3228
%crc_args,
3229
no_cols => 1
3230
);
3231
my $crc;
3232
if ( $crc_args{crc_type} =~ m/int$/ ) {
3233
$crc = "COALESCE(LOWER(CONV(BIT_XOR(CAST($row_checksum AS UNSIGNED)), "
3234
. "10, 16)), 0)";
3235
}
3236
else {
3237
my $slices = $self->_make_xor_slices(
3238
row_checksum => $row_checksum,
3239
%crc_args,
3240
);
3241
$crc = "COALESCE(LOWER(CONCAT($slices)), 0)";
3242
}
3243
3244
my $select = "COUNT(*) AS cnt, $crc AS crc";
3245
PTDEBUG && _d('Chunk checksum:', $select);
3246
return $select;
3247
}
3248
3249
sub get_checksum_columns {
3250
my ($self, %args) = @_;
3251
my @required_args = qw(tbl);
3252
foreach my $arg( @required_args ) {
3253
die "I need a $arg argument" unless $args{$arg};
3254
}
3255
my ($tbl) = @args{@required_args};
3256
my $o = $self->{OptionParser};
3257
my $q = $self->{Quoter};
3258
3259
my $trim = $o->get('trim');
3260
my $float_precision = $o->get('float-precision');
3261
3262
my $tbl_struct = $tbl->{tbl_struct};
3263
my $ignore_col = $o->get('ignore-columns') || {};
3264
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
3265
my %cols = map { lc($_) => 1 } grep { !$ignore_col->{$_} } @$all_cols;
3266
my %seen;
3267
my @cols =
3268
map {
3269
my $type = $tbl_struct->{type_for}->{$_};
3270
my $result = $q->quote($_);
3271
if ( $type eq 'timestamp' ) {
3272
$result .= ' + 0';
3273
}
3274
elsif ( $float_precision && $type =~ m/float|double/ ) {
3275
$result = "ROUND($result, $float_precision)";
3276
}
3277
elsif ( $trim && $type =~ m/varchar/ ) {
3278
$result = "TRIM($result)";
3279
}
3280
$result;
3281
}
3282
grep {
3283
$cols{$_} && !$seen{$_}++
3284
}
3285
@{$tbl_struct->{cols}};
3286
3287
return {
3288
select => \@cols,
3289
allowed => \%cols,
3290
};
3291
}
3292
3293
sub get_crc_args {
3294
my ($self, %args) = @_;
3295
my $func = $args{func} || $self->_get_hash_func(%args);
3296
my $crc_width = $args{crc_width}|| $self->_get_crc_width(%args, func=>$func);
3297
my $crc_type = $args{crc_type} || $self->_get_crc_type(%args, func=>$func);
3298
my $opt_slice;
3299
if ( $args{dbh} && $crc_type !~ m/int$/ ) {
3300
$opt_slice = $self->_optimize_xor(%args, func=>$func);
3301
}
3302
3303
return (
3304
func => $func,
3305
crc_width => $crc_width,
3306
crc_type => $crc_type,
3307
opt_slice => $opt_slice,
3308
);
3309
}
3310
3311
sub _get_hash_func {
3312
my ( $self, %args ) = @_;
3313
my @required_args = qw(dbh);
3314
foreach my $arg( @required_args ) {
3315
die "I need a $arg argument" unless $args{$arg};
3316
}
3317
my ($dbh) = @args{@required_args};
3318
my $o = $self->{OptionParser};
3319
my @funcs = qw(CRC32 FNV1A_64 FNV_64 MD5 SHA1);
3320
3321
if ( my $func = $o->get('function') ) {
3322
unshift @funcs, $func;
3323
}
3324
3325
my ($result, $error);
3326
foreach my $func ( @funcs ) {
3327
eval {
3328
my $sql = "SELECT $func('test-string')";
3329
PTDEBUG && _d($sql);
3330
$args{dbh}->do($sql);
3331
};
3332
if ( $EVAL_ERROR && $EVAL_ERROR =~ m/failed: (.*?) at \S+ line/ ) {
3333
$error .= qq{$func cannot be used because "$1"\n};
3334
PTDEBUG && _d($func, 'cannot be used because', $1);
3335
}
3336
PTDEBUG && _d('Chosen hash func:', $result);
3337
return $func;
3338
}
3339
die $error || 'No hash functions (CRC32, MD5, etc.) are available';
3340
}
3341
3342
sub _get_crc_width {
3343
my ( $self, %args ) = @_;
3344
my @required_args = qw(dbh func);
3345
foreach my $arg( @required_args ) {
3346
die "I need a $arg argument" unless $args{$arg};
3347
}
3348
my ($dbh, $func) = @args{@required_args};
3349
3350
my $crc_width = 16;
3351
if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
3352
eval {
3353
my ($val) = $dbh->selectrow_array("SELECT $func('a')");
3354
$crc_width = max(16, length($val));
3355
};
3356
}
3357
return $crc_width;
3358
}
3359
3360
sub _get_crc_type {
3361
my ( $self, %args ) = @_;
3362
my @required_args = qw(dbh func);
3363
foreach my $arg( @required_args ) {
3364
die "I need a $arg argument" unless $args{$arg};
3365
}
3366
my ($dbh, $func) = @args{@required_args};
3367
3368
my $type = '';
3369
my $length = 0;
3370
my $sql = "SELECT $func('a')";
3371
my $sth = $dbh->prepare($sql);
3372
eval {
3373
$sth->execute();
3374
$type = $sth->{mysql_type_name}->[0];
3375
$length = $sth->{mysql_length}->[0];
3376
PTDEBUG && _d($sql, $type, $length);
3377
if ( $type eq 'bigint' && $length < 20 ) {
3378
$type = 'int';
3379
}
3380
};
3381
$sth->finish;
3382
PTDEBUG && _d('crc_type:', $type, 'length:', $length);
3383
return $type;
3384
}
3385
3386
sub _optimize_xor {
3387
my ( $self, %args ) = @_;
3388
my @required_args = qw(dbh func);
3389
foreach my $arg( @required_args ) {
3390
die "I need a $arg argument" unless $args{$arg};
3391
}
3392
my ($dbh, $func) = @args{@required_args};
3393
3394
die "$func never needs BIT_XOR optimization"
3395
if $func =~ m/^(?:FNV1A_64|FNV_64|CRC32)$/i;
3396
3397
my $opt_slice = 0;
3398
my $unsliced = uc $dbh->selectall_arrayref("SELECT $func('a')")->[0]->[0];
3399
my $sliced = '';
3400
my $start = 1;
3401
my $crc_width = length($unsliced) < 16 ? 16 : length($unsliced);
3402
3403
do { # Try different positions till sliced result equals non-sliced.
3404
PTDEBUG && _d('Trying slice', $opt_slice);
3405
$dbh->do('SET @crc := "", @cnt := 0');
3406
my $slices = $self->_make_xor_slices(
3407
row_checksum => "\@crc := $func('a')",
3408
crc_width => $crc_width,
3409
opt_slice => $opt_slice,
3410
);
3411
3412
my $sql = "SELECT CONCAT($slices) AS TEST FROM (SELECT NULL) AS x";
3413
$sliced = ($dbh->selectrow_array($sql))[0];
3414
if ( $sliced ne $unsliced ) {
3415
PTDEBUG && _d('Slice', $opt_slice, 'does not work');
3416
$start += 16;
3417
++$opt_slice;
3418
}
3419
} while ( $start < $crc_width && $sliced ne $unsliced );
3420
3421
if ( $sliced eq $unsliced ) {
3422
PTDEBUG && _d('Slice', $opt_slice, 'works');
3423
return $opt_slice;
3424
}
3425
else {
3426
PTDEBUG && _d('No slice works');
3427
return undef;
3428
}
3429
}
3430
3431
sub _make_xor_slices {
3432
my ( $self, %args ) = @_;
3433
my @required_args = qw(row_checksum crc_width);
3434
foreach my $arg( @required_args ) {
3435
die "I need a $arg argument" unless $args{$arg};
3436
}
3437
my ($row_checksum, $crc_width) = @args{@required_args};
3438
my ($opt_slice) = $args{opt_slice};
3439
3440
my @slices;
3441
for ( my $start = 1; $start <= $crc_width; $start += 16 ) {
3442
my $len = $crc_width - $start + 1;
3443
if ( $len > 16 ) {
3444
$len = 16;
3445
}
3446
push @slices,
3447
"LPAD(CONV(BIT_XOR("
3448
. "CAST(CONV(SUBSTRING(\@crc, $start, $len), 16, 10) AS UNSIGNED))"
3449
. ", 10, 16), $len, '0')";
3450
}
3451
3452
if ( defined $opt_slice && $opt_slice < @slices ) {
3453
$slices[$opt_slice] =~ s/\@crc/\@crc := $row_checksum/;
3454
}
3455
else {
3456
map { s/\@crc/$row_checksum/ } @slices;
3457
}
3458
3459
return join(', ', @slices);
3460
}
3461
3462
sub find_replication_differences {
3463
my ($self, %args) = @_;
3464
my @required_args = qw(dbh repl_table);
3465
foreach my $arg( @required_args ) {
3466
die "I need a $arg argument" unless $args{$arg};
3467
}
3468
my ($dbh, $repl_table) = @args{@required_args};
3469
3470
my $sql
3471
= "SELECT CONCAT(db, '.', tbl) AS `table`, "
3472
. "chunk, chunk_index, lower_boundary, upper_boundary, "
3473
. "COALESCE(this_cnt-master_cnt, 0) AS cnt_diff, "
3474
. "COALESCE("
3475
. "this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc), 0"
3476
. ") AS crc_diff, this_cnt, master_cnt, this_crc, master_crc "
3477
. "FROM $repl_table "
3478
. "WHERE (master_cnt <> this_cnt OR master_crc <> this_crc "
3479
. "OR ISNULL(master_crc) <> ISNULL(this_crc))"
3480
. ($args{where} ? " AND ($args{where})" : "");
3481
PTDEBUG && _d($sql);
3482
my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} });
3483
return $diffs;
3484
}
3485
3486
sub _d {
3487
my ($package, undef, $line) = caller 0;
3488
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
3489
map { defined $_ ? $_ : 'undef' }
3490
@_;
3491
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
3492
}
3493
3494
1;
3495
}
3496
# ###########################################################################
3497
# End RowChecksum package
3498
# ###########################################################################
3499
3500
# ###########################################################################
3501
# NibbleIterator package
3502
# This package is a copy without comments from the original. The original
3503
# with comments and its test file can be found in the Bazaar repository at,
3504
# lib/NibbleIterator.pm
3505
# t/lib/NibbleIterator.t
3506
# See https://launchpad.net/percona-toolkit for more information.
3507
# ###########################################################################
3508
{
3509
package NibbleIterator;
3510
3511
use strict;
3512
use warnings FATAL => 'all';
3513
use English qw(-no_match_vars);
3514
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
3515
3516
use Data::Dumper;
3517
$Data::Dumper::Indent = 1;
3518
$Data::Dumper::Sortkeys = 1;
3519
$Data::Dumper::Quotekeys = 0;
3520
3521
sub new {
3522
my ( $class, %args ) = @_;
3523
my @required_args = qw(Cxn tbl chunk_size OptionParser Quoter TableNibbler TableParser);
3524
foreach my $arg ( @required_args ) {
3525
die "I need a $arg argument" unless $args{$arg};
3526
}
3527
my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args};
3528
3529
my $where = $o->get('where');
3530
my ($row_est, $mysql_index) = get_row_estimate(%args, where => $where);
3531
my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
3532
? $row_est <= $chunk_size * $o->get('chunk-size-limit')
3533
: 0;
3534
PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
3535
3536
if ( $args{resume}
3537
&& !defined $args{resume}->{lower_boundary}
3538
&& !defined $args{resume}->{upper_boundary} ) {
3539
PTDEBUG && _d('Resuming from one nibble table');
3540
$one_nibble = 1;
3541
}
3542
3543
my $index = _find_best_index(%args, mysql_index => $mysql_index);
3544
if ( !$index && !$one_nibble ) {
3545
die "There is no good index and the table is oversized.";
3546
}
3547
3548
my $tbl_struct = $tbl->{tbl_struct};
3549
my $ignore_col = $o->get('ignore-columns') || {};
3550
my $all_cols = $o->get('columns') || $tbl_struct->{cols};
3551
my @cols = grep { !$ignore_col->{$_} } @$all_cols;
3552
my $self;
3553
if ( $one_nibble ) {
3554
my $nibble_sql
3555
= ($args{dml} ? "$args{dml} " : "SELECT ")
3556
. ($args{select} ? $args{select}
3557
: join(', ', map { $q->quote($_) } @cols))
3558
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
3559
. ($where ? " WHERE $where" : '')
3560
. " /*checksum table*/";
3561
PTDEBUG && _d('One nibble statement:', $nibble_sql);
3562
3563
my $explain_nibble_sql
3564
= "EXPLAIN SELECT "
3565
. ($args{select} ? $args{select}
3566
: join(', ', map { $q->quote($_) } @cols))
3567
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
3568
. ($where ? " WHERE $where" : '')
3569
. " /*explain checksum table*/";
3570
PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql);
3571
3572
$self = {
3573
%args,
3574
one_nibble => 1,
3575
limit => 0,
3576
nibble_sql => $nibble_sql,
3577
explain_nibble_sql => $explain_nibble_sql,
3578
};
3579
}
3580
else {
3581
my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols};
3582
3583
my $asc = $args{TableNibbler}->generate_asc_stmt(
3584
%args,
3585
tbl_struct => $tbl->{tbl_struct},
3586
index => $index,
3587
cols => \@cols,
3588
asc_only => 1,
3589
);
3590
PTDEBUG && _d('Ascend params:', Dumper($asc));
3591
3592
my $from = $q->quote(@{$tbl}{qw(db tbl)}) . " FORCE INDEX(`$index`)";
3593
my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
3594
3595
my $first_lb_sql
3596
= "SELECT /*!40001 SQL_NO_CACHE */ "
3597
. join(', ', map { $q->quote($_) } @{$asc->{scols}})
3598
. " FROM $from"
3599
. ($where ? " WHERE $where" : '')
3600
. " ORDER BY $order_by"
3601
. " LIMIT 1"
3602
. " /*first lower boundary*/";
3603
PTDEBUG && _d('First lower boundary statement:', $first_lb_sql);
3604
3605
my $resume_lb_sql;
3606
if ( $args{resume} ) {
3607
$resume_lb_sql
3608
= "SELECT /*!40001 SQL_NO_CACHE */ "
3609
. join(', ', map { $q->quote($_) } @{$asc->{scols}})
3610
. " FROM $from"
3611
. " WHERE " . $asc->{boundaries}->{'>'}
3612
. ($where ? " AND ($where)" : '')
3613
. " ORDER BY $order_by"
3614
. " LIMIT 1"
3615
. " /*resume lower boundary*/";
3616
PTDEBUG && _d('Resume lower boundary statement:', $resume_lb_sql);
3617
}
3618
3619
my $last_ub_sql
3620
= "SELECT /*!40001 SQL_NO_CACHE */ "
3621
. join(', ', map { $q->quote($_) } @{$asc->{scols}})
3622
. " FROM $from"
3623
. ($where ? " WHERE $where" : '')
3624
. " ORDER BY "
3625
. join(' DESC, ', map {$q->quote($_)} @{$index_cols}) . ' DESC'
3626
. " LIMIT 1"
3627
. " /*last upper boundary*/";
3628
PTDEBUG && _d('Last upper boundary statement:', $last_ub_sql);
3629
3630
my $ub_sql
3631
= "SELECT /*!40001 SQL_NO_CACHE */ "
3632
. join(', ', map { $q->quote($_) } @{$asc->{scols}})
3633
. " FROM $from"
3634
. " WHERE " . $asc->{boundaries}->{'>='}
3635
. ($where ? " AND ($where)" : '')
3636
. " ORDER BY $order_by"
3637
. " LIMIT ?, 2"
3638
. " /*next chunk boundary*/";
3639
PTDEBUG && _d('Upper boundary statement:', $ub_sql);
3640
3641
my $nibble_sql
3642
= ($args{dml} ? "$args{dml} " : "SELECT ")
3643
. ($args{select} ? $args{select}
3644
: join(', ', map { $q->quote($_) } @{$asc->{cols}}))
3645
. " FROM $from"
3646
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
3647
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
3648
. ($where ? " AND ($where)" : '')
3649
. ($args{order_by} ? " ORDER BY $order_by" : "")
3650
. " /*checksum chunk*/";
3651
PTDEBUG && _d('Nibble statement:', $nibble_sql);
3652
3653
my $explain_nibble_sql
3654
= "EXPLAIN SELECT "
3655
. ($args{select} ? $args{select}
3656
: join(', ', map { $q->quote($_) } @{$asc->{cols}}))
3657
. " FROM $from"
3658
. " WHERE " . $asc->{boundaries}->{'>='} # lower boundary
3659
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
3660
. ($where ? " AND ($where)" : '')
3661
. ($args{order_by} ? " ORDER BY $order_by" : "")
3662
. " /*explain checksum chunk*/";
3663
PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql);
3664
3665
my $limit = $chunk_size - 1;
3666
PTDEBUG && _d('Initial chunk size (LIMIT):', $limit);
3667
3668
$self = {
3669
%args,
3670
index => $index,
3671
limit => $limit,
3672
first_lb_sql => $first_lb_sql,
3673
last_ub_sql => $last_ub_sql,
3674
ub_sql => $ub_sql,
3675
nibble_sql => $nibble_sql,
3676
explain_ub_sql => "EXPLAIN $ub_sql",
3677
explain_nibble_sql => $explain_nibble_sql,
3678
resume_lb_sql => $resume_lb_sql,
3679
sql => {
3680
columns => $asc->{scols},
3681
from => $from,
3682
where => $where,
3683
boundaries => $asc->{boundaries},
3684
order_by => $order_by,
3685
},
3686
};
3687
}
3688
3689
$self->{row_est} = $row_est;
3690
$self->{nibbleno} = 0;
3691
$self->{have_rows} = 0;
3692
$self->{rowno} = 0;
3693
$self->{oktonibble} = 1;
3694
3695
return bless $self, $class;
3696
}
3697
3698
sub next {
3699
my ($self) = @_;
3700
3701
if ( !$self->{oktonibble} ) {
3702
PTDEBUG && _d('Not ok to nibble');
3703
return;
3704
}
3705
3706
my %callback_args = (
3707
Cxn => $self->{Cxn},
3708
tbl => $self->{tbl},
3709
NibbleIterator => $self,
3710
);
3711
3712
if ($self->{nibbleno} == 0) {
3713
$self->_prepare_sths();
3714
$self->_get_bounds();
3715
if ( my $callback = $self->{callbacks}->{init} ) {
3716
$self->{oktonibble} = $callback->(%callback_args);
3717
PTDEBUG && _d('init callback returned', $self->{oktonibble});
3718
if ( !$self->{oktonibble} ) {
3719
$self->{no_more_boundaries} = 1;
3720
return;
3721
}
3722
}
3723
}
3724
3725
NIBBLE:
3726
while ( $self->{have_rows} || $self->_next_boundaries() ) {
3727
if ( !$self->{have_rows} ) {
3728
$self->{nibbleno}++;
3729
PTDEBUG && _d($self->{nibble_sth}->{Statement}, 'params:',
3730
join(', ', (@{$self->{lower}}, @{$self->{upper}})));
3731
if ( my $callback = $self->{callbacks}->{exec_nibble} ) {
3732
$self->{have_rows} = $callback->(%callback_args);
3733
}
3734
else {
3735
$self->{nibble_sth}->execute(@{$self->{lower}}, @{$self->{upper}});
3736
$self->{have_rows} = $self->{nibble_sth}->rows();
3737
}
3738
PTDEBUG && _d($self->{have_rows}, 'rows in nibble', $self->{nibbleno});
3739
}
3740
3741
if ( $self->{have_rows} ) {
3742
my $row = $self->{nibble_sth}->fetchrow_arrayref();
3743
if ( $row ) {
3744
$self->{rowno}++;
3745
PTDEBUG && _d('Row', $self->{rowno}, 'in nibble',$self->{nibbleno});
3746
return [ @$row ];
3747
}
3748
}
3749
3750
PTDEBUG && _d('No rows in nibble or nibble skipped');
3751
if ( my $callback = $self->{callbacks}->{after_nibble} ) {
3752
$callback->(%callback_args);
3753
}
3754
$self->{rowno} = 0;
3755
$self->{have_rows} = 0;
3756
}
3757
3758
PTDEBUG && _d('Done nibbling');
3759
if ( my $callback = $self->{callbacks}->{done} ) {
3760
$callback->(%callback_args);
3761
}
3762
3763
return;
3764
}
3765
3766
sub nibble_number {
3767
my ($self) = @_;
3768
return $self->{nibbleno};
3769
}
3770
3771
sub set_nibble_number {
3772
my ($self, $n) = @_;
3773
die "I need a number" unless $n;
3774
$self->{nibbleno} = $n;
3775
PTDEBUG && _d('Set new nibble number:', $n);
3776
return;
3777
}
3778
3779
sub nibble_index {
3780
my ($self) = @_;
3781
return $self->{index};
3782
}
3783
3784
sub statements {
3785
my ($self) = @_;
3786
return {
3787
nibble => $self->{nibble_sth},
3788
explain_nibble => $self->{explain_nibble_sth},
3789
upper_boundary => $self->{ub_sth},
3790
explain_upper_boundary => $self->{explain_ub_sth},
3791
}
3792
}
3793
3794
sub boundaries {
3795
my ($self) = @_;
3796
return {
3797
first_lower => $self->{first_lower},
3798
lower => $self->{lower},
3799
upper => $self->{upper},
3800
next_lower => $self->{next_lower},
3801
last_upper => $self->{last_upper},
3802
};
3803
}
3804
3805
sub set_boundary {
3806
my ($self, $boundary, $values) = @_;
3807
die "I need a boundary parameter"
3808
unless $boundary;
3809
die "Invalid boundary: $boundary"
3810
unless $boundary =~ m/^(?:lower|upper|next_lower|last_upper)$/;
3811
die "I need a values arrayref parameter"
3812
unless $values && ref $values eq 'ARRAY';
3813
$self->{$boundary} = $values;
3814
PTDEBUG && _d('Set new', $boundary, 'boundary:', Dumper($values));
3815
return;
3816
}
3817
3818
sub one_nibble {
3819
my ($self) = @_;
3820
return $self->{one_nibble};
3821
}
3822
3823
sub chunk_size {
3824
my ($self) = @_;
3825
return $self->{limit} + 1;
3826
}
3827
3828
sub set_chunk_size {
3829
my ($self, $limit) = @_;
3830
return if $self->{one_nibble};
3831
die "Chunk size must be > 0" unless $limit;
3832
$self->{limit} = $limit - 1;
3833
PTDEBUG && _d('Set new chunk size (LIMIT):', $limit);
3834
return;
3835
}
3836
3837
sub sql {
3838
my ($self) = @_;
3839
return $self->{sql};
3840
}
3841
3842
sub more_boundaries {
3843
my ($self) = @_;
3844
return !$self->{no_more_boundaries};
3845
}
3846
3847
sub row_estimate {
3848
my ($self) = @_;
3849
return $self->{row_est};
3850
}
3851
3852
sub _find_best_index {
3853
my (%args) = @_;
3854
my @required_args = qw(Cxn tbl TableParser);
3855
my ($cxn, $tbl, $tp) = @args{@required_args};
3856
my $tbl_struct = $tbl->{tbl_struct};
3857
my $indexes = $tbl_struct->{keys};
3858
3859
my $want_index = $args{chunk_index};
3860
if ( $want_index ) {
3861
PTDEBUG && _d('User wants to use index', $want_index);
3862
if ( !exists $indexes->{$want_index} ) {
3863
PTDEBUG && _d('Cannot use user index because it does not exist');
3864
$want_index = undef;
3865
}
3866
}
3867
3868
if ( !$want_index && $args{mysql_index} ) {
3869
PTDEBUG && _d('MySQL wants to use index', $args{mysql_index});
3870
$want_index = $args{mysql_index};
3871
}
3872
3873
my $best_index;
3874
my @possible_indexes;
3875
if ( $want_index ) {
3876
if ( $indexes->{$want_index}->{is_unique} ) {
3877
PTDEBUG && _d('Will use wanted index');
3878
$best_index = $want_index;
3879
}
3880
else {
3881
PTDEBUG && _d('Wanted index is a possible index');
3882
push @possible_indexes, $want_index;
3883
}
3884
}
3885
else {
3886
PTDEBUG && _d('Auto-selecting best index');
3887
foreach my $index ( $tp->sort_indexes($tbl_struct) ) {
3888
if ( $index eq 'PRIMARY' || $indexes->{$index}->{is_unique} ) {
3889
$best_index = $index;
3890
last;
3891
}
3892
else {
3893
push @possible_indexes, $index;
3894
}
3895
}
3896
}
3897
3898
if ( !$best_index && @possible_indexes ) {
3899
PTDEBUG && _d('No PRIMARY or unique indexes;',
3900
'will use index with highest cardinality');
3901
foreach my $index ( @possible_indexes ) {
3902
$indexes->{$index}->{cardinality} = _get_index_cardinality(
3903
%args,
3904
index => $index,
3905
);
3906
}
3907
@possible_indexes = sort {
3908
my $cmp
3909
= $indexes->{$b}->{cardinality} <=> $indexes->{$b}->{cardinality};
3910
if ( $cmp == 0 ) {
3911
$cmp = scalar @{$indexes->{$b}->{cols}}
3912
<=> scalar @{$indexes->{$a}->{cols}};
3913
}
3914
$cmp;
3915
} @possible_indexes;
3916
$best_index = $possible_indexes[0];
3917
}
3918
3919
PTDEBUG && _d('Best index:', $best_index);
3920
return $best_index;
3921
}
3922
3923
sub _get_index_cardinality {
3924
my (%args) = @_;
3925
my @required_args = qw(Cxn tbl index Quoter);
3926
my ($cxn, $tbl, $index, $q) = @args{@required_args};
3927
3928
my $sql = "SHOW INDEXES FROM " . $q->quote(@{$tbl}{qw(db tbl)})
3929
. " WHERE Key_name = '$index'";
3930
PTDEBUG && _d($sql);
3931
my $cardinality = 1;
3932
my $rows = $cxn->dbh()->selectall_hashref($sql, 'key_name');
3933
foreach my $row ( values %$rows ) {
3934
$cardinality *= $row->{cardinality} if $row->{cardinality};
3935
}
3936
PTDEBUG && _d('Index', $index, 'cardinality:', $cardinality);
3937
return $cardinality;
3938
}
3939
3940
sub get_row_estimate {
3941
my (%args) = @_;
3942
my @required_args = qw(Cxn tbl OptionParser TableParser Quoter);
3943
my ($cxn, $tbl, $o, $tp, $q) = @args{@required_args};
3944
3945
if ( $args{where} ) {
3946
PTDEBUG && _d('WHERE clause, using explain plan for row estimate');
3947
my $table = $q->quote(@{$tbl}{qw(db tbl)});
3948
my $sql = "EXPLAIN SELECT * FROM $table WHERE $args{where}";
3949
PTDEBUG && _d($sql);
3950
my $expl = $cxn->dbh()->selectrow_hashref($sql);
3951
PTDEBUG && _d(Dumper($expl));
3952
return ($expl->{rows} || 0), $expl->{key};
3953
}
3954
else {
3955
PTDEBUG && _d('No WHERE clause, using table status for row estimate');
3956
return $tbl->{tbl_status}->{rows} || 0;
3957
}
3958
}
3959
3960
sub _prepare_sths {
3961
my ($self) = @_;
3962
PTDEBUG && _d('Preparing statement handles');
3963
3964
my $dbh = $self->{Cxn}->dbh();
3965
3966
$self->{nibble_sth} = $dbh->prepare($self->{nibble_sql});
3967
$self->{explain_nibble_sth} = $dbh->prepare($self->{explain_nibble_sql});
3968
3969
if ( !$self->{one_nibble} ) {
3970
$self->{ub_sth} = $dbh->prepare($self->{ub_sql});
3971
$self->{explain_ub_sth} = $dbh->prepare($self->{explain_ub_sql});
3972
}
3973
3974
return;
3975
}
3976
3977
sub _get_bounds {
3978
my ($self) = @_;
3979
3980
if ( $self->{one_nibble} ) {
3981
if ( $self->{resume} ) {
3982
$self->{no_more_boundaries} = 1;
3983
}
3984
return;
3985
}
3986
3987
my $dbh = $self->{Cxn}->dbh();
3988
3989
$self->{first_lower} = $dbh->selectrow_arrayref($self->{first_lb_sql});
3990
PTDEBUG && _d('First lower boundary:', Dumper($self->{first_lower}));
3991
3992
if ( my $nibble = $self->{resume} ) {
3993
if ( defined $nibble->{lower_boundary}
3994
&& defined $nibble->{upper_boundary} ) {
3995
my $sth = $dbh->prepare($self->{resume_lb_sql});
3996
my @ub = split ',', $nibble->{upper_boundary};
3997
PTDEBUG && _d($sth->{Statement}, 'params:', @ub);
3998
$sth->execute(@ub);
3999
$self->{next_lower} = $sth->fetchrow_arrayref();
4000
$sth->finish();
4001
}
4002
}
4003
else {
4004
$self->{next_lower} = $self->{first_lower};
4005
}
4006
PTDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower}));
4007
4008
if ( !$self->{next_lower} ) {
4009
PTDEBUG && _d('At end of table, or no more boundaries to resume');
4010
$self->{no_more_boundaries} = 1;
4011
}
4012
4013
$self->{last_upper} = $dbh->selectrow_arrayref($self->{last_ub_sql});
4014
PTDEBUG && _d('Last upper boundary:', Dumper($self->{last_upper}));
4015
4016
return;
4017
}
4018
4019
sub _next_boundaries {
4020
my ($self) = @_;
4021
4022
if ( $self->{no_more_boundaries} ) {
4023
PTDEBUG && _d('No more boundaries');
4024
return; # stop nibbling
4025
}
4026
4027
if ( $self->{one_nibble} ) {
4028
$self->{lower} = $self->{upper} = [];
4029
$self->{no_more_boundaries} = 1; # for next call
4030
return 1; # continue nibbling
4031
}
4032
4033
if ( $self->identical_boundaries($self->{lower}, $self->{next_lower}) ) {
4034
PTDEBUG && _d('Infinite loop detected');
4035
my $tbl = $self->{tbl};
4036
my $index = $tbl->{tbl_struct}->{keys}->{$self->{index}};
4037
my $n_cols = scalar @{$index->{cols}};
4038
my $chunkno = $self->{nibbleno};
4039
die "Possible infinite loop detected! "
4040
. "The lower boundary for chunk $chunkno is "
4041
. "<" . join(', ', @{$self->{lower}}) . "> and the lower "
4042
. "boundary for chunk " . ($chunkno + 1) . " is also "
4043
. "<" . join(', ', @{$self->{next_lower}}) . ">. "
4044
. "This usually happens when using a non-unique single "
4045
. "column index. The current chunk index for table "
4046
. "$tbl->{db}.$tbl->{tbl} is $self->{index} which is"
4047
. ($index->{is_unique} ? '' : ' not') . " unique and covers "
4048
. ($n_cols > 1 ? "$n_cols columns" : "1 column") . ".\n";
4049
}
4050
$self->{lower} = $self->{next_lower};
4051
4052
if ( my $callback = $self->{callbacks}->{next_boundaries} ) {
4053
my $oktonibble = $callback->(
4054
Cxn => $self->{Cxn},
4055
tbl => $self->{tbl},
4056
NibbleIterator => $self,
4057
);
4058
PTDEBUG && _d('next_boundaries callback returned', $oktonibble);
4059
if ( !$oktonibble ) {
4060
$self->{no_more_boundaries} = 1;
4061
return; # stop nibbling
4062
}
4063
}
4064
4065
PTDEBUG && _d($self->{ub_sth}->{Statement}, 'params:',
4066
join(', ', @{$self->{lower}}), $self->{limit});
4067
$self->{ub_sth}->execute(@{$self->{lower}}, $self->{limit});
4068
my $boundary = $self->{ub_sth}->fetchall_arrayref();
4069
PTDEBUG && _d('Next boundary:', Dumper($boundary));
4070
if ( $boundary && @$boundary ) {
4071
$self->{upper} = $boundary->[0]; # this nibble
4072
if ( $boundary->[1] ) {
4073
$self->{next_lower} = $boundary->[1]; # next nibble
4074
}
4075
else {
4076
$self->{no_more_boundaries} = 1; # for next call
4077
PTDEBUG && _d('Last upper boundary:', Dumper($boundary->[0]));
4078
}
4079
}
4080
else {
4081
$self->{no_more_boundaries} = 1; # for next call
4082
$self->{upper} = $self->{last_upper};
4083
PTDEBUG && _d('Last upper boundary:', Dumper($self->{upper}));
4084
}
4085
$self->{ub_sth}->finish();
4086
4087
return 1; # continue nibbling
4088
}
4089
4090
sub identical_boundaries {
4091
my ($self, $b1, $b2) = @_;
4092
4093
return 0 if ($b1 && !$b2) || (!$b1 && $b2);
4094
4095
return 1 if !$b1 && !$b2;
4096
4097
die "Boundaries have different numbers of values"
4098
if scalar @$b1 != scalar @$b2; # shouldn't happen
4099
my $n_vals = scalar @$b1;
4100
for my $i ( 0..($n_vals-1) ) {
4101
return 0 if $b1->[$i] ne $b2->[$i]; # diff
4102
}
4103
return 1;
4104
}
4105
4106
sub DESTROY {
4107
my ( $self ) = @_;
4108
foreach my $key ( keys %$self ) {
4109
if ( $key =~ m/_sth$/ ) {
4110
PTDEBUG && _d('Finish', $key);
4111
$self->{$key}->finish();
4112
}
4113
}
4114
return;
4115
}
4116
4117
sub _d {
4118
my ($package, undef, $line) = caller 0;
4119
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
4120
map { defined $_ ? $_ : 'undef' }
4121
@_;
4122
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
4123
}
4124
4125
1;
4126
}
4127
# ###########################################################################
4128
# End NibbleIterator package
4129
# ###########################################################################
4130
4131
# ###########################################################################
4132
# OobNibbleIterator package
4133
# This package is a copy without comments from the original. The original
4134
# with comments and its test file can be found in the Bazaar repository at,
4135
# lib/OobNibbleIterator.pm
4136
# t/lib/OobNibbleIterator.t
4137
# See https://launchpad.net/percona-toolkit for more information.
4138
# ###########################################################################
4139
{
4140
package OobNibbleIterator;
4141
use base 'NibbleIterator';
4142
4143
use strict;
4144
use warnings FATAL => 'all';
4145
use English qw(-no_match_vars);
4146
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
4147
4148
use Data::Dumper;
4149
$Data::Dumper::Indent = 1;
4150
$Data::Dumper::Sortkeys = 1;
4151
$Data::Dumper::Quotekeys = 0;
4152
4153
sub new {
4154
my ( $class, %args ) = @_;
4155
my @required_args = qw();
4156
foreach my $arg ( @required_args ) {
4157
die "I need a $arg argument" unless $args{$arg};
4158
}
4159
4160
my $self = $class->SUPER::new(%args);
4161
4162
my $q = $self->{Quoter};
4163
my $o = $self->{OptionParser};
4164
my $where = $o->get('where');
4165
4166
if ( !$self->one_nibble() ) {
4167
my $head_sql
4168
= ($args{past_dml} || "SELECT ")
4169
. ($args{past_select}
4170
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
4171
. " FROM " . $self->{sql}->{from};
4172
4173
my $tail_sql
4174
= ($where ? " AND ($where)" : '')
4175
. " ORDER BY " . $self->{sql}->{order_by};
4176
4177
my $past_lower_sql
4178
= $head_sql
4179
. " WHERE " . $self->{sql}->{boundaries}->{'<'}
4180
. $tail_sql
4181
. " /*past lower chunk*/";
4182
PTDEBUG && _d('Past lower statement:', $past_lower_sql);
4183
4184
my $explain_past_lower_sql
4185
= "EXPLAIN SELECT "
4186
. ($args{past_select}
4187
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
4188
. " FROM " . $self->{sql}->{from}
4189
. " WHERE " . $self->{sql}->{boundaries}->{'<'}
4190
. $tail_sql
4191
. " /*explain past lower chunk*/";
4192
PTDEBUG && _d('Past lower statement:', $explain_past_lower_sql);
4193
4194
my $past_upper_sql
4195
= $head_sql
4196
. " WHERE " . $self->{sql}->{boundaries}->{'>'}
4197
. $tail_sql
4198
. " /*past upper chunk*/";
4199
PTDEBUG && _d('Past upper statement:', $past_upper_sql);
4200
4201
my $explain_past_upper_sql
4202
= "EXPLAIN SELECT "
4203
. ($args{past_select}
4204
|| join(', ', map { $q->quote($_) } @{$self->{sql}->{columns}}))
4205
. " FROM " . $self->{sql}->{from}
4206
. " WHERE " . $self->{sql}->{boundaries}->{'>'}
4207
. $tail_sql
4208
. " /*explain past upper chunk*/";
4209
PTDEBUG && _d('Past upper statement:', $explain_past_upper_sql);
4210
4211
$self->{past_lower_sql} = $past_lower_sql;
4212
$self->{past_upper_sql} = $past_upper_sql;
4213
$self->{explain_past_lower_sql} = $explain_past_lower_sql;
4214
$self->{explain_past_upper_sql} = $explain_past_upper_sql;
4215
4216
$self->{past_nibbles} = [qw(lower upper)];
4217
if ( my $nibble = $args{resume} ) {
4218
if ( !defined $nibble->{lower_boundary}
4219
|| !defined $nibble->{upper_boundary} ) {
4220
$self->{past_nibbles} = !defined $nibble->{lower_boundary}
4221
? ['upper']
4222
: [];
4223
}
4224
}
4225
PTDEBUG && _d('Nibble past', @{$self->{past_nibbles}});
4226
4227
} # not one nibble
4228
4229
return bless $self, $class;
4230
}
4231
4232
sub more_boundaries {
4233
my ($self) = @_;
4234
return $self->SUPER::more_boundaries() if $self->{one_nibble};
4235
return scalar @{$self->{past_nibbles}} ? 1 : 0;
4236
}
4237
4238
sub statements {
4239
my ($self) = @_;
4240
4241
my $sths = $self->SUPER::statements();
4242
4243
$sths->{past_lower_boundary} = $self->{past_lower_sth};
4244
$sths->{past_upper_boundary} = $self->{past_upper_sth};
4245
4246
return $sths;
4247
}
4248
4249
sub _prepare_sths {
4250
my ($self) = @_;
4251
PTDEBUG && _d('Preparing out-of-bound statement handles');
4252
4253
if ( !$self->{one_nibble} ) {
4254
my $dbh = $self->{Cxn}->dbh();
4255
$self->{past_lower_sth} = $dbh->prepare($self->{past_lower_sql});
4256
$self->{past_upper_sth} = $dbh->prepare($self->{past_upper_sql});
4257
$self->{explain_past_lower_sth} = $dbh->prepare($self->{explain_past_lower_sql});
4258
$self->{explain_past_upper_sth} = $dbh->prepare($self->{explain_past_upper_sql});
4259
}
4260
4261
return $self->SUPER::_prepare_sths();
4262
}
4263
4264
sub _next_boundaries {
4265
my ($self) = @_;
4266
4267
return $self->SUPER::_next_boundaries() unless $self->{no_more_boundaries};
4268
4269
if ( my $past = shift @{$self->{past_nibbles}} ) {
4270
if ( $past eq 'lower' ) {
4271
PTDEBUG && _d('Nibbling values below lower boundary');
4272
$self->{nibble_sth} = $self->{past_lower_sth};
4273
$self->{explain_nibble_sth} = $self->{explain_past_lower_sth};
4274
$self->{lower} = [];
4275
$self->{upper} = $self->boundaries()->{first_lower};
4276
$self->{next_lower} = undef;
4277
}
4278
elsif ( $past eq 'upper' ) {
4279
PTDEBUG && _d('Nibbling values above upper boundary');
4280
$self->{nibble_sth} = $self->{past_upper_sth};
4281
$self->{explain_nibble_sth} = $self->{explain_past_upper_sth};
4282
$self->{lower} = $self->boundaries()->{last_upper};
4283
$self->{upper} = [];
4284
$self->{next_lower} = undef;
4285
}
4286
else {
4287
die "Invalid past nibble: $past";
4288
}
4289
return 1; # continue nibbling
4290
}
4291
4292
PTDEBUG && _d('Done nibbling past boundaries');
4293
return; # stop nibbling
4294
}
4295
4296
sub DESTROY {
4297
my ( $self ) = @_;
4298
foreach my $key ( keys %$self ) {
4299
if ( $key =~ m/_sth$/ ) {
4300
PTDEBUG && _d('Finish', $key);
4301
$self->{$key}->finish();
4302
}
4303
}
4304
return;
4305
}
4306
4307
sub _d {
4308
my ($package, undef, $line) = caller 0;
4309
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
4310
map { defined $_ ? $_ : 'undef' }
4311
@_;
4312
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
4313
}
4314
4315
1;
4316
}
4317
# ###########################################################################
4318
# End OobNibbleIterator package
4319
# ###########################################################################
4320
4321
# ###########################################################################
4190
4322
# Daemon package
4191
4323
# This package is a copy without comments from the original. The original
4192
4324
# with comments and its test file can be found in the Bazaar repository at,
4200
4332
use strict;
4201
4333
use warnings FATAL => 'all';
4202
4334
use English qw(-no_match_vars);
4203
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
4335
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
4204
4336
4205
4337
use POSIX qw(setsid);
4206
4338
4218
4350
4219
4351
check_PID_file(undef, $self->{PID_file});
4220
4352
4221
MKDEBUG && _d('Daemonized child will log to', $self->{log_file});
4353
PTDEBUG && _d('Daemonized child will log to', $self->{log_file});
4222
4354
return bless $self, $class;
4223
4355
}
4224
4356
4225
4357
sub daemonize {
4226
4358
my ( $self ) = @_;
4227
4359
4228
MKDEBUG && _d('About to fork and daemonize');
4360
PTDEBUG && _d('About to fork and daemonize');
4229
4361
defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR";
4230
4362
if ( $pid ) {
4231
MKDEBUG && _d('I am the parent and now I die');
4363
PTDEBUG && _d('I am the parent and now I die');
4232
4364
exit;
4233
4365
}
4234
4366
4270
4402
}
4271
4403
}
4272
4404
4273
MKDEBUG && _d('I am the child and now I live daemonized');
4405
PTDEBUG && _d('I am the child and now I live daemonized');
4274
4406
return;
4275
4407
}
4276
4408
4277
4409
sub check_PID_file {
4278
4410
my ( $self, $file ) = @_;
4279
4411
my $PID_file = $self ? $self->{PID_file} : $file;
4280
MKDEBUG && _d('Checking PID file', $PID_file);
4412
PTDEBUG && _d('Checking PID file', $PID_file);
4281
4413
if ( $PID_file && -f $PID_file ) {
4282
4414
my $pid;
4283
4415
eval { chomp($pid = `cat $PID_file`); };
4284
4416
die "Cannot cat $PID_file: $OS_ERROR" if $EVAL_ERROR;
4285
MKDEBUG && _d('PID file exists; it contains PID', $pid);
4417
PTDEBUG && _d('PID file exists; it contains PID', $pid);
4286
4418
if ( $pid ) {
4287
4419
my $pid_is_alive = kill 0, $pid;
4288
4420
if ( $pid_is_alive ) {
4300
4432
}
4301
4433
}
4302
4434
else {
4303
MKDEBUG && _d('No PID file');
4435
PTDEBUG && _d('No PID file');
4304
4436
}
4305
4437
return;
4306
4438
}
4320
4452
4321
4453
my $PID_file = $self->{PID_file};
4322
4454
if ( !$PID_file ) {
4323
MKDEBUG && _d('No PID file to create');
4455
PTDEBUG && _d('No PID file to create');
4324
4456
return;
4325
4457
}
4326
4458
4333
4465
close $PID_FH
4334
4466
or die "Cannot close PID file $PID_file: $OS_ERROR";
4335
4467
4336
MKDEBUG && _d('Created PID file:', $self->{PID_file});
4468
PTDEBUG && _d('Created PID file:', $self->{PID_file});
4337
4469
return;
4338
4470
}
4339
4471
4342
4474
if ( $self->{PID_file} && -f $self->{PID_file} ) {
4343
4475
unlink $self->{PID_file}
4344
4476
or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR";
4345
MKDEBUG && _d('Removed PID file');
4477
PTDEBUG && _d('Removed PID file');
4346
4478
}
4347
4479
else {
4348
MKDEBUG && _d('No PID to remove');
4480
PTDEBUG && _d('No PID to remove');
4349
4481
}
4350
4482
return;
4351
4483
}
4373
4505
# ###########################################################################
4374
4506
4375
4507
# ###########################################################################
4376
# SchemaIterator r7512
4377
# Don't update this package!
4508
# SchemaIterator package
4509
# This package is a copy without comments from the original. The original
4510
# with comments and its test file can be found in the Bazaar repository at,
4511
# lib/SchemaIterator.pm
4512
# t/lib/SchemaIterator.t
4513
# See https://launchpad.net/percona-toolkit for more information.
4378
4514
# ###########################################################################
4515
{
4379
4516
package SchemaIterator;
4380
4517
4381
{ # package scope
4382
4518
use strict;
4383
4519
use warnings FATAL => 'all';
4384
4520
use English qw(-no_match_vars);
4385
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
4521
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
4386
4522
4387
4523
use Data::Dumper;
4388
4524
$Data::Dumper::Indent = 1;
4410
4546
die "I need either a dbh or file_itr argument"
4411
4547
if (!$dbh && !$file_itr) || ($dbh && $file_itr);
4412
4548
4549
my %resume;
4550
if ( my $table = $args{resume} ) {
4551
PTDEBUG && _d('Will resume from or after', $table);
4552
my ($db, $tbl) = $args{Quoter}->split_unquote($table);
4553
die "Resume table must be database-qualified: $table"
4554
unless $db && $tbl;
4555
$resume{db} = $db;
4556
$resume{tbl} = $tbl;
4557
}
4558
4413
4559
my $self = {
4414
4560
%args,
4561
resume => \%resume,
4415
4562
filters => _make_filters(%args),
4416
4563
};
4417
4564
4444
4591
if ( $is_table ) {
4445
4592
my ($db, $tbl) = $q->split_unquote($obj);
4446
4593
$db ||= '*';
4447
MKDEBUG && _d('Filter', $filter, 'value:', $db, $tbl);
4594
PTDEBUG && _d('Filter', $filter, 'value:', $db, $tbl);
4448
4595
$filters{$filter}->{$tbl} = $db;
4449
4596
}
4450
4597
else { # database
4451
MKDEBUG && _d('Filter', $filter, 'value:', $obj);
4598
PTDEBUG && _d('Filter', $filter, 'value:', $obj);
4452
4599
$filters{$filter}->{$obj} = 1;
4453
4600
}
4454
4601
}
4464
4611
my $pat = $o->get($filter);
4465
4612
next REGEX_FILTER unless $pat;
4466
4613
$filters{$filter} = qr/$pat/;
4467
MKDEBUG && _d('Filter', $filter, 'value:', $filters{$filter});
4614
PTDEBUG && _d('Filter', $filter, 'value:', $filters{$filter});
4468
4615
}
4469
4616
}
4470
4617
4471
MKDEBUG && _d('Schema object filters:', Dumper(\%filters));
4618
PTDEBUG && _d('Schema object filters:', Dumper(\%filters));
4472
4619
return \%filters;
4473
4620
}
4474
4621
4475
sub next_schema_object {
4622
sub next {
4476
4623
my ( $self ) = @_;
4477
4624
4478
my %schema_object;
4625
if ( !$self->{initialized} ) {
4626
$self->{initialized} = 1;
4627
if ( $self->{resume}->{tbl}
4628
&& !$self->table_is_allowed(@{$self->{resume}}{qw(db tbl)}) ) {
4629
PTDEBUG && _d('Will resume after',
4630
join('.', @{$self->{resume}}{qw(db tbl)}));
4631
$self->{resume}->{after} = 1;
4632
}
4633
}
4634
4635
my $schema_obj;
4479
4636
if ( $self->{file_itr} ) {
4480
%schema_object = $self->_iterate_files();
4637
$schema_obj= $self->_iterate_files();
4481
4638
}
4482
4639
else { # dbh
4483
%schema_object = $self->_iterate_dbh();
4484
}
4485
4486
MKDEBUG && _d('Next schema object:', Dumper(\%schema_object));
4487
return %schema_object;
4640
$schema_obj= $self->_iterate_dbh();
4641
}
4642
4643
if ( $schema_obj ) {
4644
if ( $schema_obj->{ddl} && $self->{TableParser} ) {
4645
$schema_obj->{tbl_struct}
4646
= $self->{TableParser}->parse($schema_obj->{ddl});
4647
}
4648
4649
delete $schema_obj->{ddl} unless $self->{keep_ddl};
4650
delete $schema_obj->{tbl_status} unless $self->{keep_tbl_status};
4651
4652
if ( my $schema = $self->{Schema} ) {
4653
$schema->add_schema_object($schema_obj);
4654
}
4655
PTDEBUG && _d('Next schema object:', $schema_obj->{db}, $schema_obj->{tbl});
4656
}
4657
4658
return $schema_obj;
4488
4659
}
4489
4660
4490
4661
sub _iterate_files {
4493
4664
if ( !$self->{fh} ) {
4494
4665
my ($fh, $file) = $self->{file_itr}->();
4495
4666
if ( !$fh ) {
4496
MKDEBUG && _d('No more files to iterate');
4667
PTDEBUG && _d('No more files to iterate');
4497
4668
return;
4498
4669
}
4499
4670
$self->{fh} = $fh;
4500
4671
$self->{file} = $file;
4501
4672
}
4502
4673
my $fh = $self->{fh};
4503
MKDEBUG && _d('Getting next schema object from', $self->{file});
4674
PTDEBUG && _d('Getting next schema object from', $self->{file});
4504
4675
4505
4676
local $INPUT_RECORD_SEPARATOR = '';
4506
4677
CHUNK:
4509
4680
my $db = $1; # XXX
4510
4681
$db =~ s/^`//; # strip leading `
4511
4682
$db =~ s/`$//; # and trailing `
4512
if ( $self->database_is_allowed($db) ) {
4683
if ( $self->database_is_allowed($db)
4684
&& $self->_resume_from_database($db) ) {
4513
4685
$self->{db} = $db;
4514
4686
}
4515
4687
}
4516
4688
elsif ($self->{db} && $chunk =~ m/CREATE TABLE/) {
4517
4689
if ($chunk =~ m/DROP VIEW IF EXISTS/) {
4518
MKDEBUG && _d('Table is a VIEW, skipping');
4690
PTDEBUG && _d('Table is a VIEW, skipping');
4519
4691
next CHUNK;
4520
4692
}
4521
4693
4522
4694
my ($tbl) = $chunk =~ m/$tbl_name/;
4523
4695
$tbl =~ s/^\s*`//;
4524
4696
$tbl =~ s/`\s*$//;
4525
if ( $self->table_is_allowed($self->{db}, $tbl) ) {
4697
if ( $self->_resume_from_table($tbl)
4698
&& $self->table_is_allowed($self->{db}, $tbl) ) {
4526
4699
my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;
4527
4700
if ( !$ddl ) {
4528
4701
warn "Failed to parse CREATE TABLE from\n" . $chunk;
4533
4706
my ($engine) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/;
4534
4707
4535
4708
if ( !$engine || $self->engine_is_allowed($engine) ) {
4536
return (
4709
return {
4537
4710
db => $self->{db},
4538
4711
tbl => $tbl,
4539
4712
ddl => $ddl,
4540
);
4713
};
4541
4714
}
4542
4715
}
4543
4716
}
4544
4717
} # CHUNK
4545
4718
4546
MKDEBUG && _d('No more schema objects in', $self->{file});
4719
PTDEBUG && _d('No more schema objects in', $self->{file});
4547
4720
close $self->{fh};
4548
4721
$self->{fh} = undef;
4549
4722
4554
4727
my ( $self ) = @_;
4555
4728
my $q = $self->{Quoter};
4556
4729
my $dbh = $self->{dbh};
4557
MKDEBUG && _d('Getting next schema object from dbh', $dbh);
4730
PTDEBUG && _d('Getting next schema object from dbh', $dbh);
4558
4731
4559
4732
if ( !defined $self->{dbs} ) {
4560
4733
my $sql = 'SHOW DATABASES';
4561
MKDEBUG && _d($sql);
4734
PTDEBUG && _d($sql);
4562
4735
my @dbs = grep { $self->database_is_allowed($_) }
4563
4736
@{$dbh->selectcol_arrayref($sql)};
4564
MKDEBUG && _d('Found', scalar @dbs, 'databases');
4737
PTDEBUG && _d('Found', scalar @dbs, 'databases');
4565
4738
$self->{dbs} = \@dbs;
4566
4739
}
4567
4740
4568
4741
if ( !$self->{db} ) {
4569
$self->{db} = shift @{$self->{dbs}};
4570
MKDEBUG && _d('Next database:', $self->{db});
4742
do {
4743
$self->{db} = shift @{$self->{dbs}};
4744
} until $self->_resume_from_database($self->{db});
4745
PTDEBUG && _d('Next database:', $self->{db});
4571
4746
return unless $self->{db};
4572
4747
}
4573
4748
4574
4749
if ( !defined $self->{tbls} ) {
4575
4750
my $sql = 'SHOW /*!50002 FULL*/ TABLES FROM ' . $q->quote($self->{db});
4576
MKDEBUG && _d($sql);
4751
PTDEBUG && _d($sql);
4577
4752
my @tbls = map {
4578
4753
$_->[0]; # (tbl, type)
4579
4754
}
4580
4755
grep {
4581
4756
my ($tbl, $type) = @$_;
4582
$self->table_is_allowed($self->{db}, $tbl)
4583
&& (!$type || ($type ne 'VIEW'));
4757
(!$type || ($type ne 'VIEW'))
4758
&& $self->_resume_from_table($tbl)
4759
&& $self->table_is_allowed($self->{db}, $tbl);
4584
4760
}
4585
4761
@{$dbh->selectall_arrayref($sql)};
4586
MKDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
4762
PTDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
4587
4763
$self->{tbls} = \@tbls;
4588
4764
}
4589
4765
4590
4766
while ( my $tbl = shift @{$self->{tbls}} ) {
4591
my $engine;
4767
my $tbl_status;
4592
4768
if ( $self->{filters}->{'engines'}
4593
|| $self->{filters}->{'ignore-engines'} ) {
4769
|| $self->{filters}->{'ignore-engines'}
4770
|| $self->{keep_tbl_status} )
4771
{
4594
4772
my $sql = "SHOW TABLE STATUS FROM " . $q->quote($self->{db})
4595
4773
. " LIKE \'$tbl\'";
4596
MKDEBUG && _d($sql);
4597
$engine = $dbh->selectrow_hashref($sql)->{engine};
4598
MKDEBUG && _d($tbl, 'uses', $engine, 'engine');
4774
PTDEBUG && _d($sql);
4775
$tbl_status = $dbh->selectrow_hashref($sql);
4776
PTDEBUG && _d(Dumper($tbl_status));
4599
4777
}
4600
4778
4601
4602
if ( !$engine || $self->engine_is_allowed($engine) ) {
4779
if ( !$tbl_status
4780
|| $self->engine_is_allowed($tbl_status->{engine}) ) {
4603
4781
my $ddl;
4604
if ( my $du = $self->{MySQLDump} ) {
4605
$ddl = $du->get_create_table($dbh, $q, $self->{db}, $tbl)->[1];
4782
if ( my $tp = $self->{TableParser} ) {
4783
$ddl = $tp->get_create_table($dbh, $self->{db}, $tbl);
4606
4784
}
4607
4785
4608
return (
4609
db => $self->{db},
4610
tbl => $tbl,
4611
ddl => $ddl,
4612
);
4786
return {
4787
db => $self->{db},
4788
tbl => $tbl,
4789
ddl => $ddl,
4790
tbl_status => $tbl_status,
4791
};
4613
4792
}
4614
4793
}
4615
4794
4616
MKDEBUG && _d('No more tables in database', $self->{db});
4795
PTDEBUG && _d('No more tables in database', $self->{db});
4617
4796
$self->{db} = undef;
4618
4797
$self->{tbls} = undef;
4619
4798
4629
4808
my $filter = $self->{filters};
4630
4809
4631
4810
if ( $db =~ m/information_schema|performance_schema|lost\+found/ ) {
4632
MKDEBUG && _d('Database', $db, 'is a system database, ignoring');
4811
PTDEBUG && _d('Database', $db, 'is a system database, ignoring');
4633
4812
return 0;
4634
4813
}
4635
4814
4636
4815
if ( $self->{filters}->{'ignore-databases'}->{$db} ) {
4637
MKDEBUG && _d('Database', $db, 'is in --ignore-databases list');
4816
PTDEBUG && _d('Database', $db, 'is in --ignore-databases list');
4638
4817
return 0;
4639
4818
}
4640
4819
4641
4820
if ( $filter->{'ignore-databases-regex'}
4642
4821
&& $db =~ $filter->{'ignore-databases-regex'} ) {
4643
MKDEBUG && _d('Database', $db, 'matches --ignore-databases-regex');
4822
PTDEBUG && _d('Database', $db, 'matches --ignore-databases-regex');
4644
4823
return 0;
4645
4824
}
4646
4825
4647
4826
if ( $filter->{'databases'}
4648
4827
&& !$filter->{'databases'}->{$db} ) {
4649
MKDEBUG && _d('Database', $db, 'is not in --databases list, ignoring');
4828
PTDEBUG && _d('Database', $db, 'is not in --databases list, ignoring');
4650
4829
return 0;
4651
4830
}
4652
4831
4653
4832
if ( $filter->{'databases-regex'}
4654
4833
&& $db !~ $filter->{'databases-regex'} ) {
4655
MKDEBUG && _d('Database', $db, 'does not match --databases-regex, ignoring');
4834
PTDEBUG && _d('Database', $db, 'does not match --databases-regex, ignoring');
4656
4835
return 0;
4657
4836
}
4658
4837
4669
4848
4670
4849
my $filter = $self->{filters};
4671
4850
4851
if ( $db eq 'mysql' && ($tbl eq 'general_log' || $tbl eq 'slow_log') ) {
4852
return 0;
4853
}
4854
4672
4855
if ( $filter->{'ignore-tables'}->{$tbl}
4673
4856
&& ($filter->{'ignore-tables'}->{$tbl} eq '*'
4674
4857
|| $filter->{'ignore-tables'}->{$tbl} eq $db) ) {
4675
MKDEBUG && _d('Table', $tbl, 'is in --ignore-tables list');
4858
PTDEBUG && _d('Table', $tbl, 'is in --ignore-tables list');
4676
4859
return 0;
4677
4860
}
4678
4861
4679
4862
if ( $filter->{'ignore-tables-regex'}
4680
4863
&& $tbl =~ $filter->{'ignore-tables-regex'} ) {
4681
MKDEBUG && _d('Table', $tbl, 'matches --ignore-tables-regex');
4864
PTDEBUG && _d('Table', $tbl, 'matches --ignore-tables-regex');
4682
4865
return 0;
4683
4866
}
4684
4867
4685
4868
if ( $filter->{'tables'}
4686
4869
&& !$filter->{'tables'}->{$tbl} ) {
4687
MKDEBUG && _d('Table', $tbl, 'is not in --tables list, ignoring');
4870
PTDEBUG && _d('Table', $tbl, 'is not in --tables list, ignoring');
4688
4871
return 0;
4689
4872
}
4690
4873
4691
4874
if ( $filter->{'tables-regex'}
4692
4875
&& $tbl !~ $filter->{'tables-regex'} ) {
4693
MKDEBUG && _d('Table', $tbl, 'does not match --tables-regex, ignoring');
4876
PTDEBUG && _d('Table', $tbl, 'does not match --tables-regex, ignoring');
4694
4877
return 0;
4695
4878
}
4696
4879
4698
4881
&& $filter->{'tables'}->{$tbl}
4699
4882
&& $filter->{'tables'}->{$tbl} ne '*'
4700
4883
&& $filter->{'tables'}->{$tbl} ne $db ) {
4701
MKDEBUG && _d('Table', $tbl, 'is only allowed in database',
4884
PTDEBUG && _d('Table', $tbl, 'is only allowed in database',
4702
4885
$filter->{'tables'}->{$tbl});
4703
4886
return 0;
4704
4887
}
4715
4898
my $filter = $self->{filters};
4716
4899
4717
4900
if ( $filter->{'ignore-engines'}->{$engine} ) {
4718
MKDEBUG && _d('Engine', $engine, 'is in --ignore-databases list');
4901
PTDEBUG && _d('Engine', $engine, 'is in --ignore-databases list');
4719
4902
return 0;
4720
4903
}
4721
4904
4722
4905
if ( $filter->{'engines'}
4723
4906
&& !$filter->{'engines'}->{$engine} ) {
4724
MKDEBUG && _d('Engine', $engine, 'is not in --engines list, ignoring');
4907
PTDEBUG && _d('Engine', $engine, 'is not in --engines list, ignoring');
4725
4908
return 0;
4726
4909
}
4727
4910
4728
4911
return 1;
4729
4912
}
4730
4913
4914
sub _resume_from_database {
4915
my ($self, $db) = @_;
4916
4917
return 1 unless $self->{resume}->{db};
4918
4919
if ( $db eq $self->{resume}->{db} ) {
4920
PTDEBUG && _d('At resume db', $db);
4921
delete $self->{resume}->{db};
4922
return 1;
4923
}
4924
4925
return 0;
4926
}
4927
4928
sub _resume_from_table {
4929
my ($self, $tbl) = @_;
4930
4931
return 1 unless $self->{resume}->{tbl};
4932
4933
if ( $tbl eq $self->{resume}->{tbl} ) {
4934
if ( !$self->{resume}->{after} ) {
4935
PTDEBUG && _d('Resuming from table', $tbl);
4936
delete $self->{resume}->{tbl};
4937
return 1;
4938
}
4939
else {
4940
PTDEBUG && _d('Resuming after table', $tbl);
4941
delete $self->{resume}->{tbl};
4942
}
4943
}
4944
4945
return 0;
4946
}
4947
4731
4948
sub _d {
4732
4949
my ($package, undef, $line) = caller 0;
4733
4950
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
4736
4953
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
4737
4954
}
4738
4955
4739
} # package scope
4740
4956
1;
4741
4957
}
4742
4958
# ###########################################################################
4743
4959
# End SchemaIterator package
4744
4960
# ###########################################################################
4757
4973
use strict;
4758
4974
use warnings FATAL => 'all';
4759
4975
use English qw(-no_match_vars);
4760
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
4976
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
4761
4977
4762
4978
sub new {
4763
4979
my ( $class, %args ) = @_;
4769
4985
4770
4986
sub retry {
4771
4987
my ( $self, %args ) = @_;
4772
my @required_args = qw(try wait);
4988
my @required_args = qw(try fail final_fail);
4773
4989
foreach my $arg ( @required_args ) {
4774
4990
die "I need a $arg argument" unless $args{$arg};
4775
4991
};
4776
my ($try, $wait) = @args{@required_args};
4992
my ($try, $fail, $final_fail) = @args{@required_args};
4993
my $wait = $args{wait} || sub { sleep 1; };
4777
4994
my $tries = $args{tries} || 3;
4778
4995
4996
my $last_error;
4779
4997
my $tryno = 0;
4998
TRY:
4780
4999
while ( ++$tryno <= $tries ) {
4781
MKDEBUG && _d("Retry", $tryno, "of", $tries);
5000
PTDEBUG && _d("Try", $tryno, "of", $tries);
4782
5001
my $result;
4783
5002
eval {
4784
5003
$result = $try->(tryno=>$tryno);
4785
5004
};
5005
if ( $EVAL_ERROR ) {
5006
PTDEBUG && _d("Try code failed:", $EVAL_ERROR);
5007
$last_error = $EVAL_ERROR;
4786
5008
4787
if ( defined $result ) {
4788
MKDEBUG && _d("Try code succeeded");
4789
if ( my $on_success = $args{on_success} ) {
4790
MKDEBUG && _d("Calling on_success code");
4791
$on_success->(tryno=>$tryno, result=>$result);
5009
if ( $tryno < $tries ) { # more retries
5010
my $retry = $fail->(tryno=>$tryno, error=>$last_error);
5011
last TRY unless $retry;
5012
PTDEBUG && _d("Calling wait code");
5013
$wait->(tryno=>$tryno);
4792
5014
}
5015
}
5016
else {
5017
PTDEBUG && _d("Try code succeeded");
4793
5018
return $result;
4794
5019
}
4795
4796
if ( $EVAL_ERROR ) {
4797
MKDEBUG && _d("Try code died:", $EVAL_ERROR);
4798
die $EVAL_ERROR unless $args{retry_on_die};
4799
}
4800
4801
if ( $tryno < $tries ) {
4802
MKDEBUG && _d("Try code failed, calling wait code");
4803
$wait->(tryno=>$tryno);
4804
}
4805
}
4806
4807
MKDEBUG && _d("Try code did not succeed");
4808
if ( my $on_failure = $args{on_failure} ) {
4809
MKDEBUG && _d("Calling on_failure code");
4810
$on_failure->();
4811
}
4812
4813
return;
5020
}
5021
5022
PTDEBUG && _d('Try code did not succeed');
5023
return $final_fail->(error=>$last_error);
4814
5024
}
4815
5025
4816
5026
sub _d {
4828
5038
# ###########################################################################
4829
5039
4830
5040
# ###########################################################################
5041
# Transformers package
5042
# This package is a copy without comments from the original. The original
5043
# with comments and its test file can be found in the Bazaar repository at,
5044
# lib/Transformers.pm
5045
# t/lib/Transformers.t
5046
# See https://launchpad.net/percona-toolkit for more information.
5047
# ###########################################################################
5048
{
5049
package Transformers;
5050
5051
use strict;
5052
use warnings FATAL => 'all';
5053
use English qw(-no_match_vars);
5054
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
5055
5056
use Time::Local qw(timegm timelocal);
5057
use Digest::MD5 qw(md5_hex);
5058
5059
require Exporter;
5060
our @ISA = qw(Exporter);
5061
our %EXPORT_TAGS = ();
5062
our @EXPORT = ();
5063
our @EXPORT_OK = qw(
5064
micro_t
5065
percentage_of
5066
secs_to_time
5067
time_to_secs
5068
shorten
5069
ts
5070
parse_timestamp
5071
unix_timestamp
5072
any_unix_timestamp
5073
make_checksum
5074
crc32
5075
);
5076
5077
our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/;
5078
our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/;
5079
our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks
5080
5081
sub micro_t {
5082
my ( $t, %args ) = @_;
5083
my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals
5084
my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals
5085
my $f;
5086
5087
$t = 0 if $t < 0;
5088
5089
$t = sprintf('%.17f', $t) if $t =~ /e/;
5090
5091
$t =~ s/\.(\d{1,6})\d*/\.$1/;
5092
5093
if ($t > 0 && $t <= 0.000999) {
5094
$f = ($t * 1000000) . 'us';
5095
}
5096
elsif ($t >= 0.001000 && $t <= 0.999999) {
5097
$f = sprintf("%.${p_ms}f", $t * 1000);
5098
$f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros
5099
}
5100
elsif ($t >= 1) {
5101
$f = sprintf("%.${p_s}f", $t);
5102
$f = ($f * 1) . 's'; # * 1 to remove insignificant zeros
5103
}
5104
else {
5105
$f = 0; # $t should = 0 at this point
5106
}
5107
5108
return $f;
5109
}
5110
5111
sub percentage_of {
5112
my ( $is, $of, %args ) = @_;
5113
my $p = $args{p} || 0; # float precision
5114
my $fmt = $p ? "%.${p}f" : "%d";
5115
return sprintf $fmt, ($is * 100) / ($of ||= 1);
5116
}
5117
5118
sub secs_to_time {
5119
my ( $secs, $fmt ) = @_;
5120
$secs ||= 0;
5121
return '00:00' unless $secs;
5122
5123
$fmt ||= $secs >= 86_400 ? 'd'
5124
: $secs >= 3_600 ? 'h'
5125
: 'm';
5126
5127
return
5128
$fmt eq 'd' ? sprintf(
5129
"%d+%02d:%02d:%02d",
5130
int($secs / 86_400),
5131
int(($secs % 86_400) / 3_600),
5132
int(($secs % 3_600) / 60),
5133
$secs % 60)
5134
: $fmt eq 'h' ? sprintf(
5135
"%02d:%02d:%02d",
5136
int(($secs % 86_400) / 3_600),
5137
int(($secs % 3_600) / 60),
5138
$secs % 60)
5139
: sprintf(
5140
"%02d:%02d",
5141
int(($secs % 3_600) / 60),
5142
$secs % 60);
5143
}
5144
5145
sub time_to_secs {
5146
my ( $val, $default_suffix ) = @_;
5147
die "I need a val argument" unless defined $val;
5148
my $t = 0;
5149
my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/;
5150
$suffix = $suffix || $default_suffix || 's';
5151
if ( $suffix =~ m/[smhd]/ ) {
5152
$t = $suffix eq 's' ? $num * 1 # Seconds
5153
: $suffix eq 'm' ? $num * 60 # Minutes
5154
: $suffix eq 'h' ? $num * 3600 # Hours
5155
: $num * 86400; # Days
5156
5157
$t *= -1 if $prefix && $prefix eq '-';
5158
}
5159
else {
5160
die "Invalid suffix for $val: $suffix";
5161
}
5162
return $t;
5163
}
5164
5165
sub shorten {
5166
my ( $num, %args ) = @_;
5167
my $p = defined $args{p} ? $args{p} : 2; # float precision
5168
my $d = defined $args{d} ? $args{d} : 1_024; # divisor
5169
my $n = 0;
5170
my @units = ('', qw(k M G T P E Z Y));
5171
while ( $num >= $d && $n < @units - 1 ) {
5172
$num /= $d;
5173
++$n;
5174
}
5175
return sprintf(
5176
$num =~ m/\./ || $n
5177
? "%.${p}f%s"
5178
: '%d',
5179
$num, $units[$n]);
5180
}
5181
5182
sub ts {
5183
my ( $time, $gmt ) = @_;
5184
my ( $sec, $min, $hour, $mday, $mon, $year )
5185
= $gmt ? gmtime($time) : localtime($time);
5186
$mon += 1;
5187
$year += 1900;
5188
my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d",
5189
$year, $mon, $mday, $hour, $min, $sec);
5190
if ( my ($us) = $time =~ m/(\.\d+)$/ ) {
5191
$us = sprintf("%.6f", $us);
5192
$us =~ s/^0\././;
5193
$val .= $us;
5194
}
5195
return $val;
5196
}
5197
5198
sub parse_timestamp {
5199
my ( $val ) = @_;
5200
if ( my($y, $m, $d, $h, $i, $s, $f)
5201
= $val =~ m/^$mysql_ts$/ )
5202
{
5203
return sprintf "%d-%02d-%02d %02d:%02d:"
5204
. (defined $f ? '%09.6f' : '%02d'),
5205
$y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s);
5206
}
5207
return $val;
5208
}
5209
5210
sub unix_timestamp {
5211
my ( $val, $gmt ) = @_;
5212
if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) {
5213
$val = $gmt
5214
? timegm($s, $i, $h, $d, $m - 1, $y)
5215
: timelocal($s, $i, $h, $d, $m - 1, $y);
5216
if ( defined $us ) {
5217
$us = sprintf('%.6f', $us);
5218
$us =~ s/^0\././;
5219
$val .= $us;
5220
}
5221
}
5222
return $val;
5223
}
5224
5225
sub any_unix_timestamp {
5226
my ( $val, $callback ) = @_;
5227
5228
if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) {
5229
$n = $suffix eq 's' ? $n # Seconds
5230
: $suffix eq 'm' ? $n * 60 # Minutes
5231
: $suffix eq 'h' ? $n * 3600 # Hours
5232
: $suffix eq 'd' ? $n * 86400 # Days
5233
: $n; # default: Seconds
5234
PTDEBUG && _d('ts is now - N[shmd]:', $n);
5235
return time - $n;
5236
}
5237
elsif ( $val =~ m/^\d{9,}/ ) {
5238
PTDEBUG && _d('ts is already a unix timestamp');
5239
return $val;
5240
}
5241
elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) {
5242
PTDEBUG && _d('ts is MySQL slow log timestamp');
5243
$val .= ' 00:00:00' unless $hms;
5244
return unix_timestamp(parse_timestamp($val));
5245
}
5246
elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) {
5247
PTDEBUG && _d('ts is properly formatted timestamp');
5248
$val .= ' 00:00:00' unless $hms;
5249
return unix_timestamp($val);
5250
}
5251
else {
5252
PTDEBUG && _d('ts is MySQL expression');
5253
return $callback->($val) if $callback && ref $callback eq 'CODE';
5254
}
5255
5256
PTDEBUG && _d('Unknown ts type:', $val);
5257
return;
5258
}
5259
5260
sub make_checksum {
5261
my ( $val ) = @_;
5262
my $checksum = uc substr(md5_hex($val), -16);
5263
PTDEBUG && _d($checksum, 'checksum for', $val);
5264
return $checksum;
5265
}
5266
5267
sub crc32 {
5268
my ( $string ) = @_;
5269
return unless $string;
5270
my $poly = 0xEDB88320;
5271
my $crc = 0xFFFFFFFF;
5272
foreach my $char ( split(//, $string) ) {
5273
my $comp = ($crc ^ ord($char)) & 0xFF;
5274
for ( 1 .. 8 ) {
5275
$comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1;
5276
}
5277
$crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp;
5278
}
5279
return $crc ^ 0xFFFFFFFF;
5280
}
5281
5282
sub _d {
5283
my ($package, undef, $line) = caller 0;
5284
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
5285
map { defined $_ ? $_ : 'undef' }
5286
@_;
5287
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
5288
}
5289
5290
1;
5291
}
5292
# ###########################################################################
5293
# End Transformers package
5294
# ###########################################################################
5295
5296
# ###########################################################################
4831
5297
# Progress package
4832
5298
# This package is a copy without comments from the original. The original
4833
5299
# with comments and its test file can be found in the Bazaar repository at,
4841
5307
use strict;
4842
5308
use warnings FATAL => 'all';
4843
5309
use English qw(-no_match_vars);
4844
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
5310
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
4845
5311
4846
5312
sub new {
4847
5313
my ( $class, %args ) = @_;
4898
5364
sub start {
4899
5365
my ( $self, $start ) = @_;
4900
5366
$self->{start} = $self->{last_reported} = $start || time();
5367
$self->{first_report} = 0;
4901
5368
}
4902
5369
4903
5370
sub update {
4904
my ( $self, $callback, $now ) = @_;
5371
my ( $self, $callback, %args ) = @_;
4905
5372
my $jobsize = $self->{jobsize};
4906
$now ||= time();
5373
my $now ||= $args{now} || time;
5374
4907
5375
$self->{iterations}++; # How many updates have happened;
4908
5376
5377
if ( !$self->{first_report} && $args{first_report} ) {
5378
$args{first_report}->();
5379
$self->{first_report} = 1;
5380
}
5381
4909
5382
if ( $self->{report} eq 'time'
4910
5383
&& $self->{interval} > $now - $self->{last_reported}
4911
5384
) {
4969
5442
# ###########################################################################
4970
5443
4971
5444
# ###########################################################################
5445
# ReplicaLagWaiter package
5446
# This package is a copy without comments from the original. The original
5447
# with comments and its test file can be found in the Bazaar repository at,
5448
# lib/ReplicaLagWaiter.pm
5449
# t/lib/ReplicaLagWaiter.t
5450
# See https://launchpad.net/percona-toolkit for more information.
5451
# ###########################################################################
5452
{
5453
package ReplicaLagWaiter;
5454
5455
use strict;
5456
use warnings FATAL => 'all';
5457
use English qw(-no_match_vars);
5458
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
5459
5460
use Time::HiRes qw(sleep time);
5461
use Data::Dumper;
5462
5463
sub new {
5464
my ( $class, %args ) = @_;
5465
my @required_args = qw(oktorun get_lag sleep max_lag slaves);
5466
foreach my $arg ( @required_args ) {
5467
die "I need a $arg argument" unless defined $args{$arg};
5468
}
5469
5470
my $self = {
5471
%args,
5472
};
5473
5474
return bless $self, $class;
5475
}
5476
5477
sub wait {
5478
my ( $self, %args ) = @_;
5479
my @required_args = qw();
5480
foreach my $arg ( @required_args ) {
5481
die "I need a $arg argument" unless $args{$arg};
5482
}
5483
my $pr = $args{Progress};
5484
5485
my $oktorun = $self->{oktorun};
5486
my $get_lag = $self->{get_lag};
5487
my $sleep = $self->{sleep};
5488
my $slaves = $self->{slaves};
5489
my $max_lag = $self->{max_lag};
5490
5491
my $worst; # most lagging slave
5492
my $pr_callback;
5493
my $pr_first_report;
5494
if ( $pr ) {
5495
$pr_callback = sub {
5496
my ($fraction, $elapsed, $remaining, $eta, $completed) = @_;
5497
my $dsn_name = $worst->{cxn}->name();
5498
if ( defined $worst->{lag} ) {
5499
print STDERR "Replica lag is " . ($worst->{lag} || '?')
5500
. " seconds on $dsn_name. Waiting.\n";
5501
}
5502
else {
5503
print STDERR "Replica $dsn_name is stopped. Waiting.\n";
5504
}
5505
return;
5506
};
5507
$pr->set_callback($pr_callback);
5508
5509
$pr_first_report = sub {
5510
my $dsn_name = $worst->{cxn}->name();
5511
if ( !defined $worst->{lag} ) {
5512
print STDERR "Replica $dsn_name is stopped. Waiting.\n";
5513
}
5514
return;
5515
};
5516
}
5517
5518
my @lagged_slaves = map { {cxn=>$_, lag=>undef} } @$slaves;
5519
while ( $oktorun->() && @lagged_slaves ) {
5520
PTDEBUG && _d('Checking slave lag');
5521
for my $i ( 0..$#lagged_slaves ) {
5522
my $lag = $get_lag->($lagged_slaves[$i]->{cxn});
5523
PTDEBUG && _d($lagged_slaves[$i]->{cxn}->name(),
5524
'slave lag:', $lag);
5525
if ( !defined $lag || $lag > $max_lag ) {
5526
$lagged_slaves[$i]->{lag} = $lag;
5527
}
5528
else {
5529
delete $lagged_slaves[$i];
5530
}
5531
}
5532
5533
@lagged_slaves = grep { defined $_ } @lagged_slaves;
5534
if ( @lagged_slaves ) {
5535
@lagged_slaves = reverse sort {
5536
defined $a->{lag} && defined $b->{lag} ? $a->{lag} <=> $b->{lag}
5537
: defined $a->{lag} ? -1
5538
: 1;
5539
} @lagged_slaves;
5540
$worst = $lagged_slaves[0];
5541
PTDEBUG && _d(scalar @lagged_slaves, 'slaves are lagging, worst:',
5542
$worst->{lag}, 'on', Dumper($worst->{cxn}->dsn()));
5543
5544
if ( $pr ) {
5545
$pr->update(
5546
sub { return 0; },
5547
first_report => $pr_first_report,
5548
);
5549
}
5550
5551
PTDEBUG && _d('Calling sleep callback');
5552
$sleep->($worst->{cxn}, $worst->{lag});
5553
}
5554
}
5555
5556
PTDEBUG && _d('All slaves caught up');
5557
return;
5558
}
5559
5560
sub _d {
5561
my ($package, undef, $line) = caller 0;
5562
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
5563
map { defined $_ ? $_ : 'undef' }
5564
@_;
5565
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
5566
}
5567
5568
1;
5569
}
5570
# ###########################################################################
5571
# End ReplicaLagWaiter package
5572
# ###########################################################################
5573
5574
# ###########################################################################
5575
# MySQLStatusWaiter package
5576
# This package is a copy without comments from the original. The original
5577
# with comments and its test file can be found in the Bazaar repository at,
5578
# lib/MySQLStatusWaiter.pm
5579
# t/lib/MySQLStatusWaiter.t
5580
# See https://launchpad.net/percona-toolkit for more information.
5581
# ###########################################################################
5582
{
5583
package MySQLStatusWaiter;
5584
5585
use strict;
5586
use warnings FATAL => 'all';
5587
use English qw(-no_match_vars);
5588
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
5589
5590
sub new {
5591
my ( $class, %args ) = @_;
5592
my @required_args = qw(spec get_status sleep oktorun);
5593
foreach my $arg ( @required_args ) {
5594
die "I need a $arg argument" unless defined $args{$arg};
5595
}
5596
5597
my $max_val_for = _parse_spec(%args);
5598
5599
my $self = {
5600
get_status => $args{get_status},
5601
sleep => $args{sleep},
5602
oktorun => $args{oktorun},
5603
max_val_for => $max_val_for,
5604
};
5605
5606
return bless $self, $class;
5607
}
5608
5609
sub _parse_spec {
5610
my ( %args ) = @_;
5611
my @required_args = qw(spec get_status);
5612
foreach my $arg ( @required_args ) {
5613
die "I need a $arg argument" unless defined $args{$arg};
5614
}
5615
my ($spec, $get_status) = @args{@required_args};
5616
5617
if ( !@$spec ) {
5618
PTDEBUG && _d('No spec, disabling status var waits');
5619
return;
5620
}
5621
5622
my %max_val_for;
5623
foreach my $var_val ( @$spec ) {
5624
my ($var, $val) = split /[:=]/, $var_val;
5625
die "Invalid spec: $var_val" unless $var;
5626
if ( !$val ) {
5627
my $init_val = $get_status->($var);
5628
PTDEBUG && _d('Initial', $var, 'value:', $init_val);
5629
$val = int(($init_val * .20) + $init_val);
5630
}
5631
PTDEBUG && _d('Wait if', $var, '>=', $val);
5632
$max_val_for{$var} = $val;
5633
}
5634
5635
return \%max_val_for;
5636
}
5637
5638
sub max_values {
5639
my ($self) = @_;
5640
return $self->{max_val_for};
5641
}
5642
5643
sub wait {
5644
my ( $self, %args ) = @_;
5645
5646
return unless $self->{max_val_for};
5647
5648
my $pr = $args{Progress}; # optional
5649
5650
my $oktorun = $self->{oktorun};
5651
my $get_status = $self->{get_status};
5652
my $sleep = $self->{sleep};
5653
5654
my %vals_too_high = %{$self->{max_val_for}};
5655
my $pr_callback;
5656
if ( $pr ) {
5657
$pr_callback = sub {
5658
print STDERR "Pausing because "
5659
. join(', ',
5660
map {
5661
"$_="
5662
. (defined $vals_too_high{$_} ? $vals_too_high{$_}
5663
: 'unknown')
5664
} sort keys %vals_too_high
5665
)
5666
. ".\n";
5667
return;
5668
};
5669
$pr->set_callback($pr_callback);
5670
}
5671
5672
while ( $oktorun->() ) {
5673
PTDEBUG && _d('Checking status variables');
5674
foreach my $var ( sort keys %vals_too_high ) {
5675
my $val = $get_status->($var);
5676
PTDEBUG && _d($var, '=', $val);
5677
if ( !$val || $val >= $self->{max_val_for}->{$var} ) {
5678
$vals_too_high{$var} = $val;
5679
}
5680
else {
5681
delete $vals_too_high{$var};
5682
}
5683
}
5684
5685
last unless scalar keys %vals_too_high;
5686
5687
PTDEBUG && _d(scalar keys %vals_too_high, 'values are too high:',
5688
%vals_too_high);
5689
if ( $pr ) {
5690
$pr->update(sub { return 0; });
5691
}
5692
PTDEBUG && _d('Calling sleep callback');
5693
$sleep->();
5694
%vals_too_high = %{$self->{max_val_for}}; # recheck all vars
5695
}
5696
5697
PTDEBUG && _d('All var vals are low enough');
5698
return;
5699
}
5700
5701
sub _d {
5702
my ($package, undef, $line) = caller 0;
5703
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
5704
map { defined $_ ? $_ : 'undef' }
5705
@_;
5706
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
5707
}
5708
5709
1;
5710
}
5711
# ###########################################################################
5712
# End MySQLStatusWaiter package
5713
# ###########################################################################
5714
5715
# ###########################################################################
5716
# WeightedAvgRate package
5717
# This package is a copy without comments from the original. The original
5718
# with comments and its test file can be found in the Bazaar repository at,
5719
# lib/WeightedAvgRate.pm
5720
# t/lib/WeightedAvgRate.t
5721
# See https://launchpad.net/percona-toolkit for more information.
5722
# ###########################################################################
5723
{
5724
package WeightedAvgRate;
5725
5726
use strict;
5727
use warnings FATAL => 'all';
5728
use English qw(-no_match_vars);
5729
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
5730
5731
sub new {
5732
my ( $class, %args ) = @_;
5733
my @required_args = qw(target_t);
5734
foreach my $arg ( @required_args ) {
5735
die "I need a $arg argument" unless defined $args{$arg};
5736
}
5737
5738
my $self = {
5739
%args,
5740
avg_n => 0,
5741
avg_t => 0,
5742
weight => $args{weight} || 0.75,
5743
};
5744
5745
return bless $self, $class;
5746
}
5747
5748
sub update {
5749
my ($self, $n, $t) = @_;
5750
PTDEBUG && _d('Master op time:', $n, 'n /', $t, 's');
5751
5752
if ( $self->{avg_n} && $self->{avg_t} ) {
5753
$self->{avg_n} = ($self->{avg_n} * $self->{weight}) + $n;
5754
$self->{avg_t} = ($self->{avg_t} * $self->{weight}) + $t;
5755
$self->{avg_rate} = $self->{avg_n} / $self->{avg_t};
5756
PTDEBUG && _d('Weighted avg rate:', $self->{avg_rate}, 'n/s');
5757
}
5758
else {
5759
$self->{avg_n} = $n;
5760
$self->{avg_t} = $t;
5761
$self->{avg_rate} = $self->{avg_n} / $self->{avg_t};
5762
PTDEBUG && _d('Initial avg rate:', $self->{avg_rate}, 'n/s');
5763
}
5764
5765
my $new_n = int($self->{avg_rate} * $self->{target_t});
5766
PTDEBUG && _d('Adjust n to', $new_n);
5767
return $new_n;
5768
}
5769
5770
sub _d {
5771
my ($package, undef, $line) = caller 0;
5772
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
5773
map { defined $_ ? $_ : 'undef' }
5774
@_;
5775
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
5776
}
5777
5778
1;
5779
}
5780
# ###########################################################################
5781
# End WeightedAvgRate package
5782
# ###########################################################################
5783
5784
# ###########################################################################
4972
5785
# This is a combination of modules and programs in one -- a runnable module.
4973
5786
# http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last
4974
5787
# Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition.
4978
5791
# ###########################################################################
4979
5792
package pt_table_checksum;
4980
5793
5794
use strict;
5795
use warnings FATAL => 'all';
4981
5796
use English qw(-no_match_vars);
4982
use List::Util qw(max maxstr);
4983
use Time::HiRes qw(gettimeofday sleep);
5797
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
5798
5799
use POSIX qw(signal_h);
5800
use List::Util qw(max);
5801
use Time::HiRes qw(sleep time);
4984
5802
use Data::Dumper;
4985
$Data::Dumper::Indent = 0;
5803
$Data::Dumper::Indent = 1;
5804
$Data::Dumper::Sortkeys = 1;
4986
5805
$Data::Dumper::Quotekeys = 0;
4987
5806
4988
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
4989
4990
$OUTPUT_AUTOFLUSH = 1;
4991
4992
# Global variables.
4993
my $checksum_table_data;
4994
my ( $fetch_sth, $update_sth, $savesince_sth );
4995
my ( $crc_wid, $md5sum_fmt );
4996
my $already_checksummed;
4997
# %tables_to_checksum has the following structure:
4998
# database => [
4999
# { table },
5000
# ...
5001
# ],
5002
# ...
5003
my %tables_to_checksum;
5807
use sigtrap 'handler', \&sig_int, 'normal-signals';
5808
5809
my $oktorun = 1;
5810
my $print_header = 1;
5004
5811
5005
5812
sub main {
5006
@ARGV = @_; # set global ARGV for this package
5007
5008
# Reset global vars else tests which run this tool as a module
5009
# will have strange, overlapping results.
5010
$checksum_table_data = undef;
5011
( $fetch_sth, $update_sth, $savesince_sth ) = (undef, undef, undef);
5012
( $crc_wid, $md5sum_fmt ) = (undef, undef);
5013
$already_checksummed = undef;
5014
%tables_to_checksum = ();
5015
5016
my $q = new Quoter();
5813
# Reset global vars else tests will fail in strange ways.
5814
@ARGV = @_;
5815
$oktorun = 1;
5816
$print_header = 1;
5817
5017
5818
my $exit_status = 0;
5018
5819
5019
5820
# ########################################################################
5020
5821
# Get configuration information.
5021
5822
# ########################################################################
5022
# Because of --arg-table, $final_o is the OptionParser obj used to get
5023
# most options (see my $final_o below).
5024
5823
my $o = new OptionParser();
5025
5824
$o->get_specs();
5026
5825
$o->get_opts();
5028
5827
my $dp = $o->DSNParser();
5029
5828
$dp->prop('set-vars', $o->get('set-vars'));
5030
5829
5031
# This list contains all the command-line arguments that can be overridden
5032
# by a table that contains arguments for each table to be checksummed.
5033
# The long form of each argument is given. The values are read from the
5034
# POD by finding the magical token.
5035
my %overridable_args;
5036
{
5037
my $para = $o->read_para_after(
5038
__FILE__, qr/MAGIC_overridable_args/);
5039
foreach my $arg ( $para =~ m/([\w-]+)/g ) {
5040
die "Magical argument $arg mentioned in POD is not a "
5041
. "command-line argument" unless $o->has($arg);
5042
$overridable_args{$arg} = 1;
5043
}
5044
};
5045
5046
# Post-process command-line options and arguments.
5047
if ( $o->get('replicate') ) {
5048
# --replicate says that it disables these options. We don't
5049
# check got() because these opts aren't used in do_tbl_replicate()
5050
# or its caller so they're completely useless with --replicate.
5051
$o->set('lock', undef);
5052
$o->set('wait', undef);
5053
$o->set('slave-lag', undef);
5054
}
5055
else {
5056
$o->set('lock', 1) if $o->get('wait');
5057
$o->set('slave-lag', 1) if $o->get('lock');
5058
}
5059
5060
if ( !@ARGV ) {
5061
$o->save_error("No hosts specified.");
5062
}
5063
5064
my @hosts;
5065
my $dsn_defaults = $dp->parse_options($o);
5066
{
5067
foreach my $arg ( unique(@ARGV) ) {
5068
push @hosts, $dp->parse($arg, $hosts[0], $dsn_defaults);
5069
}
5070
}
5071
5072
if ( $o->get('explain-hosts') ) {
5073
foreach my $host ( @hosts ) {
5074
print "Server $host->{h}:\n ", $dp->as_string($host), "\n";
5075
}
5076
return 0;
5077
}
5078
5079
# Checksumming table data is the normal operation. But if we're only to
5080
# compare schemas, then we can skip a lot of work, like selecting an algo,
5081
# replication stuff, etc.
5082
$checksum_table_data = $o->get('schema') ? 0 : 1;
5083
5084
if ( $o->get('checksum') ) {
5085
$o->set('count', 0);
5086
}
5087
5088
if ( $o->get('explain') ) {
5089
@hosts = $hosts[0];
5090
}
5091
5092
# --replicate auto-enables --throttle-method slavelag unless user
5093
# set --throttle-method explicitly.
5094
$o->set('throttle-method', 'slavelag')
5095
if $o->get('replicate') && !$o->got('throttle-method');
5096
5097
# These options are only needed if a --chunk-size is specified.
5098
if ( !$o->get('chunk-size') ) {
5099
$o->set('chunk-size-limit', undef);
5100
$o->set('unchunkable-tables', 1);
5101
}
5830
# Add the --replicate table to --ignore-tables.
5831
my %ignore_tables = (
5832
%{$o->get('ignore-tables')},
5833
$o->get('replicate') => 1,
5834
);
5835
$o->set('ignore-tables', \%ignore_tables);
5836
5837
$o->set('chunk-time', 0) if $o->got('chunk-size');
5102
5838
5103
5839
if ( !$o->get('help') ) {
5104
if ( $o->get('replicate-check') && !$o->get('replicate') ) {
5105
$o->save_error("--replicate-check requires --replicate.");
5106
}
5107
if ( $o->get('save-since') && !$o->get('arg-table') ) {
5108
$o->save_error("--save-since requires --arg-table.");
5109
}
5110
elsif ( $o->get('replicate') && @hosts > 1 ) {
5111
$o->save_error("You can only specify one host with --replicate.");
5112
}
5113
5114
if ( $o->get('resume-replicate') && !$o->get('replicate') ) {
5115
$o->save_error("--resume-replicate requires --replicate.");
5116
}
5117
if ( $o->get('resume') && $o->get('replicate') ) {
5118
$o->save_error('--resume does not work with --replicate. '
5119
. 'Use --resume-replicate instead.');
5120
}
5121
5122
if ( my $throttle_method = $o->get('throttle-method') ) {
5123
$throttle_method = lc $throttle_method;
5124
if ( !grep { $throttle_method eq $_ } qw(none slavelag) ) {
5125
$o->save_error("Invalid --throttle-method: $throttle_method");
5126
}
5127
}
5128
5129
if ( $o->get('check-slave-lag') && $o->get('throttle-method') eq 'none') {
5130
# User specified --check-slave-lag DSN and --throttle-method none.
5131
# They probably meant just --check-slave-lag DSN.
5132
$o->save_error('-throttle-method=none contradicts --check-slave-lag '
5133
. 'because --check-slave-lag implies --throttle-method=slavelag');
5134
}
5135
if ( $o->get('throttle-method') ne 'none' && !$o->get('replicate') ) {
5136
# User did --throttle-method (explicitly) without --replicate.
5137
$o->save_error('--throttle-method ', $o->get('throttle-method'),
5138
' requires --replicate');
5139
}
5140
5141
# Make sure --replicate has a db.
5142
if ( my $replicate_table = $o->get('replicate') ) {
5143
my ($db, $tbl) = $q->split_unquote($replicate_table);
5144
if ( !$db ) {
5145
$o->save_error('The --replicate table must be database-qualified');
5146
}
5147
}
5148
5149
if ( $o->get('chunk-size-limit') ) {
5150
my $factor = $o->get('chunk-size-limit');
5151
if ( $factor < 0 # can't be negative
5152
|| ($factor > 0 && $factor < 1) ) # can't be less than 1
5153
{
5840
if ( @ARGV > 1 ) {
5841
$o->save_error("More than one host specified; only one allowed");
5842
}
5843
5844
if ( ($o->get('replicate') || '') !~ m/[\w`]\.[\w`]/ ) {
5845
$o->save_error('The --replicate table must be database-qualified');
5846
}
5847
5848
if ( my $limit = $o->get('chunk-size-limit') ) {
5849
if ( $limit < 0 || ($limit > 0 && $limit < 1) ) {
5154
5850
$o->save_error('--chunk-size-limit must be >= 1 or 0 to disable');
5155
5851
}
5156
5852
}
5162
5858
$o->save_error("--progress $EVAL_ERROR");
5163
5859
}
5164
5860
}
5165
5166
if ( my $chunk_range = $o->get('chunk-range') ) {
5167
$chunk_range = lc $chunk_range;
5168
my $para = $o->read_para_after(__FILE__, qr/MAGIC_chunk_range/);
5169
my @vals = $para =~ m/\s+([a-z]+)\s+[A-Z]+/g;
5170
if ( !grep { $chunk_range eq $_} @vals ) {
5171
$o->save_error("Invalid value for --chunk-range. "
5172
. "Valid values are: " . join(", ", @vals));
5173
}
5174
}
5175
5861
}
5176
5862
5177
5863
$o->usage_or_errors();
5178
5864
5179
5865
# ########################################################################
5180
# If --pid, check it first since we'll die if it already exits.
5866
# If --pid, check it first since we'll die if it already exists.
5181
5867
# ########################################################################
5182
5868
my $daemon;
5183
5869
if ( $o->get('pid') ) {
5189
5875
}
5190
5876
5191
5877
# ########################################################################
5192
# Ready to work now.
5878
# Connect to the master.
5193
5879
# ########################################################################
5194
5880
my $vp = new VersionParser();
5195
my $tp = new TableParser(Quoter => $q);
5196
my $tc = new TableChecksum(Quoter=> $q, VersionParser => $vp);
5197
my $ms = new MasterSlave(VersionParser => $vp);
5198
my $du = new MySQLDump();
5199
my $ch = new TableChunker(Quoter => $q, MySQLDump => $du);
5200
my %common_modules = (
5201
ch => $ch,
5202
dp => $dp,
5203
du => $du,
5204
o => $o,
5205
ms => $ms,
5206
q => $q,
5207
tc => $tc,
5208
tp => $tp,
5209
vp => $vp,
5210
);
5211
5212
my $main_dbh = get_cxn($hosts[0], %common_modules);
5213
5214
# #########################################################################
5215
# Prepare --throttle-method.
5216
# #########################################################################
5217
my $throttle_method = $o->get('throttle-method');
5218
my @slaves;
5219
if ( lc($throttle_method) eq 'slavelag' ) {
5220
if ( $o->get('check-slave-lag') ) {
5221
MKDEBUG && _d('Using --check-slave-lag DSN for throttle');
5222
# OptionParser can't auto-copy DSN vals from a cmd line DSN
5223
# to an opt DSN, so we copy them manually.
5224
my $dsn = $dp->copy($hosts[0], $o->get('check-slave-lag'));
5225
push @slaves, { dsn=>$dsn, dbh=>get_cxn($dsn, %common_modules) };
5226
}
5227
else {
5228
MKDEBUG && _d('Recursing to slaves for throttle');
5229
$ms->recurse_to_slaves(
5230
{ dbh => $main_dbh,
5231
dsn => $hosts[0],
5232
dsn_parser => $dp,
5233
recurse => $o->get('recurse'),
5234
method => $o->get('recursion-method'),
5235
callback => sub {
5236
my ( $dsn, $dbh, $level, $parent ) = @_;
5237
return unless $level;
5238
MKDEBUG && _d('throttle slave:', $dp->as_string($dsn));
5239
$dbh->{InactiveDestroy} = 1; # Prevent destroying on fork.
5240
$dbh->{FetchHashKeyName} = 'NAME_lc';
5241
push @slaves, { dsn=>$dsn, dbh=>$dbh };
5242
return;
5243
},
5244
}
5245
);
5246
}
5247
}
5248
5249
# ########################################################################
5250
# Load --arg-table information.
5251
# ########################################################################
5252
my %args_for;
5253
if ( my $arg_tbl = $o->get('arg-table') ) {
5254
my %col_in_argtable;
5255
my $rows = $main_dbh->selectall_arrayref(
5256
"SELECT * FROM $arg_tbl", { Slice => {} });
5257
foreach my $row ( @$rows ) {
5258
die "Invalid entry in --arg-table: db and tbl must be set"
5259
unless $row->{db} && $row->{tbl};
5260
$args_for{$row->{db}}->{$row->{tbl}} = {
5261
map { $_ => $row->{$_} }
5262
grep { $overridable_args{$_} && defined $row->{$_} }
5263
keys %$row
5264
};
5265
if ( !%col_in_argtable ) { # do only once
5266
foreach my $key ( keys %$row ) {
5267
next if $key =~ m/^(db|tbl|ts)$/;
5268
die "Column $key (from $arg_tbl given by --arg-table) is not "
5269
. "an overridable argument" unless $overridable_args{$key};
5270
$col_in_argtable{$key} = 1;
5271
}
5272
}
5273
}
5274
if ( $col_in_argtable{since} ) {
5275
$savesince_sth = $main_dbh->prepare(
5276
"UPDATE $arg_tbl SET since=COALESCE(?, NOW()) WHERE db=? AND tbl=?");
5277
}
5278
}
5279
5280
# ########################################################################
5281
# Check for replication filters.
5282
# ########################################################################
5283
if ( $o->get('replicate') && $o->get('check-replication-filters') ) {
5284
MKDEBUG && _d("Recursing to slaves to check for replication filters");
5285
my @all_repl_filters;
5286
$ms->recurse_to_slaves(
5287
{ dbh => $main_dbh,
5288
dsn => $hosts[0],
5289
dsn_parser => $dp,
5290
recurse => undef, # check for filters anywhere
5291
method => $o->get('recursion-method'),
5292
callback => sub {
5293
my ( $dsn, $dbh, $level, $parent ) = @_;
5294
my $repl_filters = $ms->get_replication_filters(dbh=>$dbh);
5295
if ( keys %$repl_filters ) {
5296
my $host = $dp->as_string($dsn);
5297
push @all_repl_filters,
5298
{ name => $host,
5299
filters => $repl_filters,
5300
};
5301
}
5302
return;
5303
},
5304
}
5305
);
5306
if ( @all_repl_filters ) {
5307
my $msg = "Cannot checksum with --replicate because replication "
5308
. "filters are set on these hosts:\n";
5309
foreach my $host ( @all_repl_filters ) {
5310
my $filters = $host->{filters};
5311
$msg .= " $host->{name}\n"
5312
. join("\n", map { " $_ = $host->{filters}->{$_}" }
5313
keys %{$host->{filters}})
5314
. "\n";
5315
}
5316
$msg .= "Please read the --check-replication-filters documentation "
5317
. "to learn how to solve this problem.";
5318
warn $msg;
5319
return 1;
5320
}
5321
}
5322
5323
# ########################################################################
5324
# Check replication slaves if desired. If only --replicate-check is given,
5325
# then we will exit here. If --recheck is also given, then we'll continue
5326
# through the entire script but checksum only the inconsistent tables found
5327
# here.
5328
# ########################################################################
5329
if ( defined $o->get('replicate-check') ) {
5330
MKDEBUG && _d("Recursing to slaves for replicate check, depth",
5331
$o->get('replicate-check'));
5332
my $callback = $o->get('recheck')
5333
? \&save_inconsistent_tbls
5334
: \&print_inconsistent_tbls;
5335
$ms->recurse_to_slaves(
5336
{ dbh => $main_dbh,
5337
dsn => $hosts[0],
5338
dsn_parser => $dp,
5339
recurse => $o->get('replicate-check'),
5340
method => $o->get('recursion-method'),
5341
callback => sub {
5342
my ( $dsn, $dbh, $level, $parent ) = @_;
5343
my @tbls = $tc->find_replication_differences(
5344
$dbh, $o->get('replicate'));
5345
return unless @tbls;
5346
$exit_status = 1;
5347
# Call the callback that does something useful with
5348
# the inconsistent tables.
5349
# o dbh db tbl args_for
5350
$callback->(
5351
dsn => $dsn,
5352
dbh => $dbh,
5353
level => $level,
5354
parent => $parent,
5355
tbls => \@tbls,
5356
args_for => \%args_for,
5357
%common_modules
5358
);
5359
},
5360
}
5361
);
5362
return $exit_status unless $o->get('recheck');
5363
}
5364
5365
# ########################################################################
5366
# Otherwise get ready to checksum table data, unless we have only to check
5367
# schemas in which case we can skip all such work, knowing already that we
5368
# will use CRC32.
5369
# ########################################################################
5370
if ( $checksum_table_data ) {
5371
# Verify that CONCAT_WS is compatible across all servers. On older
5372
# versions of MySQL it skips both empty strings and NULL; on newer
5373
# just NULL.
5374
if ( $o->get('verify') && @hosts > 1 ) {
5375
verify_checksum_compat(hosts=>\@hosts, %common_modules);
5376
}
5377
5378
($fetch_sth, $update_sth)
5379
= check_repl_table(dbh=>$main_dbh, %common_modules);
5380
}
5381
else {
5382
$crc_wid = 16; # Wider than the widest CRC32.
5383
}
5384
5385
# ########################################################################
5386
# If resuming a previous run, figure out what the previous run finished.
5387
# ########################################################################
5388
if ( $o->get('replicate') && $o->get('resume-replicate') ) {
5389
$already_checksummed = read_repl_table(
5390
dbh => $main_dbh,
5391
host => $hosts[0]->{h},
5392
%common_modules,
5393
);
5394
}
5395
elsif ( $o->get('resume') ) {
5396
$already_checksummed = parse_resume_file($o->get('resume'));
5397
}
5398
5399
# ########################################################################
5400
# Set transaction isolation level.
5401
# http://code.google.com/p/maatkit/issues/detail?id=720
5402
# ########################################################################
5403
if ( $o->get('replicate') ) {
5404
my $sql = "SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ";
5881
5882
my $set_on_connect = sub {
5883
my ($dbh) = @_;
5884
return if $o->get('explain');
5885
my $sql;
5886
5887
# https://bugs.launchpad.net/percona-toolkit/+bug/919352
5888
# The tool shouldn't blindly attempt to change binlog_format;
5889
# instead, it should check if it's already set to STATEMENT.
5890
# This is becase starting with MySQL 5.1.29, changing the format
5891
# requires a SUPER user.
5892
if ( $vp->version_ge($dbh, '5.1.5') ) {
5893
$sql = 'SELECT @@binlog_format';
5894
PTDEBUG && _d($dbh, $sql);
5895
my ($original_binlog_format) = $dbh->selectrow_array($sql);
5896
PTDEBUG && _d('Original binlog_format:', $original_binlog_format);
5897
if ( $original_binlog_format !~ /STATEMENT/i ) {
5898
$sql = q{/*!50108 SET @@binlog_format := 'STATEMENT'*/};
5899
eval {
5900
PTDEBUG && _d($dbh, $sql);
5901
$dbh->do($sql);
5902
};
5903
if ( $EVAL_ERROR ) {
5904
die "Failed to $sql: $EVAL_ERROR\n"
5905
. "This tool requires binlog_format=STATEMENT, "
5906
. "but the current binlog_format is set to "
5907
."$original_binlog_format and an error occurred while "
5908
. "attempting to change it. If running MySQL 5.1.29 or newer, "
5909
. "setting binlog_format requires the SUPER privilege. "
5910
. "You will need to manually set binlog_format to 'STATEMENT' "
5911
. "before running this tool.\n";
5912
}
5913
}
5914
}
5915
5916
# Set transaction isolation level. We set binlog_format to STATEMENT,
5917
# but if the transaction isolation level is set to READ COMMITTED and the
5918
# --replicate table is in InnoDB format, the tool fails with the following
5919
# message:
5920
#
5921
# Binary logging not possible. Message: Transaction level 'READ-COMMITTED'
5922
# in InnoDB is not safe for binlog mode 'STATEMENT'
5923
#
5924
# See also http://code.google.com/p/maatkit/issues/detail?id=720
5925
$sql = 'SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ';
5405
5926
eval {
5406
MKDEBUG && _d($main_dbh, $sql);
5407
$main_dbh->do($sql);
5927
PTDEBUG && _d($dbh, $sql);
5928
$dbh->do($sql);
5408
5929
};
5409
5930
if ( $EVAL_ERROR ) {
5410
5931
die "Failed to $sql: $EVAL_ERROR\n"
5411
5932
. "If the --replicate table is InnoDB and the default server "
5412
5933
. "transaction isolation level is not REPEATABLE-READ then "
5413
. "checksumming may fail with errors like \"Binary logging not "
5934
. "checksumming may fail with errors such as \"Binary logging not "
5414
5935
. "possible. Message: Transaction level 'READ-COMMITTED' in "
5415
5936
. "InnoDB is not safe for binlog mode 'STATEMENT'\". In that "
5416
5937
. "case you will need to manually set the transaction isolation "
5417
. "level to REPEATABLE-READ.";
5418
}
5419
}
5420
5421
# ########################################################################
5422
# Iterate through databases and tables and do the checksums.
5423
# ########################################################################
5424
5425
# Get table info for all hosts, all slaves, unless we're in the special
5426
# "repl-re-check" mode in which case %tables_to_checksum has already the
5427
# inconsistent tables that we need to re-checksum.
5428
get_all_tbls_info(
5429
dbh => $main_dbh,
5430
args_for => \%args_for,
5431
%common_modules,
5432
) unless ($o->get('replicate-check') && $o->get('recheck'));
5433
5434
# Finally, checksum the tables.
5435
foreach my $database ( keys %tables_to_checksum ) {
5436
my $tables = $tables_to_checksum{$database};
5437
$exit_status |= checksum_tables(
5438
dbh => $main_dbh,
5439
db => $database,
5440
tbls => $tables,
5441
hosts => \@hosts,
5442
slaves => \@slaves,
5443
%common_modules
5444
);
5445
}
5446
5938
. "level to REPEATABLE-READ.\n";
5939
}
5940
5941
# We set innodb_lock_wait_timeout=1 so that if this tool happens to cause
5942
# some locking, it will be more likely to be the victim than other
5943
# connections to the server, and thus disrupt the server less.
5944
$sql = 'SHOW SESSION VARIABLES LIKE "innodb_lock_wait_timeout"';
5945
PTDEBUG && _d($dbh, $sql);
5946
my (undef, $lock_wait_timeout) = $dbh->selectrow_array($sql);
5947
PTDEBUG && _d('innodb_lock_wait_timeout', $lock_wait_timeout);
5948
if ( ($lock_wait_timeout || 0) > $o->get('lock-wait-timeout') ) {
5949
$sql = 'SET SESSION innodb_lock_wait_timeout=1';
5950
eval {
5951
PTDEBUG && _d($dbh, $sql);
5952
$dbh->do($sql);
5953
};
5954
if ( $EVAL_ERROR ) {
5955
warn "Failed to $sql: $EVAL_ERROR\n"
5956
. "The current innodb_lock_wait_timeout value "
5957
. "$lock_wait_timeout is higher than the --lock-wait-timeout "
5958
. "value " . $o->get('lock-wait-timeout') . " and the variable "
5959
. "cannot be changed. innodb_lock_wait_timeout is only dynamic "
5960
. "when using the InnoDB plugin. To prevent this warning, either "
5961
. "specify --lock-wait-time=$lock_wait_timeout, or manually "
5962
. "set innodb_lock_wait_timeout to a value less than or equal "
5963
. "to " . $o->get('lock-wait-timeout') . " and restart MySQL.\n";
5964
}
5965
}
5966
};
5967
5968
# Do not call "new Cxn(" directly; use this sub so that set_on_connect
5969
# is applied to every cxn.
5970
# TODO: maybe this stuff only needs to be set on master cxn?
5971
my $make_cxn = sub {
5972
my (%args) = @_;
5973
my $cxn = new Cxn(
5974
%args,
5975
DSNParser => $dp,
5976
OptionParser => $o,
5977
set => $args{set_vars} ? $set_on_connect : undef,
5978
);
5979
eval { $cxn->connect() }; # connect or die trying
5980
if ( $EVAL_ERROR ) {
5981
die ts($EVAL_ERROR);
5982
}
5983
return $cxn;
5984
};
5985
5986
# The dbh and dsn can be used before checksumming starts, but once
5987
# inside the main TABLE loop, only use the master cxn because its
5988
# dbh may be recreated.
5989
my $master_cxn = $make_cxn->(set_vars => 1, dsn_string => shift @ARGV);
5990
my $master_dbh = $master_cxn->dbh(); # just for brevity
5991
my $master_dsn = $master_cxn->dsn(); # just for brevity
5992
5993
# ########################################################################
5994
# If this is not a dry run (--explain was not specified), then we're
5995
# going to checksum the tables, so do the necessary preparations and
5996
# checks. Else, this all can be skipped because all we need for a
5997
# dry run is a connection to the master.
5998
# ########################################################################
5999
my $q = new Quoter();
6000
my $tp = new TableParser(Quoter => $q);
6001
my $rc = new RowChecksum(Quoter=> $q, OptionParser => $o);
6002
my $ms = new MasterSlave(VersionParser => $vp);
6003
6004
my $slaves; # all slaves (that we can find)
6005
my $slave_lag_cxns; # slaves whose lag we'll check
6006
6007
my $replica_lag; # ReplicaLagWaiter object
6008
my $replica_lag_pr; # Progress for ReplicaLagWaiter
6009
my $sys_load; # MySQLStatusWaiter object
6010
my $sys_load_pr; # Progress for MySQLStatusWaiter object
6011
6012
my $repl_table = $q->quote($q->split_unquote($o->get('replicate')));
6013
my $fetch_sth; # fetch chunk from repl table
6014
my $update_sth; # update master_cnt and master_cnt in repl table
6015
my $delete_sth; # delete checksums for one db.tbl from repl table
6016
6017
if ( !$o->get('explain') ) {
6018
# #####################################################################
6019
# Find and connect to slaves.
6020
# #####################################################################
6021
$slaves = $ms->get_slaves(
6022
dbh => $master_dbh,
6023
dsn => $master_dsn,
6024
OptionParser => $o,
6025
DSNParser => $dp,
6026
Quoter => $q,
6027
make_cxn => sub {
6028
return $make_cxn->(@_, prev_dsn => $master_cxn->dsn());
6029
},
6030
);
6031
PTDEBUG && _d(scalar @$slaves, 'slaves found');
6032
6033
if ( $o->get('check-slave-lag') ) {
6034
PTDEBUG && _d('Will use --check-slave-lag to check for slave lag');
6035
my $cxn = $make_cxn->(
6036
dsn_string => $o->get('check-slave-lag'),
6037
prev_dsn => $master_cxn->dsn(),
6038
);
6039
$slave_lag_cxns = [ $cxn ];
6040
}
6041
else {
6042
PTDEBUG && _d('Will check slave lag on all slaves');
6043
$slave_lag_cxns = $slaves;
6044
}
6045
6046
# #####################################################################
6047
# Possibly check replication slaves and exit.
6048
# #####################################################################
6049
if ( $o->get('replicate-check') && $o->get('replicate-check-only') ) {
6050
PTDEBUG && _d('Will --replicate-check and exit');
6051
6052
foreach my $slave ( @$slaves ) {
6053
my $diffs = $rc->find_replication_differences(
6054
dbh => $slave->dbh(),
6055
repl_table => $repl_table,
6056
);
6057
PTDEBUG && _d(scalar @$diffs, 'checksum diffs on',
6058
$slave->name());
6059
if ( @$diffs ) {
6060
$exit_status |= 1;
6061
if ( $o->get('quiet') < 2 ) {
6062
print_checksum_diffs(
6063
cxn => $slave,
6064
diffs => $diffs,
6065
);
6066
}
6067
}
6068
}
6069
6070
PTDEBUG && _d('Exit status', $exit_status, 'oktorun', $oktorun);
6071
return $exit_status;
6072
}
6073
6074
# #####################################################################
6075
# Check for replication filters.
6076
# #####################################################################
6077
if ( $o->get('check-replication-filters') ) {
6078
PTDEBUG && _d("Checking slave replication filters");
6079
my @all_repl_filters;
6080
foreach my $slave ( @$slaves ) {
6081
my $repl_filters = $ms->get_replication_filters(
6082
dbh => $slave->dbh(),
6083
);
6084
if ( keys %$repl_filters ) {
6085
push @all_repl_filters,
6086
{ name => $slave->name(),
6087
filters => $repl_filters,
6088
};
6089
}
6090
}
6091
if ( @all_repl_filters ) {
6092
my $msg = "Replication filters are set on these hosts:\n";
6093
foreach my $host ( @all_repl_filters ) {
6094
my $filters = $host->{filters};
6095
$msg .= " $host->{name}\n"
6096
. join("\n", map { " $_ = $host->{filters}->{$_}" }
6097
keys %{$host->{filters}})
6098
. "\n";
6099
}
6100
$msg .= "Please read the --check-replication-filters documentation "
6101
. "to learn how to solve this problem.";
6102
die ts($msg);
6103
}
6104
}
6105
6106
# #####################################################################
6107
# Check that the replication table exists, or possibly create it.
6108
# #####################################################################
6109
eval {
6110
check_repl_table(
6111
dbh => $master_dbh,
6112
repl_table => $repl_table,
6113
OptionParser => $o,
6114
TableParser => $tp,
6115
Quoter => $q,
6116
);
6117
};
6118
if ( $EVAL_ERROR ) {
6119
die ts($EVAL_ERROR);
6120
}
6121
6122
# #####################################################################
6123
# Make a ReplicaLagWaiter to help wait for slaves after each chunk.
6124
# #####################################################################
6125
my $sleep = sub {
6126
# Don't let the master dbh die while waiting for slaves because we
6127
# may wait a very long time for slaves.
6128
6129
# This is called from within the main TABLE loop, so use the
6130
# master cxn; do not use $master_dbh.
6131
my $dbh = $master_cxn->dbh();
6132
if ( !$dbh || !$dbh->ping() ) {
6133
PTDEBUG && _d('Lost connection to master while waiting for slave lag');
6134
eval { $dbh = $master_cxn->connect() }; # connect or die trying
6135
if ( $EVAL_ERROR ) {
6136
$oktorun = 0; # Fatal error
6137
chomp $EVAL_ERROR;
6138
die "Lost connection to master while waiting for replica lag "
6139
. "($EVAL_ERROR)";
6140
}
6141
}
6142
$dbh->do("SELECT 'pt-table-checksum keepalive'");
6143
sleep $o->get('check-interval');
6144
return;
6145
};
6146
6147
my $get_lag = sub {
6148
my ($cxn) = @_;
6149
my $dbh = $cxn->dbh();
6150
if ( !$dbh || !$dbh->ping() ) {
6151
PTDEBUG && _d('Lost connection to slave', $cxn->name(),
6152
'while waiting for slave lag');
6153
eval { $dbh = $cxn->connect() }; # connect or die trying
6154
if ( $EVAL_ERROR ) {
6155
$oktorun = 0; # Fatal error
6156
chomp $EVAL_ERROR;
6157
die "Lost connection to replica " . $cxn->name()
6158
. " while attempting to get its lag ($EVAL_ERROR)";
6159
}
6160
}
6161
return $ms->get_slave_lag($dbh);
6162
};
6163
6164
$replica_lag = new ReplicaLagWaiter(
6165
slaves => $slave_lag_cxns,
6166
max_lag => $o->get('max-lag'),
6167
oktorun => sub { return $oktorun },
6168
get_lag => $get_lag,
6169
sleep => $sleep,
6170
);
6171
6172
my $get_status;
6173
{
6174
my $sql = "SHOW GLOBAL STATUS LIKE ?";
6175
my $sth = $master_cxn->dbh()->prepare($sql);
6176
6177
$get_status = sub {
6178
my ($var) = @_;
6179
PTDEBUG && _d($sth->{Statement}, $var);
6180
$sth->execute($var);
6181
my (undef, $val) = $sth->fetchrow_array();
6182
return $val;
6183
};
6184
}
6185
6186
$sys_load = new MySQLStatusWaiter(
6187
spec => $o->get('max-load'),
6188
get_status => $get_status,
6189
oktorun => sub { return $oktorun },
6190
sleep => $sleep,
6191
);
6192
6193
if ( $o->get('progress') ) {
6194
$replica_lag_pr = new Progress(
6195
jobsize => scalar @$slaves,
6196
spec => $o->get('progress'),
6197
name => "Waiting for replicas to catch up", # not used
6198
);
6199
6200
$sys_load_pr = new Progress(
6201
jobsize => scalar @{$o->get('max-load')},
6202
spec => $o->get('progress'),
6203
name => "Waiting for --max-load", # not used
6204
);
6205
}
6206
6207
# #####################################################################
6208
# Prepare statement handles to update the repl table on the master.
6209
# #####################################################################
6210
$fetch_sth = $master_dbh->prepare(
6211
"SELECT this_crc, this_cnt FROM $repl_table "
6212
. "WHERE db = ? AND tbl = ? AND chunk = ?");
6213
$update_sth = $master_dbh->prepare(
6214
"UPDATE $repl_table SET chunk_time = ?, master_crc = ?, master_cnt = ? "
6215
. "WHERE db = ? AND tbl = ? AND chunk = ?");
6216
$delete_sth = $master_dbh->prepare(
6217
"DELETE FROM $repl_table WHERE db = ? AND tbl = ?");
6218
} # !$o->get('explain')
6219
6220
# ########################################################################
6221
# Checksum args and the DMS part of the checksum query for each table.
6222
# ########################################################################
6223
my %crc_args = $rc->get_crc_args(dbh => $master_dbh);
6224
my $checksum_dml = "REPLACE INTO $repl_table "
6225
. "(db, tbl, chunk, chunk_index,"
6226
. " lower_boundary, upper_boundary, this_cnt, this_crc) "
6227
. "SELECT ?, ?, ?, ?, ?, ?,";
6228
my $past_cols = " COUNT(*), '0'";
6229
6230
# ########################################################################
6231
# Get last chunk for --resume.
6232
# ########################################################################
6233
my $last_chunk;
6234
if ( $o->get('resume') ) {
6235
$last_chunk = last_chunk(
6236
dbh => $master_dbh,
6237
repl_table => $repl_table,
6238
);
6239
}
6240
6241
my $schema_iter = new SchemaIterator(
6242
dbh => $master_dbh,
6243
resume => $last_chunk ? $q->quote(@{$last_chunk}{qw(db tbl)})
6244
: "",
6245
keep_tbl_status => 1,
6246
OptionParser => $o,
6247
TableParser => $tp,
6248
Quoter => $q,
6249
);
6250
6251
if ( $last_chunk &&
6252
!$schema_iter->table_is_allowed(@{$last_chunk}{qw(db tbl)}) ) {
6253
PTDEBUG && _d('Ignoring last table', @{$last_chunk}{qw(db tbl)},
6254
'and resuming from next table');
6255
$last_chunk = undef;
6256
}
6257
6258
# ########################################################################
6259
# Various variables and modules for checksumming the tables.
6260
# ########################################################################
6261
my $total_rows = 0;
6262
my $total_time = 0;
6263
my $total_rate = 0;
6264
my $limit = $o->get('chunk-size-limit');
6265
my $tn = new TableNibbler(TableParser => $tp, Quoter => $q);
6266
my $retry = new Retry();
6267
6268
# ########################################################################
6269
# Callbacks for each table's nibble iterator. All checksum work is done
6270
# in these callbacks and the subs that they call.
6271
# ########################################################################
6272
my $callbacks = {
6273
init => sub {
6274
my (%args) = @_;
6275
my $tbl = $args{tbl};
6276
my $nibble_iter = $args{NibbleIterator};
6277
my $oktonibble = 1;
6278
6279
if ( $last_chunk ) { # resuming
6280
if ( have_more_chunks(%args, last_chunk => $last_chunk) ) {
6281
$nibble_iter->set_nibble_number($last_chunk->{chunk});
6282
PTDEBUG && _d('Have more chunks; resuming from',
6283
$last_chunk->{chunk}, 'at', $last_chunk->{ts});
6284
if ( !$o->get('quiet') ) {
6285
print "Resuming from $tbl->{db}.$tbl->{tbl} chunk "
6286
. "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n";
6287
}
6288
}
6289
else {
6290
# Problem resuming or no next lower boundary.
6291
PTDEBUG && _d('No more chunks; resuming from next table');
6292
$oktonibble = 0; # don't nibble table; next table
6293
}
6294
6295
# Just need to call us once to kick-start the resume process.
6296
$last_chunk = undef;
6297
}
6298
6299
if ( $o->get('explain') ) {
6300
# --explain level 1: print the checksum and next boundary
6301
# statements.
6302
print "--\n",
6303
"-- $tbl->{db}.$tbl->{tbl}\n",
6304
"--\n\n";
6305
my $statements = $nibble_iter->statements();
6306
foreach my $sth ( sort keys %$statements ) {
6307
next if $sth =~ m/^explain/;
6308
if ( $statements->{$sth} ) {
6309
print $statements->{$sth}->{Statement}, "\n\n";
6310
}
6311
}
6312
6313
if ( $o->get('explain') < 2 ) {
6314
$oktonibble = 0; # don't nibble table; next table
6315
}
6316
}
6317
else {
6318
if ( $nibble_iter->one_nibble() ) {
6319
PTDEBUG && _d('Getting table row estimate on replicas');
6320
my $chunk_size_limit = $o->get('chunk-size-limit');
6321
my @too_large;
6322
foreach my $slave ( @$slaves ) {
6323
my ($n_rows) = NibbleIterator::get_row_estimate(
6324
Cxn => $slave,
6325
tbl => $tbl,
6326
where => $o->get('where') || "1=1",
6327
OptionParser => $o,
6328
TableParser => $tp,
6329
Quoter => $q,
6330
);
6331
PTDEBUG && _d('Table on', $slave->name(),
6332
'has', $n_rows, 'rows');
6333
if ( $n_rows
6334
&& $n_rows > ($tbl->{chunk_size} * $chunk_size_limit) )
6335
{
6336
PTDEBUG && _d('Table too large on', $slave->name());
6337
push @too_large, [$slave->name(), $n_rows || 0];
6338
}
6339
}
6340
if ( @too_large ) {
6341
if ( $o->get('quiet') < 2 ) {
6342
my $msg
6343
= "Skipping table $tbl->{db}.$tbl->{tbl} because"
6344
. " on the master it would be checksummed in one chunk"
6345
. " but on these replicas it has too many rows:\n";
6346
foreach my $info ( @too_large ) {
6347
$msg .= " $info->[1] rows on $info->[0]\n";
6348
}
6349
$msg .= "The current chunk size limit is "
6350
. ($tbl->{chunk_size} * $chunk_size_limit)
6351
. " rows (chunk size=$tbl->{chunk_size}"
6352
. " * chunk size limit=$chunk_size_limit).\n";
6353
warn ts($msg);
6354
}
6355
$tbl->{checksum_results}->{errors}++;
6356
$oktonibble = 0;
6357
}
6358
}
6359
6360
if ( $oktonibble && $o->get('empty-replicate-table') ) {
6361
use_repl_db(
6362
dbh => $master_cxn->dbh(),
6363
repl_table => $repl_table,
6364
OptionParser => $o,
6365
Quoter => $q,
6366
);
6367
PTDEBUG && _d($delete_sth->{Statement});
6368
$delete_sth->execute($tbl->{db}, $tbl->{tbl});
6369
}
6370
6371
# USE the correct db while checksumming this table. The "correct"
6372
# db is a complicated subject; see sub for comments.
6373
use_repl_db(
6374
dbh => $master_cxn->dbh(),
6375
tbl => $tbl, # XXX working on this table
6376
repl_table => $repl_table,
6377
OptionParser => $o,
6378
Quoter => $q,
6379
);
6380
# #########################################################
6381
# XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX
6382
# #########################################################
6383
}
6384
6385
return $oktonibble; # continue nibbling table?
6386
},
6387
next_boundaries => sub {
6388
my (%args) = @_;
6389
my $tbl = $args{tbl};
6390
my $nibble_iter = $args{NibbleIterator};
6391
my $sth = $nibble_iter->statements();
6392
my $boundary = $nibble_iter->boundaries();
6393
6394
return 1 if $nibble_iter->one_nibble();
6395
6396
# Check that MySQL will use the nibble index for the next upper
6397
# boundary sql. This check applies to the next nibble. So if
6398
# the current nibble number is 5, then nibble 5 is already done
6399
# and we're checking nibble number 6.
6400
my $expl = explain_statement(
6401
tbl => $tbl,
6402
sth => $sth->{explain_upper_boundary},
6403
vals => [ @{$boundary->{lower}}, $nibble_iter->chunk_size() ],
6404
);
6405
if ( lc($expl->{key} || '')
6406
ne lc($nibble_iter->nibble_index() || '') ) {
6407
PTDEBUG && _d('Cannot nibble next chunk, aborting table');
6408
if ( $o->get('quiet') < 2 ) {
6409
my $msg
6410
= "Aborting table $tbl->{db}.$tbl->{tbl} at chunk "
6411
. ($nibble_iter->nibble_number() + 1)
6412
. " because it is not safe to chunk. Chunking should "
6413
. "use the "
6414
. ($nibble_iter->nibble_index() || '?')
6415
. " index, but MySQL EXPLAIN reports that "
6416
. ($expl->{key} ? "the $expl->{key}" : "no")
6417
. " index will be used.\n";
6418
warn ts($msg);
6419
}
6420
$tbl->{checksum_results}->{errors}++;
6421
return 0; # stop nibbling table
6422
}
6423
6424
# Once nibbling begins for a table, control does not return to this
6425
# tool until nibbling is done because, as noted above, all work is
6426
# done in these callbacks. This callback is the only place where we
6427
# can prematurely stop nibbling by returning false. This allows
6428
# Ctrl-C to stop the tool between nibbles instead of between tables.
6429
return $oktorun; # continue nibbling table?
6430
},
6431
exec_nibble => sub {
6432
my (%args) = @_;
6433
my $tbl = $args{tbl};
6434
my $nibble_iter = $args{NibbleIterator};
6435
my $sth = $nibble_iter->statements();
6436
my $boundary = $nibble_iter->boundaries();
6437
6438
# Count every chunk, even if it's ultimately skipped, etc.
6439
$tbl->{checksum_results}->{n_chunks}++;
6440
6441
# --explain level 2: print chunk,lower boundary values,upper
6442
# boundary values.
6443
if ( $o->get('explain') > 1 ) {
6444
my $chunk = $nibble_iter->nibble_number();
6445
if ( $nibble_iter->one_nibble() ) {
6446
printf "%d 1=1\n", $chunk;
6447
}
6448
else {
6449
my $lb_quoted = join(
6450
',', map { defined $_ ? $_ : 'NULL'} @{$boundary->{lower}});
6451
my $ub_quoted = join(
6452
',', map { defined $_ ? $_ : 'NULL'} @{$boundary->{upper}});
6453
printf "%d %s %s\n", $chunk, $lb_quoted, $ub_quoted;
6454
}
6455
if ( !$nibble_iter->more_boundaries() ) {
6456
print "\n"; # blank line between this table and the next table
6457
}
6458
return 0; # next boundary
6459
}
6460
6461
# If the table is being chunk (i.e., it's not small enough to be
6462
# consumed by one nibble), then check index usage and chunk size.
6463
if ( !$nibble_iter->one_nibble() ) {
6464
my $expl = explain_statement(
6465
tbl => $tbl,
6466
sth => $sth->{explain_nibble},
6467
vals => [ @{$boundary->{lower}}, @{$boundary->{upper}} ],
6468
);
6469
my $oversize_chunk
6470
= $limit ? ($expl->{rows} || 0) >= $tbl->{chunk_size} * $limit
6471
: 0;
6472
6473
# Ensure that MySQL is using the chunk index.
6474
if ( lc($expl->{key} || '')
6475
ne lc($nibble_iter->nibble_index() || '') ) {
6476
PTDEBUG && _d('Chunk', $args{nibbleno}, 'of table',
6477
"$tbl->{db}.$tbl->{tbl} not using chunk index, skipping");
6478
return 0; # next boundary
6479
}
6480
6481
# Check chunk size limit if the upper boundary and next lower
6482
# boundary are identical.
6483
if ( $limit ) {
6484
my $boundary = $nibble_iter->boundaries();
6485
if ( $nibble_iter->identical_boundaries(
6486
$boundary->{upper}, $boundary->{next_lower})
6487
&& $oversize_chunk ) {
6488
PTDEBUG && _d('Chunk', $args{nibbleno}, 'of table',
6489
"$tbl->{db}.$tbl->{tbl} is too large, skipping");
6490
return 0; # next boundary
6491
}
6492
}
6493
}
6494
6495
# Exec and time the chunk checksum query.
6496
$tbl->{nibble_time} = exec_nibble(
6497
%args,
6498
Retry => $retry,
6499
Quoter => $q,
6500
OptionParser => $o,
6501
);
6502
PTDEBUG && _d('Nibble time:', $tbl->{nibble_time});
6503
6504
# We're executing REPLACE queries which don't return rows.
6505
# Returning 0 from this callback causes the nibble iter to
6506
# get the next boundaries/nibble.
6507
return 0;
6508
},
6509
after_nibble => sub {
6510
my (%args) = @_;
6511
my $tbl = $args{tbl};
6512
my $nibble_iter = $args{NibbleIterator};
6513
6514
# Don't need to do anything here if we're just --explain'ing.
6515
return if $o->get('explain');
6516
6517
# Chunk/nibble number that we just inserted or skipped.
6518
my $chunk = $nibble_iter->nibble_number();
6519
6520
# Nibble time will be zero if the chunk was skipped.
6521
if ( !defined $tbl->{nibble_time} ) {
6522
PTDEBUG && _d('Skipping chunk', $chunk);
6523
$tbl->{checksum_results}->{skipped}++;
6524
return;
6525
}
6526
6527
# Max chunk number that worked. This may be less than the total
6528
# number of chunks if, for example, chunk 16 of 16 times out, but
6529
# chunk 15 worked. The max chunk is used for checking for diffs
6530
# on the slaves, in the done callback.
6531
$tbl->{max_chunk} = $chunk;
6532
6533
# Fetch the checksum that we just executed from the replicate table.
6534
$fetch_sth->execute(@{$tbl}{qw(db tbl)}, $chunk);
6535
my ($crc, $cnt) = $fetch_sth->fetchrow_array();
6536
6537
$tbl->{checksum_results}->{n_rows} += $cnt || 0;
6538
6539
# We're working on the master, so update the checksum's master_cnt
6540
# and master_crc.
6541
$update_sth->execute(
6542
# UPDATE repl_table SET
6543
sprintf('%.6f', $tbl->{nibble_time}), # chunk_time
6544
$crc, # master_crc
6545
$cnt, # master_cnt
6546
# WHERE
6547
$tbl->{db},
6548
$tbl->{tbl},
6549
$chunk,
6550
);
6551
6552
# Should be done automatically, but I like to be explicit.
6553
$fetch_sth->finish();
6554
$update_sth->finish();
6555
6556
# Update rate, chunk size, and progress if the nibble actually
6557
# selected some rows.
6558
if ( ($cnt || 0) > 0 ) {
6559
# Update the rate of rows per second for the entire server.
6560
# This is used for the initial chunk size of the next table.
6561
$total_rows += $cnt;
6562
$total_time += $tbl->{nibble_time};
6563
$total_rate = int($total_rows / $total_time);
6564
PTDEBUG && _d('Total avg rate:', $total_rate);
6565
6566
# Adjust chunk size. This affects the next chunk.
6567
if ( $o->get('chunk-time') ) {
6568
$tbl->{chunk_size}
6569
= $tbl->{rate}->update($cnt, $tbl->{nibble_time});
6570
6571
if ( $tbl->{chunk_size} < 1 ) {
6572
# This shouldn't happen. WeightedAvgRate::update() may return
6573
# a value < 1, but minimum chunk size is 1.
6574
$tbl->{chunk_size} = 1;
6575
6576
# This warning is printed once per table.
6577
if ( !$tbl->{warned_slow} && $o->get('quiet') < 2 ) {
6578
warn ts("Checksum queries for table "
6579
. "$tbl->{db}.$tbl->{tbl} are executing very slowly. "
6580
. "--chunk-size has been automatically reduced to 1. "
6581
. "Check that the server is not being overloaded, "
6582
. "or increase --chunk-time. The last chunk, number "
6583
. "$chunk of table $tbl->{db}.$tbl->{tbl}, "
6584
. "selected $cnt rows and took "
6585
. sprintf('%.3f', $tbl->{nibble_time})
6586
. " seconds to execute.\n");
6587
$tbl->{warned_slow} = 1;
6588
}
6589
}
6590
6591
# Update chunk-size based on rows/s checksum rate.
6592
$nibble_iter->set_chunk_size($tbl->{chunk_size});
6593
}
6594
6595
# Every table should have a Progress obj; update it.
6596
if ( my $tbl_pr = $tbl->{progress} ) {
6597
$tbl_pr->update(sub {return $tbl->{checksum_results}->{n_rows}});
6598
}
6599
}
6600
6601
# Wait forever for slaves to catch up.
6602
$replica_lag_pr->start() if $replica_lag_pr;
6603
$replica_lag->wait(Progress => $replica_lag_pr);
6604
6605
# Wait forever for system load to abate.
6606
$sys_load_pr->start() if $sys_load_pr;
6607
$sys_load->wait(Progress => $sys_load_pr);
6608
6609
return;
6610
},
6611
done => sub { # done nibbling table
6612
my (%args) = @_;
6613
my $tbl = $args{tbl};
6614
my $nibble_iter = $args{NibbleIterator};
6615
my $max_chunk = $tbl->{max_chunk};
6616
6617
# Don't need to do anything here if we're just --explain'ing.
6618
return if $o->get('explain');
6619
6620
# Wait for all slaves to run all checksum chunks,
6621
# then check for differences.
6622
if ( $max_chunk && $o->get('replicate-check') && scalar @$slaves ) {
6623
PTDEBUG && _d('Checking slave diffs');
6624
6625
my $check_pr;
6626
if ( $o->get('progress') ) {
6627
$check_pr = new Progress(
6628
jobsize => $max_chunk,
6629
spec => $o->get('progress'),
6630
name => "Waiting to check replicas for differences",
6631
);
6632
}
6633
6634
# Wait for the last checksum of this table to replicate
6635
# to each slave.
6636
wait_for_last_checksum(
6637
tbl => $tbl,
6638
repl_table => $repl_table,
6639
slaves => $slaves,
6640
max_chunk => $max_chunk,
6641
check_pr => $check_pr,
6642
OptionParser => $o,
6643
);
6644
6645
# Check each slave for checksum diffs.
6646
foreach my $slave ( @$slaves ) {
6647
eval {
6648
my $diffs = $rc->find_replication_differences(
6649
dbh => $slave->dbh(),
6650
repl_table => $repl_table,
6651
where => "db='$tbl->{db}' AND tbl='$tbl->{tbl}'",
6652
);
6653
PTDEBUG && _d(scalar @$diffs, 'checksum diffs on',
6654
$slave->name());
6655
if ( @$diffs ) {
6656
$tbl->{checksum_results}->{diffs} = scalar @$diffs;
6657
}
6658
};
6659
if ($EVAL_ERROR) {
6660
if ( $o->get('quiet') < 2 ) {
6661
warn ts("Error checking for checksum differences of table "
6662
. "$tbl->{db}.$tbl->{tbl} on replica " . $slave->name()
6663
. ": $EVAL_ERROR\n"
6664
. "Check that the replica is running and has the "
6665
. "replicate table $repl_table.\n");
6666
}
6667
$tbl->{checksum_results}->{errors}++;
6668
}
6669
}
6670
}
6671
6672
# Print table's checksum results if we're not being quiet,
6673
# else print if table has diffs and we're not being completely
6674
# quiet.
6675
if ( !$o->get('quiet')
6676
|| $o->get('quiet') < 2 && $tbl->{checksum_results}->{diffs} ) {
6677
print_checksum_results(tbl => $tbl);
6678
}
6679
6680
return;
6681
},
6682
};
6683
6684
# ########################################################################
6685
# Checksum each table.
6686
# ########################################################################
6687
6688
TABLE:
6689
while ( $oktorun && (my $tbl = $schema_iter->next()) ) {
6690
eval {
6691
# Results, stats, and info related to checksuming this table can
6692
# be saved here. print_checksum_results() uses this info.
6693
$tbl->{checksum_results} = {};
6694
6695
# Set table's initial chunk size. If this is the first table,
6696
# then total rate will be zero, so use --chunk-size. Or, if
6697
# --chunk-time=0, then only use --chunk-size for every table.
6698
# Else, the initial chunk size is based on the total rates of
6699
# rows/s from all previous tables. If --chunk-time is really
6700
# small, like 0.001, then Perl int() will probably round the
6701
# chunk size to zero, which is invalid, so we default to 1.
6702
my $chunk_time = $o->get('chunk-time');
6703
my $chunk_size = $chunk_time && $total_rate
6704
? int($total_rate * $chunk_time) || 1
6705
: $o->get('chunk-size');
6706
$tbl->{chunk_size} = $chunk_size;
6707
6708
# Make a nibble iterator for this table. This should only fail
6709
# if the table has no indexes and is too large to checksum in
6710
# one chunk.
6711
my $checksum_cols = $rc->make_chunk_checksum(
6712
dbh => $master_cxn->dbh(),
6713
tbl => $tbl,
6714
%crc_args
6715
);
6716
my $nibble_iter;
6717
eval {
6718
$nibble_iter = new OobNibbleIterator(
6719
Cxn => $master_cxn,
6720
tbl => $tbl,
6721
chunk_size => $tbl->{chunk_size},
6722
chunk_index => $o->get('chunk-index'),
6723
dml => $checksum_dml,
6724
select => $checksum_cols,
6725
past_dml => $checksum_dml,
6726
past_select => $past_cols,
6727
callbacks => $callbacks,
6728
resume => $last_chunk,
6729
OptionParser => $o,
6730
Quoter => $q,
6731
TableNibbler => $tn,
6732
TableParser => $tp,
6733
RowChecksum => $rc,
6734
);
6735
};
6736
if ( $EVAL_ERROR ) {
6737
if ( $o->get('quiet') < 2 ) {
6738
warn ts("Cannot checksum table $tbl->{db}.$tbl->{tbl}: "
6739
. "$EVAL_ERROR\n");
6740
}
6741
$tbl->{checksum_results}->{errors}++;
6742
}
6743
else {
6744
# Init a new weighted avg rate calculator for the table.
6745
$tbl->{rate} = new WeightedAvgRate(target_t => $chunk_time);
6746
6747
# Make a Progress obj for this table. It may not be used;
6748
# depends on how many rows, chunk size, how fast the server
6749
# is, etc. But just in case, all tables have a Progress obj.
6750
if ( $o->get('progress')
6751
&& !$nibble_iter->one_nibble()
6752
&& $nibble_iter->row_estimate() )
6753
{
6754
$tbl->{progress} = new Progress(
6755
jobsize => $nibble_iter->row_estimate(),
6756
spec => $o->get('progress'),
6757
name => "Checksumming $tbl->{db}.$tbl->{tbl}",
6758
);
6759
}
6760
6761
# Finally, checksum the table.
6762
# The "1 while" loop is necessary because we're executing REPLACE
6763
# statements which don't return rows and NibbleIterator only
6764
# returns if it has rows to return. So all the work is done via
6765
# the callbacks. -- print_checksum_results(), which is called
6766
# from the done callback, uses this start time.
6767
$tbl->{checksum_results}->{start_time} = time;
6768
1 while $nibble_iter->next();
6769
}
6770
};
6771
if ( $EVAL_ERROR ) {
6772
# This should not happen. If it does, it's probably some bug
6773
# or error that we're not catching.
6774
warn ts(($oktorun ? "Error " : "Fatal error ")
6775
. "checksumming table $tbl->{db}.$tbl->{tbl}: "
6776
. "$EVAL_ERROR\n");
6777
$tbl->{checksum_results}->{errors}++;
6778
6779
# Print whatever checksums results we got before dying, regardless
6780
# of --quiet because at this point we need all the info we can get.
6781
print_checksum_results(tbl => $tbl);
6782
}
6783
6784
# Update the tool's exit status.
6785
if ( $tbl->{checksum_results}->{errors}
6786
|| $tbl->{checksum_results}->{diffs} ) {
6787
$exit_status |= 1;
6788
}
6789
}
6790
6791
PTDEBUG && _d('Exit status', $exit_status, 'oktorun', $oktorun);
5447
6792
return $exit_status;
5448
6793
}
5449
6794
5450
6795
# ############################################################################
5451
6796
# Subroutines
5452
6797
# ############################################################################
5453
5454
sub get_all_tbls_info {
5455
my ( %args ) = @_;
5456
foreach my $arg ( qw(o dbh q tp du ch args_for) ) {
5457
die "I need a $arg argument" unless $args{$arg};
5458
}
5459
my $dbh = $args{dbh};
5460
MKDEBUG && _d('Getting all schema objects');
5461
5462
my $si = new SchemaIterator(
5463
dbh => $dbh,
5464
OptionParser => $args{o},
5465
Quoter => $args{q},
5466
);
5467
while ( my %schema_obj = $si->next_schema_object() ) {
5468
my $final_o = get_final_opts(
5469
%args,
5470
%schema_obj,
5471
);
5472
save_tbl_to_checksum(
5473
%args,
5474
%schema_obj,
5475
final_o => $final_o,
5476
);
5477
}
5478
5479
return;
5480
}
5481
5482
sub save_tbl_to_checksum {
5483
my ( %args ) = @_;
5484
foreach my $arg ( qw(q ch du final_o tp dbh db tbl du tp ch vp) ) {
5485
die "I need a $arg argument" unless $args{$arg};
5486
}
5487
my $du = $args{du};
5488
my $tp = $args{tp};
5489
my $ch = $args{ch};
5490
my $final_o = $args{final_o};
5491
my $dbh = $args{dbh};
5492
my $db = $args{db};
5493
my $tbl = $args{tbl};
5494
my $q = $args{q};
5495
my $vp = $args{vp};
5496
5497
# Skip the table in which checksums are stored.
5498
return if ($final_o->get('replicate')
5499
&& $final_o->get('replicate') eq "$db.$tbl");
5500
5501
eval { # Catch errors caused by tables being dropped during work.
5502
5503
# Parse the table and determine a column that's chunkable. This is
5504
# used not only for chunking, but also for --since.
5505
my $create = $du->get_create_table($dbh, $q, $db, $tbl);
5506
my $struct = $tp->parse($create);
5507
5508
# If there's a --where clause and the user didn't specify a chunk index
5509
# a chunk they want, then get MySQL's chosen index for the where clause
5510
# and make it the preferred index.
5511
# http://code.google.com/p/maatkit/issues/detail?id=378
5512
if ( $final_o->get('where')
5513
&& !$final_o->get('chunk-column')
5514
&& !$final_o->get('chunk-index') )
5515
{
5516
my ($mysql_chosen_index) = $tp->find_possible_keys(
5517
$dbh, $db, $tbl, $q, $final_o->get('where'));
5518
MKDEBUG && _d("Index chosen by MySQL for --where:",
5519
$mysql_chosen_index);
5520
$final_o->set('chunk-index', $mysql_chosen_index)
5521
if $mysql_chosen_index;
5522
}
5523
5524
5525
# Get the first chunkable column and index, taking into account
5526
# --chunk-column and --chunk-index. If either of those options
5527
# is specified, get_first_chunkable_column() will try to satisfy
5528
# the request but there's no guarantee either will be selected.
5529
# http://code.google.com/p/maatkit/issues/detail?id=519
5530
my ($chunk_col, $chunk_index) = $ch->get_first_chunkable_column(
5531
%args,
5532
chunk_column => $final_o->get('chunk-column'),
5533
chunk_index => $final_o->get('chunk-index'),
5534
tbl_struct => $struct,
5535
);
5536
5537
my $index_hint;
5538
if ( $final_o->get('use-index') && $chunk_col ) {
5539
my $hint = $vp->version_ge($dbh, '4.0.9') ? 'FORCE' : 'USE';
5540
$index_hint = "$hint INDEX (" . $q->quote($chunk_index) . ")";
5541
}
5542
MKDEBUG && _d('Index hint:', $index_hint);
5543
5544
my @chunks = '1=1'; # Default.
5545
my $rows_per_chunk = undef;
5546
my $maxval = undef;
5547
if ( $final_o->get('chunk-size') ) {
5548
($rows_per_chunk) = $ch->size_to_rows(
5549
dbh => $dbh,
5550
db => $db,
5551
tbl => $tbl,
5552
chunk_size => $final_o->get('chunk-size'),
5553
);
5554
5555
if ( $chunk_col ) {
5556
# Calculate chunks for this table.
5557
my %params = $ch->get_range_statistics(
5558
dbh => $dbh,
5559
db => $db,
5560
tbl => $tbl,
5561
chunk_col => $chunk_col,
5562
tbl_struct => $struct,
5563
);
5564
if ( !grep { !defined $params{$_} } qw(min max rows_in_range) ) {
5565
@chunks = $ch->calculate_chunks(
5566
dbh => $dbh,
5567
db => $db,
5568
tbl => $tbl,
5569
tbl_struct => $struct,
5570
chunk_col => $chunk_col,
5571
chunk_size => $rows_per_chunk,
5572
zero_chunk => $final_o->get('zero-chunk'),
5573
chunk_range => $final_o->get('chunk-range'),
5574
%params,
5575
);
5576
$maxval = $params{max};
5577
}
5578
}
5579
}
5580
5581
push @{ $tables_to_checksum{$db} }, {
5582
struct => $struct,
5583
create => $create,
5584
database => $db,
5585
table => $tbl,
5586
column => $chunk_col,
5587
chunk_index => $chunk_index,
5588
chunk_size => $rows_per_chunk,
5589
maxval => $maxval,
5590
index => $index_hint,
5591
chunks => \@chunks,
5592
final_o => $final_o,
5593
};
5594
};
5595
if ( $EVAL_ERROR ) {
5596
print_err($final_o, $EVAL_ERROR, $db, $tbl);
5597
}
5598
5599
return;
5600
}
5601
5602
# Checksum the tables in the given database.
5603
# A separate report for each database and its tables is printed.
5604
sub checksum_tables {
5605
my ( %args ) = @_;
5606
foreach my $arg ( qw(tc du o q db dbh hosts tbls) ) {
5607
die "I need a $arg argument" unless $args{$arg};
5608
}
5609
my $tc = $args{tc};
5610
my $du = $args{du};
5611
my $o = $args{o};
5612
my $db = $args{db};
5613
my $dbh = $args{dbh};
5614
my $hosts = $args{hosts};
5615
my $tbls = $args{tbls};
5616
my $q = $args{q};
5617
5618
my ($hdr, $explain);
5619
my $exit_status = 0;
5620
5621
# NOTE: remember, you can't 'next TABLE' inside the eval{}.
5622
# NOTE: remember to use the final_o embedded within each $table, not $o
5623
foreach my $table ( @$tbls ) {
5624
MKDEBUG && _d("Doing", $db, '.', $table->{table});
5625
MKDEBUG && _d("Table:", Dumper($table));
5626
my $final_o = $table->{final_o};
5627
5628
my $is_chunkable_table = 1; # table should be chunkable unless...
5629
5630
# If there's a chunk size but no chunk index and unchunkable tables
5631
# aren't allowed (they're not by default), then table may still be
5632
# chunkable if it's small, i.e. total rows in table <= chunk size.
5633
if ( $table->{chunk_size}
5634
&& !$table->{chunk_index}
5635
&& !$final_o->get('unchunkable-tables') )
5636
{
5637
$is_chunkable_table = is_chunkable_table(
5638
dbh => $dbh,
5639
db => $db,
5640
tbl => $table->{table},
5641
chunk_size => $table->{chunk_size},
5642
where => $final_o->{where},
5643
Quoter => $q,
5644
);
5645
MKDEBUG && _d("Unchunkable table small enough to chunk:",
5646
$is_chunkable_table ? 'yes' : 'no');
5647
}
5648
5649
if ( !$is_chunkable_table ) {
5650
$exit_status |= 1;
5651
print "# cannot chunk $table->{database} $table->{table}\n";
5652
}
5653
else {
5654
eval {
5655
my $do_table = 1;
5656
5657
# Determine the checksum strategy for every table because it
5658
# might change given various --arg-table opts for each table.
5659
my $strat_ref;
5660
my ( $strat, $crc_type, $func, $opt_slice );
5661
if ( $checksum_table_data && $do_table ) {
5662
$strat_ref = determine_checksum_strat(
5663
dbh => $dbh,
5664
tc => $tc,
5665
o => $final_o,
5666
);
5667
( $strat, $crc_wid, $crc_type, $func, $opt_slice )
5668
= @$strat_ref{ qw(strat crc_wid crc_type func opt_slice) };
5669
MKDEBUG && _d("Checksum strat:", Dumper($strat_ref));
5670
}
5671
else {
5672
# --schema doesn't use a checksum strategy, but do_tbl()
5673
# requires a strat arg.
5674
$strat = '--schema';
5675
}
5676
$md5sum_fmt = "%-${crc_wid}s %s.%s.%s.%d\n";
5677
5678
# Design and print header unless we are resuming in which case
5679
# we should have already re-printed the partial output of the
5680
# resume file in parse_resume_file(). This only has to be done
5681
# once and done here because we need $crc_wid which is determined
5682
# by the checksum strat above.
5683
if ( !$hdr ) {
5684
if ( $o->get('tab') ) {
5685
$hdr = "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n";
5686
$explain = "%s\t%s\t%s\n";
5687
}
5688
else {
5689
my $max_tbl = max(5, map { length($_->{table}) } @$tbls);
5690
my $max_db = max(8, length($db));
5691
my $max_host = max(4, map { length($_->{h}) } @$hosts);
5692
$hdr = "%-${max_db}s %-${max_tbl}s %5s "
5693
. "%-${max_host}s %-6s %10s %${crc_wid}s %4s %4s %4s %4s\n";
5694
$explain = "%-${max_db}s %-${max_tbl}s %s\n";
5695
}
5696
my @hdr_args = qw(DATABASE TABLE CHUNK HOST ENGINE
5697
COUNT CHECKSUM TIME WAIT STAT LAG);
5698
unless ( $o->get('quiet')
5699
|| $o->get('explain')
5700
|| $o->get('checksum')
5701
|| $o->get('resume') )
5702
{
5703
printf($hdr, @hdr_args)
5704
or die "Cannot print: $OS_ERROR";
5705
}
5706
}
5707
5708
# Clean out the replication table entry for this table.
5709
# http://code.google.com/p/maatkit/issues/detail?id=304
5710
if ( (my $replicate_table = $final_o->get('replicate'))
5711
&& !$final_o->get('explain') ) {
5712
use_repl_db(%args); # USE the proper replicate db
5713
my $max_chunkno = scalar @{$table->{chunks}} - 1;
5714
my $del_sql = "DELETE FROM $replicate_table "
5715
. "WHERE db=? AND tbl=? AND chunk > ?";
5716
MKDEBUG && _d($dbh, $del_sql, $db, $table->{table},$max_chunkno);
5717
$dbh->do($del_sql, {}, $db, $table->{table}, $max_chunkno);
5718
}
5719
5720
# If --since is given, figure out either
5721
# 1) for temporal sinces, if the table has an update time and that
5722
# time is newer than --since, then checksum the whole table,
5723
# otherwise skip it; or
5724
# 2) for "numerical" sinces, which column to use: either the
5725
# specified column (--sincecolumn) or the auto-discovered one,
5726
# whichever exists in the table, in that order.
5727
# Then, if --savesince is given, save either 1) the current timestamp
5728
# or 2) the resulting WHERE clause.
5729
if ( $final_o->get('since') ) {
5730
if ( is_temporal($final_o->get('since')) ) {
5731
MKDEBUG && _d('--since is temporal');
5732
my ( $stat )
5733
= $du->get_table_status($dbh, $q, $db, $table->{table});
5734
my $time = $stat->{update_time};
5735
if ( $time && $time lt $final_o->get('since') ) {
5736
MKDEBUG && _d("Skipping table because --since value",
5737
$final_o->get('since'), "is newer than", $time);
5738
$do_table = 0;
5739
$table->{chunks} = [];
5740
}
5741
}
5742
else {
5743
MKDEBUG && _d('--since is numerical');
5744
# For numerical sinces, choose the column to apply --since to.
5745
# It may not be the column the user said to use! If the user
5746
# didn't specify a column that's good to chunk on, we'll use
5747
# something else instead.
5748
5749
# $table->{column} is the first chunkable column returned from
5750
# the call to get_first_chunkable_column() in
5751
# save_tbl_to_checksum().
5752
my ( $sincecol ) =
5753
grep { $_ && $table->{struct}->{is_col}->{$_} }
5754
( $table->{column}, $final_o->get('since-column') );
5755
5756
if ( $sincecol ) {
5757
MKDEBUG && _d('Column for numerical --since:',
5758
$db, '.', $table->{table}, '.', $sincecol);
5759
# This ends up being an additional WHERE clause.
5760
$table->{since} = $q->quote($sincecol)
5761
. '>=' . $q->quote_val($final_o->get('since'));
5762
}
5763
else {
5764
MKDEBUG && _d('No column for numerical --since for',
5765
$db, '.', $table->{table});
5766
}
5767
}
5768
}
5769
5770
# ##################################################################
5771
# The query is independent of the chunk, so I make it once for every
5772
# one.
5773
# ##################################################################
5774
my $query;
5775
if ( $checksum_table_data && $do_table ) {
5776
$query = $tc->make_checksum_query(
5777
db => $db,
5778
tbl => $table->{table},
5779
tbl_struct => $table->{struct},
5780
algorithm => $strat,
5781
function => $func,
5782
crc_wid => $crc_wid,
5783
crc_type => $crc_type,
5784
opt_slice => $opt_slice,
5785
cols => $final_o->get('columns'),
5786
sep => $final_o->get('separator'),
5787
replicate => $final_o->get('replicate'),
5788
float_precision => $final_o->get('float-precision'),
5789
trim => $final_o->get('trim'),
5790
ignorecols => $final_o->get('ignore-columns'),
5791
);
5792
}
5793
else { # --schema
5794
$query = undef;
5795
}
5796
5797
$exit_status |= checksum_chunks(
5798
%args,
5799
tbl => $table,
5800
query => $query,
5801
hdr => $hdr,
5802
explain => $explain,
5803
final_o => $final_o,
5804
strat => $strat,
5805
);
5806
5807
# Save the --since value if
5808
# 1) it's temporal and the tbl had changed since --since; or
5809
# 2) it's "numerical" and it had a chunkable or nibble-able
5810
# column and it wasn't empty
5811
# See issues 121 and 122.
5812
if ( $final_o->get('save-since') && $savesince_sth ) {
5813
if ( is_temporal($final_o->get('since')) ) {
5814
MKDEBUG && _d(
5815
"Saving temporal --since value: current timestamp for",
5816
$db, '.', $table->{table});
5817
$savesince_sth->execute(undef,
5818
$db, $table->{table});
5819
}
5820
elsif ( defined $table->{maxval} ) {
5821
MKDEBUG && _d("Saving numerical --since value:",
5822
$table->{maxval}, "for", $db, '.', $table->{table});
5823
$savesince_sth->execute($table->{maxval},
5824
$db, $table->{table});
5825
}
5826
else {
5827
MKDEBUG && _d("Cannot save --since value:",
5828
$table->{maxval}, "for", $db, '.', $table->{table});
5829
}
5830
}
5831
};
5832
if ( $EVAL_ERROR ) {
5833
print_err($o, $EVAL_ERROR, $db, $table->{table});
5834
}
5835
} # chunkable table
5836
}
5837
5838
return $exit_status;
5839
}
5840
5841
sub checksum_chunks {
5842
my ( %args ) = @_;
5843
foreach my $arg ( qw(dp final_o ms o q db tbl hosts hdr explain) ) {
5844
die "I need a $arg argument" unless $args{$arg};
5845
}
5846
my $dp = $args{dp};
5847
my $du = $args{du};
5848
my $final_o = $args{final_o};
5849
my $ms = $args{ms};
5850
my $o = $args{o};
5851
my $q = $args{q};
5852
my $db = $args{db};
5853
my $dbh = $args{dbh};
5854
my @hosts = @{$args{hosts}};
5855
my $tbl = $args{tbl};
5856
5857
my $retry = new Retry();
5858
5859
# ##################################################################
5860
# This loop may seem suboptimal, because it causes a new child to be
5861
# forked for each table, for each host, for each chunk. It also
5862
# causes the program to parallelize only within the chunk; that is,
5863
# no two child processes are running on different chunks at a time.
5864
# This is by design. It lets me unlock the table on the master
5865
# between chunks.
5866
# ##################################################################
5867
my $exit_status = 0;
5868
my $num_chunks = scalar(@{$tbl->{chunks}});
5869
my $throttle_method = $o->get('throttle-method');
5870
MKDEBUG && _d('Checksumming', $num_chunks, 'chunks');
5871
CHUNK:
5872
foreach my $chunk_num ( 0 .. $num_chunks - 1 ) {
5873
5874
if ( $final_o->get('chunk-size-limit')
5875
&& $final_o->get('chunk-size')
5876
&& $tbl->{chunk_size}
5877
&& !$final_o->get('explain') )
5878
{
5879
my $is_oversize_chunk = is_oversize_chunk(
5880
%args,
5881
db => $tbl->{database},
5882
tbl => $tbl->{table},
5883
chunk => $tbl->{chunks}->[$chunk_num],
5884
chunk_size => $tbl->{chunk_size},
5885
index_hint => $tbl->{index},
5886
where => [$final_o->get('where'), $tbl->{since}],
5887
limit => $final_o->get('chunk-size-limit'),
5888
Quoter => $q,
5889
);
5890
if ( $is_oversize_chunk ) {
5891
$exit_status |= 1;
5892
if ( !$final_o->get('quiet') ) {
5893
if ( $final_o->get('checksum') ) {
5894
printf($md5sum_fmt, 'NULL', '',
5895
@{$tbl}{qw(database table)}, $chunk_num)
5896
or die "Cannot print: $OS_ERROR";
5897
}
5898
else {
5899
printf($args{hdr},
5900
@{$tbl}{qw(database table)}, $chunk_num,
5901
$hosts[0]->{h}, $tbl->{struct}->{engine}, 'OVERSIZE',
5902
'NULL', 'NULL', 'NULL', 'NULL', 'NULL')
5903
or die "Cannot print: $OS_ERROR";
5904
}
5905
}
5906
next CHUNK;
5907
}
5908
}
5909
5910
if ( $throttle_method eq 'slavelag' ) {
5911
my $pr;
5912
if ( $o->get('progress') ) {
5913
$pr = new Progress(
5914
jobsize => scalar @{$args{slaves}},
5915
spec => $o->get('progress'),
5916
name => "Wait for slave(s) to catch up",
5917
);
5918
}
5919
wait_for_slaves(
5920
slaves => $args{slaves},
5921
max_lag => $o->get('max-lag'),
5922
check_interval => $o->get('check-interval'),
5923
DSNParser => $dp,
5924
MasterSlave => $ms,
5925
progress => $pr,
5926
);
5927
}
5928
5929
if ( ($num_chunks > 1 || $final_o->get('single-chunk'))
5930
&& $checksum_table_data
5931
&& defined $final_o->get('probability')
5932
&& rand(100) >= $final_o->get('probability') ) {
5933
MKDEBUG && _d('Skipping chunk because of --probability');
5934
next CHUNK;
5935
}
5936
5937
if ( $num_chunks > 1
5938
&& $checksum_table_data
5939
&& $final_o->get('modulo')
5940
&& ($chunk_num % $final_o->get('modulo') != $final_o->get('offset')))
5941
{
5942
MKDEBUG && _d('Skipping chunk', $chunk_num, 'because of --modulo');
5943
next CHUNK;
5944
}
5945
5946
my $chunk_start_time = gettimeofday();
5947
MKDEBUG && _d('Starting chunk', $chunk_num, 'at', $chunk_start_time);
5948
5949
if ( $final_o->get('replicate') ) {
5950
# We're in --replicate mode.
5951
5952
# If resuming, check if this db.tbl.chunk.host can be skipped.
5953
if ( $o->get('resume-replicate') ) {
5954
if ( already_checksummed($tbl->{database},
5955
$tbl->{table},
5956
$chunk_num,
5957
$hosts[0]->{h}) ) {
5958
print "# already checksummed:"
5959
. " $tbl->{database}"
5960
. " $tbl->{table}"
5961
. " $chunk_num "
5962
. $hosts[0]->{h}
5963
. "\n"
5964
unless $o->get('quiet');
5965
next CHUNK;
5966
}
5967
}
5968
5969
$hosts[0]->{dbh} ||= $dbh;
5970
5971
do_tbl_replicate(
5972
$chunk_num,
5973
%args,
5974
host => $hosts[0],
5975
retry => $retry,
5976
);
5977
}
5978
else {
5979
# We're in "normal" mode. Lock table and get position on the master.
5980
5981
if ( !$final_o->get('explain') ) {
5982
if ( $final_o->get('lock') ) {
5983
my $sql = "LOCK TABLES "
5984
. $q->quote($db, $tbl->{table}) . " READ";
5985
MKDEBUG && _d($sql);
5986
$dbh->do($sql);
5987
}
5988
if ( $final_o->get('wait') ) {
5989
$tbl->{master_status} = $ms->get_master_status($dbh);
5990
}
5991
}
5992
5993
my %children;
5994
HOST:
5995
foreach my $i ( 0 .. $#hosts ) {
5996
my $is_master = $i == 0; # First host is assumed to be master.
5997
my $host = $hosts[$i];
5998
5999
# Open a single connection for each host. Re-use the
6000
# connection for the master/single host.
6001
if ( $is_master ) {
6002
$dbh->{InactiveDestroy} = 1; # Ensure that this is set.
6003
$host->{dbh} ||= $dbh;
6004
}
6005
else {
6006
$host->{dbh} ||= get_cxn($host, %args);
6007
}
6008
6009
# If resuming, check if this db.tbl.chunk.host can be skipped.
6010
if ( $final_o->get('resume') ) {
6011
next HOST if already_checksummed($tbl->{database},
6012
$tbl->{table},
6013
$chunk_num,
6014
$host->{h});
6015
}
6016
6017
# Fork, but only if there's more than one host.
6018
my $pid = @hosts > 1 ? fork() : undef;
6019
6020
if ( @hosts == 1 || (defined($pid) && $pid == 0) ) {
6021
# Do the work (I'm a child, or there's only one host)
6022
6023
eval {
6024
do_tbl(
6025
$chunk_num,
6026
$is_master,
6027
%args,
6028
dbh => $host->{dbh},
6029
host => $host,
6030
);
6031
};
6032
if ( $EVAL_ERROR ) {
6033
print_err($o, $EVAL_ERROR, $db, $tbl->{table},
6034
$dp->as_string($host));
6035
exit(1) if @hosts > 1; # exit only if I'm a child
6036
}
6037
6038
exit(0) if @hosts > 1; # exit only if I'm a child
6039
}
6040
elsif ( @hosts > 1 && !defined($pid) ) {
6041
die("Unable to fork!");
6042
}
6043
6044
# I already exited if I'm a child, so I'm the parent.
6045
$children{$host->{h}} = $pid if @hosts > 1;
6046
}
6047
6048
# Wait for the children to exit.
6049
foreach my $host ( keys %children ) {
6050
my $pid = waitpid($children{$host}, 0);
6051
MKDEBUG && _d("Child", $pid, "exited with", $CHILD_ERROR);
6052
$exit_status ||= $CHILD_ERROR >> 8;
6053
}
6054
if ( ($final_o->get('lock') && !$final_o->get('explain')) ) {
6055
my $sql = "UNLOCK TABLES";
6056
MKDEBUG && _d($dbh, $sql);
6057
$dbh->do($sql);
6058
}
6059
}
6060
6061
my $chunk_stop_time = gettimeofday();
6062
MKDEBUG && _d('Finished chunk at', $chunk_stop_time);
6063
6064
# --sleep between chunks. Don't sleep if this is the last/only chunk.
6065
if ( $chunk_num < $num_chunks - 1 ) {
6066
if ( $final_o->get('sleep') && !$final_o->get('explain') ) {
6067
MKDEBUG && _d('Sleeping', $final_o->get('sleep'));
6068
sleep($final_o->get('sleep'));
6069
}
6070
elsif ( $final_o->get('sleep-coef') && !$final_o->get('explain') ) {
6071
my $sleep_time
6072
= ($chunk_stop_time - $chunk_start_time)
6073
* $final_o->get('sleep-coef');
6074
MKDEBUG && _d('Sleeping', $sleep_time);
6075
if ( $sleep_time < 0 ) {
6076
warn "Calculated invalid sleep time: "
6077
. "$sleep_time = ($chunk_stop_time - $chunk_start_time) * "
6078
. $final_o->get('sleep-coef')
6079
. ". Sleep time set to 1 second instead.";
6080
$sleep_time = 1;
6081
}
6082
sleep($sleep_time);
6083
}
6084
}
6085
} # End foreach CHUNK
6086
6087
return $exit_status;
6088
}
6089
6090
# Override the command-line arguments with those from --arg-table
6091
# if necessary. Returns a cloned OptionParser object ($final_o).
6092
# This clone is only a partial OptionParser object.
6093
sub get_final_opts {
6094
my ( %args ) = @_;
6095
foreach my $arg ( qw(o dbh db tbl args_for) ) {
6096
die "I need a $arg argument" unless $args{$arg};
6097
}
6098
my $o = $args{o};
6099
my $dbh = $args{dbh};
6100
my $db = $args{db};
6101
my $tbl = $args{tbl};
6102
my $args_for = $args{args_for};
6103
6104
my $final_o = $o->clone();
6105
if ( my $override = $args_for->{$db}->{$tbl} ) {
6106
map { $final_o->set($_, $override->{$_}); } keys %$override;
6107
}
6108
6109
# --since and --offset are potentially expressions that should be
6110
# evaluated by the DB server. This has to be done after the override
6111
# from the --arg-table table.
6112
foreach my $opt ( qw(since offset) ) {
6113
# Don't get MySQL to evaluate if it's temporal, as 2008-08-01 --> 1999
6114
my $val = $final_o->get($opt);
6115
if ( $val && !is_temporal($val) ) {
6116
$final_o->set($opt, eval_expr($opt, $val, $dbh));
6117
}
6118
}
6119
6120
return $final_o;
6121
}
6122
6123
sub is_temporal {
6124
my ( $val ) = @_;
6125
return $val && $val =~ m/^\d{4}-\d{2}-\d{2}(?:.[0-9:]+)?/;
6126
}
6127
6128
sub print_inconsistent_tbls {
6129
my ( %args ) = @_;
6130
foreach my $arg ( qw(o dp dsn tbls) ) {
6131
die "I need a $arg argument" unless $args{$arg};
6132
}
6133
my $o = $args{o};
6134
my $dp = $args{dp};
6135
my $dsn = $args{dsn};
6136
my $tbls = $args{tbls};
6137
6138
return if $o->get('quiet');
6139
6140
my @headers = qw(db tbl chunk cnt_diff crc_diff boundaries);
6141
print "Differences on " . $dp->as_string($dsn, [qw(h P F)]) . "\n";
6142
my $max_db = max(5, map { length($_->{db}) } @$tbls);
6143
my $max_tbl = max(5, map { length($_->{tbl}) } @$tbls);
6144
my $fmt = "%-${max_db}s %-${max_tbl}s %5s %8s %8s %s\n";
6145
printf($fmt, map { uc } @headers) or die "Cannot print: $OS_ERROR";
6146
foreach my $tbl ( @$tbls ) {
6147
printf($fmt, @{$tbl}{@headers}) or die "Cannot print: $OS_ERROR";
6148
}
6149
print "\n" or die "Cannot print: $OS_ERROR";
6150
6151
return;
6152
}
6153
6154
sub save_inconsistent_tbls {
6155
my ( %args ) = @_;
6156
foreach my $arg ( qw(dbh tbls) ) {
6157
die "I need a $arg argument" unless $args{$arg};
6158
}
6159
my $dbh = $args{dbh};
6160
my $tbls = $args{tbls};
6161
6162
foreach my $tbl ( @$tbls ) {
6163
MKDEBUG && _d("Will recheck", $tbl->{db}, '.', $tbl->{tbl},
6164
"(chunk:", $tbl->{boundaries}, ')');
6165
my $final_o = get_final_opts(
6166
%args,
6167
db => $tbl->{db},
6168
tbl => $tbl->{tbl},
6169
);
6170
my $chunks = [ $tbl->{boundaries} ];
6171
save_tbl_to_checksum(
6172
%args,
6173
db => $tbl->{db},
6174
tbl => $tbl->{tbl},
6175
final_o => $final_o,
6176
);
6177
}
6178
return;
6179
}
6180
6181
# The value may be an expression like 'NOW() - INTERVAL 7 DAY'
6182
# and we should evaluate it.
6183
sub eval_expr {
6184
my ( $name, $val, $dbh ) = @_;
6185
my $result = $val;
6186
eval {
6187
($result) = $dbh->selectrow_array("SELECT $val");
6188
MKDEBUG && _d("option", $name, "evaluates to:", $result);
6189
};
6190
if ( $EVAL_ERROR && MKDEBUG ) {
6191
chomp $EVAL_ERROR;
6192
_d("Error evaluating option", $name, $EVAL_ERROR);
6193
}
6194
return $result;
6195
}
6196
6197
sub determine_checksum_strat {
6198
my ( %args ) = @_;
6199
foreach my $arg ( qw(o dbh tc) ) {
6200
die "I need a $arg argument" unless $args{$arg};
6201
}
6202
my $o = $args{o};
6203
my $dbh = $args{dbh};
6204
my $tc = $args{tc};
6205
6206
my $ret = { # return vals in easy-to-swallow hash form
6207
strat => undef,
6208
crc_type => 'varchar',
6209
crc_wid => 16,
6210
func => undef,
6211
opt_slice => undef,
6212
};
6213
6214
$ret->{strat} = $tc->best_algorithm(
6215
algorithm => $o->get('algorithm'),
6216
dbh => $dbh,
6217
where => $o->get('where') || $o->get('since'),
6218
chunk => $o->get('chunk-size'),
6219
replicate => $o->get('replicate'),
6220
count => $o->get('count'),
6221
);
6222
6223
if ( $o->get('algorithm') && $o->get('algorithm') ne $ret->{strat} ) {
6224
warn "--algorithm=".$o->get('algorithm')." can't be used; "
6225
. "falling back to $ret->{strat}\n";
6226
}
6227
6228
# If using a cryptographic hash strategy, decide what hash function to use,
6229
# and if using BIT_XOR whether and which slice to place the user variable in.
6230
if ( $tc->is_hash_algorithm( $ret->{strat} ) ) {
6231
$ret->{func} = $tc->choose_hash_func(
6232
function => $o->get('function'),
6233
dbh => $dbh,
6234
);
6235
if ( $o->get('function') && $o->get('function') ne $ret->{func} ) {
6236
warn "Checksum function ".$o->get('function')." cannot be used; "
6237
. "using $ret->{func}\n";
6238
}
6239
$ret->{crc_wid} = $tc->get_crc_wid($dbh, $ret->{func});
6240
($ret->{crc_type}) = $tc->get_crc_type($dbh, $ret->{func});
6241
6242
if ( $o->get('optimize-xor') && $ret->{strat} eq 'BIT_XOR' ) {
6243
if ( $ret->{crc_type} !~ m/int$/ ) {
6244
$ret->{opt_slice}
6245
= $tc->optimize_xor(dbh => $dbh, function => $ret->{func});
6246
if ( !defined $ret->{opt_slice} ) {
6247
warn "Cannot use --optimize-xor, disabling";
6248
$o->set('optimize-xor', 0);
6249
}
6250
}
6251
else {
6252
# FNV_64 doesn't need the optimize_xor gizmo.
6253
$o->get('optimize-xor', 0);
6254
}
6255
}
6256
}
6257
6258
return $ret;
6259
}
6260
6261
sub verify_checksum_compat {
6262
my ( %args ) = @_;
6263
foreach my $arg ( qw(o hosts) ) {
6264
die "I need a $arg argument" unless $args{$arg};
6265
}
6266
my $o = $args{o};
6267
my $hosts = $args{hosts};
6268
6269
my @verify_sums;
6270
foreach my $host ( @$hosts ) {
6271
my $dbh = get_cxn($host, %args);
6272
my $sql = "SELECT MD5(CONCAT_WS(',', '1', ''))";
6273
MKDEBUG && _d($dbh, $sql);
6274
my $cks = $dbh->selectall_arrayref($sql)->[0]->[0];
6275
push @verify_sums, {
6276
host => $host->{h},
6277
ver => $dbh->{mysql_serverinfo},
6278
sum => $cks,
6279
};
6280
}
6281
if ( unique(map { $_->{sum} } @verify_sums ) > 1 ) {
6282
my $max = max(map { length($_->{h}) } @$hosts);
6283
die "Not all servers have compatible versions. Some return different\n"
6284
. "checksum values for the same query, and cannot be compared. This\n"
6285
. "behavior changed in MySQL 4.0.14. Here is info on each host:\n\n"
6286
. join("\n",
6287
map { sprintf("%-${max}s %-32s %s", @{$_}{qw(host sum ver)}) }
6288
{ host => 'HOST', sum => 'CHECKSUM', ver => 'VERSION'},
6289
@verify_sums
6290
)
6291
. "\n\nYou can disable this check with --no-verify.\n";
6292
}
6293
return;
6294
}
6295
6296
# Check for existence and privileges on the replication table before
6297
# starting, and prepare the statements that will be used to update it.
6298
# Also clean out the checksum table. And create it if needed.
6299
# Returns fetch and update statement handles.
6798
sub ts {
6799
my ($msg) = @_;
6800
my ($s, $m, $h, $d, $M) = localtime;
6801
my $ts = sprintf('%02d-%02dT%02d:%02d:%02d', $M+1, $d, $h, $m, $s);
6802
return $msg ? "$ts $msg" : $ts;
6803
}
6804
6805
{
6806
# Completely ignore these error codes.
6807
my %ignore_code = (
6808
# Error: 1592 SQLSTATE: HY000 (ER_BINLOG_UNSAFE_STATEMENT)
6809
# Message: Statement may not be safe to log in statement format.
6810
# Ignore this warning because we have purposely set statement-based
6811
# replication.
6812
1592 => 1,
6813
);
6814
6815
# Warn once per-table for these error codes if the error message
6816
# matches the pattern.
6817
my %warn_code = (
6818
# Error: 1265 SQLSTATE: 01000 (WARN_DATA_TRUNCATED)
6819
# Message: Data truncated for column '%s' at row %ld
6820
1265 => {
6821
# any pattern
6822
# use MySQL's message for this warning
6823
},
6824
);
6825
6826
sub exec_nibble {
6827
my (%args) = @_;
6828
my @required_args = qw(Cxn tbl NibbleIterator Retry Quoter OptionParser);
6829
foreach my $arg ( @required_args ) {
6830
die "I need a $arg argument" unless $args{$arg};
6831
}
6832
my ($cxn, $tbl, $nibble_iter, $retry, $q, $o)= @args{@required_args};
6833
6834
my $dbh = $cxn->dbh();
6835
my $sth = $nibble_iter->statements();
6836
my $boundary = $nibble_iter->boundaries();
6837
my $lb_quoted = $q->serialize_list(@{$boundary->{lower}});
6838
my $ub_quoted = $q->serialize_list(@{$boundary->{upper}});
6839
my $chunk = $nibble_iter->nibble_number();
6840
my $chunk_index = $nibble_iter->nibble_index();
6841
6842
return $retry->retry(
6843
tries => $o->get('retries'),
6844
wait => sub { return; },
6845
try => sub {
6846
# ###################################################################
6847
# Start timing the checksum query.
6848
# ###################################################################
6849
my $t_start = time;
6850
6851
# Execute the REPLACE...SELECT checksum query.
6852
PTDEBUG && _d($sth->{nibble}->{Statement},
6853
'lower boundary:', @{$boundary->{lower}},
6854
'upper boundary:', @{$boundary->{upper}});
6855
$sth->{nibble}->execute(
6856
# REPLACE INTO repl_table SELECT
6857
$tbl->{db}, # db
6858
$tbl->{tbl}, # tbl
6859
$chunk, # chunk (number)
6860
$chunk_index, # chunk_index
6861
$lb_quoted, # lower_boundary
6862
$ub_quoted, # upper_boundary
6863
# this_cnt, this_crc WHERE
6864
@{$boundary->{lower}}, # upper boundary values
6865
@{$boundary->{upper}}, # lower boundary values
6866
);
6867
6868
my $t_end = time;
6869
# ###################################################################
6870
# End timing the checksum query.
6871
# ###################################################################
6872
6873
# Check if checksum query caused any warnings.
6874
my $sql_warn = 'SHOW WARNINGS';
6875
PTDEBUG && _d($sql_warn);
6876
my $warnings = $dbh->selectall_arrayref($sql_warn, { Slice => {} } );
6877
foreach my $warning ( @$warnings ) {
6878
my $code = ($warning->{code} || 0);
6879
my $message = $warning->{message};
6880
if ( $ignore_code{$code} ) {
6881
PTDEBUG && _d('Ignoring warning:', $code, $message);
6882
next;
6883
}
6884
elsif ( $warn_code{$code}
6885
&& (!$warn_code{$code}->{pattern}
6886
|| $message =~ m/$warn_code{$code}->{pattern}/) )
6887
{
6888
if ( !$tbl->{"warned_code_$code"} ) { # warn once per table
6889
if ( $o->get('quiet') < 2 ) {
6890
warn ts("Checksum query for table $tbl->{db}.$tbl->{tbl} "
6891
. "caused MySQL error $code: "
6892
. ($warn_code{$code}->{message}
6893
? $warn_code{$code}->{message}
6894
: $message)
6895
. "\n");
6896
}
6897
$tbl->{"warned_code_$code"} = 1;
6898
$tbl->{checksum_results}->{errors}++;
6899
}
6900
}
6901
else {
6902
# This die will propagate to fail which will return 0
6903
# and propagate it to final_fail which will die with
6904
# this error message. (So don't wrap it in ts().)
6905
die "Checksum query for table $tbl->{db}.$tbl->{tbl} "
6906
. "caused MySQL error $code:\n"
6907
. " Level: " . ($warning->{level} || '') . "\n"
6908
. " Code: " . ($warning->{code} || '') . "\n"
6909
. " Message: " . ($warning->{message} || '') . "\n"
6910
. " Query: " . $sth->{nibble}->{Statement} . "\n";
6911
}
6912
}
6913
6914
# Success: no warnings, no errors. Return nibble time.
6915
return $t_end - $t_start;
6916
},
6917
fail => sub {
6918
my (%args) = @_;
6919
my $error = $args{error};
6920
6921
if ( $error =~ m/Lock wait timeout exceeded/
6922
|| $error =~ m/Query execution was interrupted/
6923
) {
6924
# These errors/warnings can be retried, so don't print
6925
# a warning yet; do that in final_fail.
6926
return 1;
6927
}
6928
elsif ( $error =~ m/MySQL server has gone away/
6929
|| $error =~ m/Lost connection to MySQL server/
6930
) {
6931
# The 2nd pattern means that MySQL itself died or was stopped.
6932
# The 3rd pattern means that our cxn was killed (KILL <id>).
6933
eval { $dbh = $cxn->connect(); };
6934
return 1 unless $EVAL_ERROR; # reconnected, retry checksum query
6935
$oktorun = 0; # failed to reconnect, exit tool
6936
}
6937
6938
# At this point, either the error/warning cannot be retried,
6939
# or we failed to reconnect. So stop trying and call final_fail.
6940
return 0;
6941
},
6942
final_fail => sub {
6943
my (%args) = @_;
6944
my $error = $args{error};
6945
6946
if ( $error =~ /Lock wait timeout exceeded/
6947
|| $error =~ /Query execution was interrupted/
6948
) {
6949
# These errors/warnings are not fatal but only cause this
6950
# nibble to be skipped.
6951
if ( $o->get('quiet') < 2 ) {
6952
warn "$error\n";
6953
}
6954
return; # skip this nibble
6955
}
6956
6957
# This die will be caught by the eval inside the TABLE loop.
6958
# Checksumming for this table will stop, which is probably
6959
# good because by this point the error or warning indicates
6960
# that something fundamental is broken or wrong. Checksumming
6961
# will continue with the next table, unless the fail code set
6962
# oktorun=0, in which case the error/warning is fatal.
6963
die "Error executing checksum query: $args{error}\n";
6964
}
6965
);
6966
}
6967
}
6968
6969
{
6970
my $line_fmt = "%14s %6s %6s %8s %7s %7s %7s %-s\n";
6971
my @headers = qw(TS ERRORS DIFFS ROWS CHUNKS SKIPPED TIME TABLE);
6972
6973
sub print_checksum_results {
6974
my (%args) = @_;
6975
my @required_args = qw(tbl);
6976
foreach my $arg ( @required_args ) {
6977
die "I need a $arg argument" unless $args{$arg};
6978
}
6979
my ($tbl) = @args{@required_args};
6980
6981
if ($print_header) {
6982
printf $line_fmt, @headers;
6983
$print_header = 0;
6984
}
6985
6986
my $res = $tbl->{checksum_results};
6987
printf $line_fmt,
6988
ts(),
6989
$res->{errors} || 0,
6990
$res->{diffs} || 0,
6991
$res->{n_rows} || 0,
6992
$res->{n_chunks} || 0,
6993
$res->{skipped} || 0,
6994
sprintf('%.3f', $res->{start_time} ? time - $res->{start_time} : 0),
6995
"$tbl->{db}.$tbl->{tbl}";
6996
6997
return;
6998
}
6999
}
7000
7001
{
7002
my @headers = qw(table chunk cnt_diff crc_diff chunk_index lower_boundary upper_boundary);
7003
7004
sub print_checksum_diffs {
7005
my ( %args ) = @_;
7006
my @required_args = qw(cxn diffs);
7007
foreach my $arg ( @required_args ) {
7008
die "I need a $arg argument" unless $args{$arg};
7009
}
7010
my ($cxn, $diffs) = @args{@required_args};
7011
7012
print "Differences on ", $cxn->name(), "\n";
7013
print join(' ', map { uc $_ } @headers), "\n";
7014
foreach my $diff ( @$diffs ) {
7015
print join(' ', map { defined $_ ? $_ : '' } @{$diff}{@headers}), "\n";
7016
}
7017
print "\n";
7018
7019
return;
7020
}
7021
}
7022
6300
7023
sub check_repl_table {
6301
7024
my ( %args ) = @_;
6302
foreach my $arg ( qw(o dbh tp q) ) {
7025
my @required_args = qw(dbh repl_table OptionParser TableParser Quoter);
7026
foreach my $arg ( @required_args ) {
6303
7027
die "I need a $arg argument" unless $args{$arg};
6304
7028
}
6305
my $o = $args{o};
6306
my $dbh = $args{dbh};
6307
my $tp = $args{tp};
6308
my $q = $args{q};
6309
6310
my $replicate_table = $o->get('replicate');
6311
return unless $replicate_table;
6312
6313
use_repl_db(%args); # USE the proper replicate db
6314
6315
my ($db, $tbl) = $q->split_unquote($replicate_table);
7029
my ($dbh, $repl_table, $o, $tp, $q) = @args{@required_args};
7030
PTDEBUG && _d('Checking --replicate table', $repl_table);
7031
7032
# If the repl db doesn't exit, auto-create it, maybe.
7033
my ($db, $tbl) = $q->split_unquote($repl_table);
7034
my $sql = "SHOW DATABASES LIKE '$db'";
7035
PTDEBUG && _d($sql);
7036
my @db_exists = $dbh->selectrow_array($sql);
7037
if ( !@db_exists && $o->get('create-replicate-table') ) {
7038
$sql = "CREATE DATABASE " . $q->quote($db) . " /* pt-table-checksum */";
7039
eval {
7040
$dbh->do($sql);
7041
};
7042
if ( $EVAL_ERROR ) {
7043
die "--replicate database $db does not exist and it cannot be "
7044
. "created automatically. You need to create the database.\n";
7045
}
7046
}
7047
7048
# USE the correct db (probably the repl db, but maybe --replicate-database).
7049
use_repl_db(%args);
7050
7051
# Check if the repl table exists; if not, create it, maybe.
6316
7052
my $tbl_exists = $tp->check_table(
6317
7053
dbh => $dbh,
6318
7054
db => $db,
6321
7057
if ( !$tbl_exists ) {
6322
7058
if ( $o->get('create-replicate-table') ) {
6323
7059
create_repl_table(%args)
6324
or die "--create-replicate-table failed to create "
6325
. $replicate_table;
6326
7060
}
6327
7061
else {
6328
die "--replicate table $replicate_table does not exist; "
7062
die "--replicate table $repl_table does not exist; "
6329
7063
. "read the documentation or use --create-replicate-table "
6330
. "to create it.";
7064
. "to create it.\n";
6331
7065
}
6332
7066
}
6333
7067
else {
6334
MKDEBUG && _d('--replicate table', $replicate_table, 'already exists');
7068
PTDEBUG && _d('--replicate table', $repl_table, 'already exists');
6335
7069
# Check it again but this time check the privs.
6336
7070
my $have_tbl_privs = $tp->check_table(
6337
7071
dbh => $dbh,
6339
7073
tbl => $tbl,
6340
7074
all_privs => 1,
6341
7075
);
6342
die "User does not have all necessary privileges on $replicate_table"
6343
unless $have_tbl_privs;
6344
}
6345
6346
# Clean out the replicate table globally.
6347
if ( $o->get('empty-replicate-table') ) {
6348
my $del_sql = "DELETE FROM $replicate_table";
6349
MKDEBUG && _d($dbh, $del_sql);
6350
$dbh->do($del_sql);
6351
}
6352
6353
my $fetch_sth = $dbh->prepare(
6354
"SELECT this_crc, this_cnt FROM $replicate_table "
6355
. "WHERE db = ? AND tbl = ? AND chunk = ?");
6356
my $update_sth = $dbh->prepare(
6357
"UPDATE $replicate_table SET master_crc = ?, master_cnt = ? "
6358
. "WHERE db = ? AND tbl = ? AND chunk = ?");
6359
6360
return ($fetch_sth, $update_sth);
7076
die "User does not have all privileges on --replicate table "
7077
. "$repl_table.\n" unless $have_tbl_privs;
7078
}
7079
7080
return; # success, repl table is ready to go
6361
7081
}
6362
7082
6363
# This sub should be called before any work is done with the
6364
# --replicate table. It will USE the correct replicate db.
6365
# If there's a tbl arg then its db will be used unless --replicate-database
6366
# was specified. A tbl arg means we're checksumming that table,
6367
# so we've been called from do_tbl_replicate(). Other callers
6368
# won't pass a tbl arg because they're just doing something to
6369
# the --replicate table.
6370
# See http://code.google.com/p/maatkit/issues/detail?id=982
7083
# Sub: use_repl_db
7084
# USE the correct database for the --replicate table.
7085
# This sub must be called before any work is done with the --replicatte
7086
# table because replication filters can really complicate replicating the
7087
# checksums. The originally issue is,
7088
# http://code.google.com/p/maatkit/issues/detail?id=982,
7089
# but here's what you need to know:
7090
# - If there is no active DB, then if there's any do-db or ignore-db
7091
# settings, the checksums will get filtered out of replication. So we
7092
# have to have some DB be the current one.
7093
# - Other places in the code may change the DB and we might not know it.
7094
# Opportunity for bugs. The SHOW CREATE TABLE, for example. In the
7095
# end, a bunch of USE statements isn't a big deal, it just looks noisy
7096
# when you analyze the logs this tool creates. But it's better to just
7097
# have them even if they're no-op.
7098
# - We need to always let the user specify, because there are so many
7099
# possibilities that the tool can't guess the right thing in all of
7100
# them.
7101
# - The right default behavior, which the user can override, is:
7102
# * When running queries on the --replicate table itself, such as
7103
# emptying it, USE that table's database.
7104
# * When running checksum queries, USE the database of the table that's
7105
# being checksummed.
7106
# * When the user specifies --replicate-database, in contrast, always
7107
# USE that database.
7108
# - This behavior is the best compromise by default, because users who
7109
# explicitly replicate some databases and filter out others will be
7110
# very likely to run pt-table-checksum and limit its checksumming to
7111
# only the databases that are replicated. I've seen people do this,
7112
# including Peter. In this case, the tool will work okay even without
7113
# an explicit --replicate-database setting.
7114
#
7115
# Required Arguments:
7116
# dbh - dbh
7117
# repl_table - Full quoted --replicate table name
7118
# OptionParser - <OptionParser>
7119
# Quoter - <Quoter>
7120
#
7121
# Optional Arguments:
7122
# tbl - Standard tbl hashref of table being checksummed
7123
#
7124
# Returns:
7125
# Nothing or dies on error
6371
7126
sub use_repl_db {
6372
7127
my ( %args ) = @_;
6373
my @required_args = qw(dbh o q);
7128
my @required_args = qw(dbh repl_table OptionParser Quoter);
6374
7129
foreach my $arg ( @required_args ) {
6375
7130
die "I need a $arg argument" unless $args{$arg};
6376
7131
}
6377
my ($dbh, $o, $q) = @args{@required_args};
6378
6379
my $replicate_table = $o->get('replicate');
6380
return unless $replicate_table;
6381
6382
# db and tbl from --replicate
6383
my ($db, $tbl) = $q->split_unquote($replicate_table);
6384
7132
my ($dbh, $repl_table, $o, $q) = @args{@required_args};
7133
PTDEBUG && _d('use_repl_db');
7134
7135
my ($db, $tbl) = $q->split_unquote($repl_table);
6385
7136
if ( my $tbl = $args{tbl} ) {
6386
# Caller is checksumming this table, USE its db unless
6387
# --replicate-database is in effect.
7137
# If there's a tbl arg then its db will be used unless
7138
# --replicate-database was specified. A tbl arg means
7139
# we're checksumming that table. Other callers won't
7140
# pass a tbl arg when they're just doing something to
7141
# the --replicate table.
6388
7142
$db = $o->get('replicate-database') ? $o->get('replicate-database')
6389
: $tbl->{database};
7143
: $tbl->{db};
6390
7144
}
6391
7145
else {
6392
7146
# Caller is doing something just to the --replicate table.
6397
7151
6398
7152
eval {
6399
7153
my $sql = "USE " . $q->quote($db);
6400
MKDEBUG && _d($dbh, $sql);
7154
PTDEBUG && _d($sql);
6401
7155
$dbh->do($sql);
6402
7156
};
6403
7157
if ( $EVAL_ERROR ) {
6405
7159
my $opt = $o->get('replicate-database') ? "--replicate-database"
6406
7160
: "--replicate database";
6407
7161
if ( $EVAL_ERROR =~ m/unknown database/i ) {
6408
die "$opt `$db` does not exist: $EVAL_ERROR";
7162
die "$opt $db does not exist. You need to create the "
7163
. "database or specify a database for $opt that exists.\n";
6409
7164
}
6410
7165
else {
6411
die "Error using $opt `$db`: $EVAL_ERROR";
7166
die "Error using $opt $db: $EVAL_ERROR\n";
6412
7167
}
6413
7168
}
6414
7169
6415
7170
return;
6416
7171
}
6417
7172
6418
# Returns 1 on successful creation of the replicate table,
6419
# or 0 on failure.
6420
7173
sub create_repl_table {
6421
7174
my ( %args ) = @_;
6422
foreach my $arg ( qw(o dbh) ) {
7175
my @required_args = qw(dbh repl_table OptionParser);
7176
foreach my $arg ( @required_args ) {
6423
7177
die "I need a $arg argument" unless $args{$arg};
6424
7178
}
6425
my $o = $args{o};
6426
my $dbh = $args{dbh};
6427
6428
my $replicate_table = $o->get('replicate');
6429
6430
my $sql = $o->read_para_after(
6431
__FILE__, qr/MAGIC_create_replicate/);
6432
$sql =~ s/CREATE TABLE checksum/CREATE TABLE $replicate_table/;
7179
my ($dbh, $repl_table, $o) = @args{@required_args};
7180
PTDEBUG && _d('Creating --replicate table', $repl_table);
7181
my $sql = $o->read_para_after(__FILE__, qr/MAGIC_create_replicate/);
7182
$sql =~ s/CREATE TABLE checksums/CREATE TABLE $repl_table/;
6433
7183
$sql =~ s/;$//;
6434
MKDEBUG && _d($dbh, $sql);
7184
PTDEBUG && _d($dbh, $sql);
6435
7185
eval {
6436
7186
$dbh->do($sql);
6437
7187
};
6438
7188
if ( $EVAL_ERROR ) {
6439
MKDEBUG && _d('--create-replicate-table failed:', $EVAL_ERROR);
6440
return 0;
6441
}
6442
6443
return 1;
6444
}
6445
6446
sub read_repl_table {
6447
my ( %args ) = @_;
6448
foreach my $arg ( qw(o dbh host) ) {
6449
die "I need a $arg argument" unless $args{$arg};
6450
}
6451
my $o = $args{o};
6452
my $dbh = $args{dbh};
6453
my $host = $args{host};
6454
6455
my $replicate_table = $o->get('replicate');
6456
die "Cannot read replicate table because --replicate was not specified"
6457
unless $replicate_table;
6458
6459
# Read checksums from replicate table.
6460
my $already_checksummed;
6461
my $checksums
6462
= $dbh->selectall_arrayref("SELECT db, tbl, chunk FROM $replicate_table");
6463
6464
# Save each finished checksum.
6465
foreach my $checksum ( @$checksums ) {
6466
my ( $db, $tbl, $chunk ) = @$checksum[0..2];
6467
$already_checksummed->{$db}->{$tbl}->{$chunk}->{$host} = 1;
6468
}
6469
6470
return $already_checksummed;
6471
}
6472
6473
sub parse_resume_file {
6474
my ( $resume_file ) = @_;
6475
6476
open my $resume_fh, '<', $resume_file
6477
or die "Cannot open resume file $resume_file: $OS_ERROR";
6478
6479
# The resume file, being the output from a previous run, should
6480
# have the columns DATABASE TABLE CHUNK HOST ... (in that order).
6481
# We only need those first 4 columns. We re-print every line of
6482
# the resume file so the end result will be the whole, finished
6483
# output: what the previous run got done plus what we are about
6484
# to resume and finish.
6485
my $already_checksummed;
6486
while ( my $line = <$resume_fh> ) {
6487
# Re-print every line.
6488
print $line;
6489
6490
# If the line is a checksum line, parse from it the db, tbl,
6491
# checksum and host.
6492
if ( $line =~ m/^\S+\s+\S+\s+\d+\s+/ ) {
6493
my ( $db, $tbl, $chunk, $host ) = $line =~ m/(\S+)/g;
6494
$already_checksummed->{$db}->{$tbl}->{$chunk}->{$host} = 1;
6495
}
6496
}
6497
6498
close $resume_fh;
6499
MKDEBUG && _d("Already checksummed:", Dumper($already_checksummed));
6500
6501
return $already_checksummed;
6502
}
6503
6504
sub already_checksummed {
6505
my ( $d, $t, $c, $h ) = @_; # db, tbl, chunk num, host
6506
if ( exists $already_checksummed->{$d}->{$t}->{$c}->{$h} ) {
6507
MKDEBUG && _d("Skipping chunk because of --resume:", $d, $t, $c, $h);
6508
return 1;
6509
}
6510
return 0;
6511
}
6512
6513
sub do_tbl_replicate {
6514
my ( $chunk_num, %args ) = @_;
6515
foreach my $arg ( qw(q host query tbl hdr explain final_o ch retry) ) {
6516
die "I need a $arg argument" unless $args{$arg};
6517
}
6518
my $ch = $args{ch};
6519
my $final_o = $args{final_o};
6520
my $q = $args{q};
6521
my $host = $args{host};
6522
my $hdr = $args{hdr};
6523
my $explain = $args{explain};
6524
my $tbl = $args{tbl};
6525
my $retry = $args{retry};
6526
6527
MKDEBUG && _d('Replicating chunk', $chunk_num,
6528
'of table', $tbl->{database}, '.', $tbl->{table},
6529
'on', $host->{h}, ':', $host->{P});
6530
6531
my $dbh = $host->{dbh};
6532
my $sql;
6533
6534
use_repl_db(%args); # USE the proper replicate db
6535
6536
my $cnt = 'NULL';
6537
my $crc = 'NULL';
6538
my $beg = time();
6539
$sql = $ch->inject_chunks(
6540
query => $args{query},
6541
database => $tbl->{database},
6542
table => $tbl->{table},
6543
chunks => $tbl->{chunks},
6544
chunk_num => $chunk_num,
6545
where => [$final_o->get('where'), $tbl->{since}],
6546
index_hint => $tbl->{index},
6547
);
6548
6549
if ( MKDEBUG && $chunk_num == 0 ) {
6550
_d("SQL for inject chunk 0:", $sql);
6551
}
6552
6553
my $where = $tbl->{chunks}->[$chunk_num];
6554
if ( $final_o->get('explain') ) {
6555
if ( $chunk_num == 0 ) {
6556
printf($explain, @{$tbl}{qw(database table)}, $sql)
6557
or die "Cannot print: $OS_ERROR";
6558
}
6559
printf($explain, @{$tbl}{qw(database table)}, $where)
6560
or die "Cannot print: $OS_ERROR";
6561
return;
6562
}
6563
6564
# Actually run the checksum query
6565
$retry->retry(
6566
tries => 2,
6567
wait => sub { return; },
6568
retry_on_die => 1,
6569
try => sub {
6570
$dbh->do('SET @crc := "", @cnt := 0 /*!50108 , '
6571
. '@@binlog_format := "STATEMENT"*/');
6572
$dbh->do($sql, {}, @{$tbl}{qw(database table)}, $where);
6573
return 1;
6574
},
6575
on_failure => sub {
6576
die $EVAL_ERROR; # caught in checksum_tables()
6577
},
6578
);
6579
6580
# Catch any warnings thrown....
6581
my $sql_warn = 'SHOW WARNINGS';
6582
MKDEBUG && _d($sql_warn);
6583
my $warnings = $dbh->selectall_arrayref($sql_warn, { Slice => {} } );
6584
foreach my $warning ( @$warnings ) {
6585
if ( $warning->{message} =~ m/Data truncated for column 'boundaries'/ ) {
6586
_d("Warning: WHERE clause too large for boundaries column; ",
6587
"pt-table-sync may fail; value:", $where);
6588
}
6589
elsif ( ($warning->{code} || 0) == 1592 ) {
6590
# Error: 1592 SQLSTATE: HY000 (ER_BINLOG_UNSAFE_STATEMENT)
6591
# Message: Statement may not be safe to log in statement format.
6592
# Ignore this warning because we have purposely set statement-based
6593
# replication.
6594
MKDEBUG && _d('Ignoring warning:', $warning->{message});
6595
}
6596
else {
6597
# die doesn't permit extra line breaks so warn then die.
6598
warn "\nChecksum query caused a warning:\n"
6599
. join("\n",
6600
map { "\t$_: " . $warning->{$_} || '' } qw(level code message)
6601
)
6602
. "\n\tquery: $sql\n\n";
6603
die;
6604
}
6605
}
6606
6607
# Update the master_crc etc columns
6608
$fetch_sth->execute(@{$tbl}{qw(database table)}, $chunk_num);
6609
( $crc, $cnt ) = $fetch_sth->fetchrow_array();
6610
$update_sth->execute($crc, $cnt, @{$tbl}{qw(database table)}, $chunk_num);
6611
6612
my $end = time();
6613
$crc ||= 'NULL';
6614
if ( !$final_o->get('quiet') && !$final_o->get('explain') ) {
6615
if ( $final_o->get('checksum') ) {
6616
printf($md5sum_fmt, $crc, $host->{h},
6617
@{$tbl}{qw(database table)}, $chunk_num)
6618
or die "Cannot print: $OS_ERROR";
6619
}
6620
else {
6621
printf($hdr,
6622
@{$tbl}{qw(database table)}, $chunk_num,
6623
$host->{h}, $tbl->{struct}->{engine}, $cnt, $crc,
6624
$end - $beg, 'NULL', 'NULL', 'NULL')
6625
or die "Cannot print: $OS_ERROR";
6626
}
6627
}
6628
6629
return;
6630
}
6631
6632
sub do_tbl {
6633
my ( $chunk_num, $is_master, %args ) = @_;
6634
foreach my $arg ( qw(du final_o ms q tc dbh host tbl hdr explain strat) ) {
6635
die "I need a $arg argument" unless $args{$arg};
6636
}
6637
my $du = $args{du};
6638
my $final_o = $args{final_o};
6639
my $ms = $args{ms};
6640
my $tc = $args{tc};
6641
my $tp = $args{tp};
6642
my $q = $args{q};
6643
my $host = $args{host};
6644
my $tbl = $args{tbl};
6645
my $explain = $args{explain};
6646
my $hdr = $args{hdr};
6647
my $strat = $args{strat};
6648
6649
MKDEBUG && _d('Checksumming chunk', $chunk_num,
6650
'of table', $tbl->{database}, '.', $tbl->{table},
6651
'on', $host->{h}, ':', $host->{P},
6652
'using algorithm', $strat);
6653
6654
my $dbh = $host->{dbh};
6655
$dbh->do("USE " . $q->quote($tbl->{database}));
6656
6657
my $cnt = 'NULL';
6658
my $crc = 'NULL';
6659
my $sta = 'NULL';
6660
my $lag = 'NULL';
6661
6662
# Begin timing the checksum operation.
6663
my $beg = time();
6664
6665
# I'm a slave. Wait to catch up to the master. Calculate slave lag.
6666
if ( !$is_master && !$final_o->get('explain') ) {
6667
if ( $final_o->get('wait') ) {
6668
MKDEBUG && _d('Waiting to catch up to master for --wait');
6669
my $result = $ms->wait_for_master(
6670
master_status => $tbl->{master_status},
6671
slave_dbh => $dbh,
6672
timeout => $final_o->get('wait'),
6673
);
6674
$sta = $result && defined $result->{result}
6675
? $result->{result}
6676
: 'NULL';
6677
}
6678
6679
if ( $final_o->get('slave-lag') ) {
6680
MKDEBUG && _d('Getting slave lag for --slave-lag');
6681
my $res = $ms->get_slave_status($dbh);
6682
$lag = $res && defined $res->{seconds_behind_master}
6683
? $res->{seconds_behind_master}
6684
: 'NULL';
6685
}
6686
}
6687
6688
# Time the checksum operation and the wait-for-master operation separately.
6689
my $mid = time();
6690
6691
# Check that table exists on slave.
6692
my $have_table = 1;
6693
if ( !$is_master || !$checksum_table_data ) {
6694
$have_table = $tp->check_table(
6695
dbh => $dbh,
6696
db => $tbl->{database},
6697
tbl => $tbl->{table},
6698
);
6699
warn "$tbl->{database}.$tbl->{table} does not exist on slave"
6700
. ($host->{h} ? " $host->{h}" : '')
6701
. ($host->{P} ? ":$host->{P}" : '')
6702
unless $have_table;
6703
}
6704
6705
if ( $have_table ) {
6706
# Do the checksum operation.
6707
if ( $checksum_table_data ) {
6708
if ( $strat eq 'CHECKSUM' ) {
6709
if ( $final_o->get('crc') ) {
6710
$crc = do_checksum(%args);
6711
}
6712
if ( $final_o->get('count') ) {
6713
$cnt = do_count($chunk_num, %args);
6714
}
6715
}
6716
elsif ( $final_o->get('crc') ) {
6717
( $cnt, $crc ) = do_var_crc($chunk_num, %args);
6718
$crc ||= 'NULL';
6719
}
6720
else {
6721
$cnt = do_count($chunk_num, %args);
6722
}
6723
}
6724
else { # Checksum SHOW CREATE TABLE for --schema.
6725
my $create
6726
= $du->get_create_table($dbh, $q, $tbl->{database}, $tbl->{table});
6727
$create = $create->[1];
6728
$create = $tp->remove_auto_increment($create);
6729
$crc = $tc->crc32($create);
6730
}
6731
}
6732
6733
my $end = time();
6734
6735
if ( !$final_o->get('quiet') && !$final_o->get('explain') ) {
6736
if ( $final_o->get('checksum') ) {
6737
printf($md5sum_fmt, $crc, $host->{h},
6738
@{$tbl}{qw(database table)}, $chunk_num)
6739
or die "Cannot print: $OS_ERROR";
6740
}
6741
else {
6742
printf($hdr,
6743
@{$tbl}{qw(database table)}, $chunk_num,
6744
$host->{h}, $tbl->{struct}->{engine}, $cnt, $crc,
6745
$end - $mid, $mid - $beg, $sta, $lag)
6746
or die "Cannot print: $OS_ERROR";
6747
}
6748
}
6749
6750
return;
6751
}
6752
6753
sub get_cxn {
6754
my ( $dsn, %args ) = @_;
6755
foreach my $arg ( qw(o dp) ) {
6756
die "I need a $arg argument" unless $args{$arg};
6757
}
6758
my $dp = $args{dp};
6759
my $o = $args{o};
6760
6761
if ( $o->get('ask-pass') && !defined $dsn->{p} ) {
6762
$dsn->{p} = OptionParser::prompt_noecho("Enter password for $dsn->{h}: ");
6763
}
6764
6765
my $ac = $o->get('lock') ? 0 : 1;
6766
my $dbh = $dp->get_dbh(
6767
$dp->get_cxn_params($dsn), { AutoCommit => $ac });
6768
$dp->fill_in_dsn($dbh, $dsn);
6769
$dbh->{InactiveDestroy} = 1; # Prevent destroying on fork.
6770
$dbh->{FetchHashKeyName} = 'NAME_lc';
6771
return $dbh;
6772
}
6773
6774
sub do_var_crc {
6775
my ( $chunk_num, %args ) = @_;
6776
foreach my $arg ( qw(ch dbh query tbl explain final_o) ) {
6777
die "I need a $arg argument" unless $args{$arg};
6778
}
6779
my $final_o = $args{final_o};
6780
my $ch = $args{ch};
6781
my $tbl = $args{tbl};
6782
my $explain = $args{explain};
6783
my $dbh = $args{dbh};
6784
6785
MKDEBUG && _d("do_var_crc for", $tbl->{table});
6786
6787
my $sql = $ch->inject_chunks(
6788
query => $args{query},
6789
database => $tbl->{database},
6790
table => $tbl->{table},
6791
chunks => $tbl->{chunks},
6792
chunk_num => $chunk_num,
6793
where => [$final_o->get('where'), $tbl->{since}],
6794
index_hint => $tbl->{index},
6795
);
6796
6797
if ( MKDEBUG && $chunk_num == 0 ) {
6798
_d("SQL for chunk 0:", $sql);
6799
}
6800
6801
if ( $final_o->get('explain') ) {
6802
if ( $chunk_num == 0 ) {
6803
printf($explain, @{$tbl}{qw(database table)}, $sql)
6804
or die "Cannot print: $OS_ERROR";
6805
}
6806
printf($explain, @{$tbl}{qw(database table)},$tbl->{chunks}->[$chunk_num])
6807
or die "Cannot print: $OS_ERROR";
6808
return;
6809
}
6810
6811
$dbh->do('set @crc := "", @cnt := 0');
6812
my $res = $dbh->selectall_arrayref($sql, { Slice => {} })->[0];
6813
return ($res->{cnt}, $res->{crc});
6814
}
6815
6816
sub do_checksum {
6817
my ( %args ) = @_;
6818
foreach my $arg ( qw(dbh query tbl explain final_o) ) {
6819
die "I need a $arg argument" unless $args{$arg};
6820
}
6821
my $dbh = $args{dbh};
6822
my $final_o = $args{final_o};
6823
my $tbl = $args{tbl};
6824
my $query = $args{query};
6825
my $explain = $args{explain};
6826
6827
MKDEBUG && _d("do_checksum for", $tbl->{table});
6828
6829
if ( $final_o->get('explain') ) {
6830
printf($explain, @{$tbl}{qw(database table)}, $query)
6831
or die "Cannot print: $OS_ERROR";
6832
}
6833
else {
6834
my $res = $dbh->selectrow_hashref($query);
6835
if ( $res ) {
6836
my ($key) = grep { m/checksum/i } keys %$res;
6837
return defined $res->{$key} ? $res->{$key} : 'NULL';
6838
}
6839
}
6840
6841
return;
6842
}
6843
6844
sub do_count {
6845
my ( $chunk_num, %args ) = @_;
6846
foreach my $arg ( qw(q dbh tbl explain final_o) ) {
6847
die "I need a $arg argument" unless $args{$arg};
6848
}
6849
my $final_o = $args{final_o};
6850
my $tbl = $args{tbl};
6851
my $explain = $args{explain};
6852
my $dbh = $args{dbh};
6853
my $q = $args{q};
6854
6855
MKDEBUG && _d("do_count for", $tbl->{table});
6856
6857
my $sql = "SELECT COUNT(*) FROM "
6858
. $q->quote(@{$tbl}{qw(database table)});
6859
if ( $final_o->get('where') || $final_o->get('since') ) {
6860
my $where_since = ($final_o->get('where'), $final_o->get('since'));
6861
$sql .= " WHERE ("
6862
. join(" AND ", map { "($_)" } grep { $_ } @$where_since )
6863
. ")";
6864
}
6865
if ( $final_o->get('explain') ) {
6866
printf($explain, @{$tbl}{qw(database table)}, $sql)
6867
or die "Cannot print: $OS_ERROR";
6868
}
6869
else {
6870
return $dbh->selectall_arrayref($sql)->[0]->[0];
6871
}
6872
6873
return;
6874
}
6875
6876
sub unique {
6877
my %seen;
6878
grep { !$seen{$_}++ } @_;
6879
}
6880
6881
# Tries to extract the MySQL error message and print it
6882
sub print_err {
6883
my ( $o, $msg, $db, $tbl, $host ) = @_;
6884
return if !defined $msg
6885
# Honor --quiet in the (common?) event of dropped tables or deadlocks
6886
or ($o->get('quiet')
6887
&& $EVAL_ERROR =~ m/: Table .*? doesn't exist|Deadlock found/);
6888
$msg =~ s/^.*?failed: (.*?) at \S+ line (\d+).*$/$1 at line $2/s;
6889
$msg =~ s/\s+/ /g;
6890
if ( $db && $tbl ) {
6891
$msg .= " while doing $db.$tbl";
6892
}
6893
if ( $host ) {
6894
$msg .= " on $host";
6895
}
6896
print STDERR $msg, "\n";
6897
}
6898
6899
# Returns when Seconds_Behind_Master on all the given slaves
6900
# is < max_lag, waits check_interval seconds between checks
6901
# if a slave is lagging too much.
6902
sub wait_for_slaves {
6903
my ( %args ) = @_;
6904
my $slaves = $args{slaves};
6905
my $max_lag = $args{max_lag};
6906
my $check_interval = $args{check_interval};
6907
my $dp = $args{DSNParser};
6908
my $ms = $args{MasterSlave};
6909
my $pr = $args{progress};
6910
6911
return unless scalar @$slaves;
6912
my $n_slaves = @$slaves;
6913
6914
my $pr_callback;
6915
if ( $pr ) {
6916
# If you use the default Progress report callback, you'll need to
6917
# to add Transformers.pm to this tool.
6918
my $reported = 0;
6919
$pr_callback = sub {
6920
my ($fraction, $elapsed, $remaining, $eta, $slave_no) = @_;
6921
if ( !$reported ) {
6922
print STDERR "Waiting for slave(s) to catchup...\n";
6923
$reported = 1;
6924
}
6925
else {
6926
print STDERR "Still waiting ($elapsed seconds)...\n";
6927
}
6928
return;
6929
};
6930
$pr->set_callback($pr_callback);
6931
}
6932
6933
for my $slave_no ( 0..($n_slaves-1) ) {
6934
my $slave = $slaves->[$slave_no];
6935
MKDEBUG && _d('Checking slave', $dp->as_string($slave->{dsn}),
6936
'lag for throttle');
6937
my $lag = $ms->get_slave_lag($slave->{dbh});
6938
while ( !defined $lag || $lag > $max_lag ) {
6939
MKDEBUG && _d('Slave lag', $lag, '>', $max_lag,
6940
'; sleeping', $check_interval);
6941
6942
# Report what we're waiting for before we wait.
6943
$pr->update(sub { return $slave_no; }) if $pr;
6944
6945
sleep $check_interval;
6946
$lag = $ms->get_slave_lag($slave->{dbh});
6947
}
6948
MKDEBUG && _d('Slave ready, lag', $lag, '<=', $max_lag);
6949
}
6950
6951
return;
6952
}
6953
6954
# Sub: is_oversize_chunk
6955
# Determine if the chunk is oversize.
6956
#
6957
# Parameters:
6958
# %args - Arguments
6959
#
6960
# Required Arguments:
6961
# * dbh - dbh
6962
# * db - db name, not quoted
6963
# * tbl - tbl name, not quoted
6964
# * chunk_size - chunk size in number of rows
6965
# * chunk - chunk, e.g. "`a` > 10"
6966
# * limit - oversize if rows > factor * chunk_size
6967
# * Quoter - <Quoter> object
6968
#
6969
# Optional Arguments:
6970
# * where - Arrayref of WHERE clauses added to chunk
6971
# * index_hint - FORCE INDEX clause
6972
#
6973
# Returns:
6974
# True if EXPLAIN rows is >= chunk_size * limit, else false
6975
sub is_oversize_chunk {
6976
my ( %args ) = @_;
6977
my @required_args = qw(dbh db tbl chunk_size chunk limit Quoter);
6978
foreach my $arg ( @required_args ) {
6979
die "I need a $arg argument" unless $args{$arg};
6980
}
6981
6982
my $where = [$args{chunk}, $args{where} ? @{$args{where}} : ()];
6983
my $expl;
6984
eval {
6985
$expl = _explain(%args, where => $where);
6986
};
6987
if ( $EVAL_ERROR ) {
6988
# This shouldn't happen in production but happens in testing because
6989
# we chunk tables that don't actually exist.
6990
MKDEBUG && _d("Failed to EXPLAIN chunk:", $EVAL_ERROR);
6991
return $args{chunk};
6992
}
6993
MKDEBUG && _d("Chunk", $args{chunk}, "covers", ($expl->{rows} || 0), "rows");
6994
6995
return ($expl->{rows} || 0) >= $args{chunk_size} * $args{limit} ? 1 : 0;
6996
}
6997
6998
# Sub: is_chunkable_table
6999
# Determine if the table is chunkable.
7000
#
7001
# Parameters:
7002
# %args - Arguments
7003
#
7004
# Required Arguments:
7005
# * dbh - dbh
7006
# * db - db name, not quoted
7007
# * tbl - tbl name, not quoted
7008
# * chunk_size - chunk size in number of rows
7009
# * Quoter - <Quoter> object
7010
#
7011
# Optional Arguments:
7012
# * where - Arrayref of WHERE clauses added to chunk
7013
# * index_hint - FORCE INDEX clause
7014
#
7015
# Returns:
7016
# True if EXPLAIN rows is <= chunk_size, else false
7017
sub is_chunkable_table {
7018
my ( %args ) = @_;
7019
my @required_args = qw(dbh db tbl chunk_size Quoter);
7020
foreach my $arg ( @required_args ) {
7021
die "I need a $arg argument" unless $args{$arg};
7022
}
7023
7024
my $expl;
7025
eval {
7026
$expl = _explain(%args);
7027
};
7028
if ( $EVAL_ERROR ) {
7029
# This shouldn't happen in production but happens in testing because
7030
# we chunk tables that don't actually exist.
7031
MKDEBUG && _d("Failed to EXPLAIN table:", $EVAL_ERROR);
7032
return; # errr on the side of caution: not chunkable if not explainable
7033
}
7034
MKDEBUG && _d("Table has", ($expl->{rows} || 0), "rows");
7035
7036
return ($expl->{rows} || 0) <= $args{chunk_size} ? 1 : 0;
7037
}
7038
7039
# Sub: _explain
7040
# EXPLAIN a chunk or table.
7041
#
7042
# Parameters:
7043
# %args - Arguments
7044
#
7045
# Required Arguments:
7046
# * dbh - dbh
7047
# * db - db name, not quoted
7048
# * tbl - tbl name, not quoted
7049
# * Quoter - <Quoter> object
7050
#
7051
# Optional Arguments:
7052
# * where - Arrayref of WHERE clauses added to chunk
7053
# * index_hint - FORCE INDEX clause
7054
#
7055
# Returns:
7056
# Hashref of first EXPLAIN row
7057
sub _explain {
7058
my ( %args ) = @_;
7059
my @required_args = qw(dbh db tbl Quoter);
7060
foreach my $arg ( @required_args ) {
7061
die "I need a $arg argument" unless $args{$arg};
7062
}
7063
my ($dbh, $db, $tbl, $q) = @args{@required_args};
7064
7065
my $db_tbl = $q->quote($db, $tbl);
7066
my $where;
7067
if ( $args{where} && @{$args{where}} ) {
7068
$where = join(" AND ", map { "($_)" } grep { defined } @{$args{where}});
7069
}
7070
my $sql = "EXPLAIN SELECT * FROM $db_tbl"
7071
. ($args{index_hint} ? " $args{index_hint}" : "")
7072
. ($args{where} ? " WHERE $where" : "");
7073
MKDEBUG && _d($dbh, $sql);
7074
7075
my $expl = $dbh->selectrow_hashref($sql);
7189
die ts("--create-replicate-table failed: $EVAL_ERROR");
7190
}
7191
7192
return;
7193
}
7194
7195
# Sub: explain_statement
7196
# EXPLAIN a statement.
7197
#
7198
# Required Arguments:
7199
# * tbl - Standard tbl hashref
7200
# * sth - Sth with EXLAIN <statement>
7201
# * vals - Values for sth, if any
7202
#
7203
# Returns:
7204
# Hashref with EXPLAIN plan
7205
sub explain_statement {
7206
my ( %args ) = @_;
7207
my @required_args = qw(tbl sth vals);
7208
foreach my $arg ( @required_args ) {
7209
die "I need a $arg argument" unless defined $args{$arg};
7210
}
7211
my ($tbl, $sth, $vals) = @args{@required_args};
7212
7213
my $expl;
7214
eval {
7215
PTDEBUG && _d($sth->{Statement}, 'params:', @$vals);
7216
$sth->execute(@$vals);
7217
$expl = $sth->fetchrow_hashref();
7218
$sth->finish();
7219
};
7220
if ( $EVAL_ERROR ) {
7221
# This shouldn't happen.
7222
warn ts("Error executing " . $sth->{Statement} . ": $EVAL_ERROR\n");
7223
$tbl->{checksum_results}->{errors}++;
7224
}
7225
PTDEBUG && _d('EXPLAIN plan:', Dumper($expl));
7076
7226
return $expl;
7077
7227
}
7078
7228
7229
sub last_chunk {
7230
my (%args) = @_;
7231
my @required_args = qw(dbh repl_table);
7232
foreach my $arg ( @required_args ) {
7233
die "I need a $arg argument" unless $args{$arg};
7234
}
7235
my ($dbh, $repl_table, $q) = @args{@required_args};
7236
PTDEBUG && _d('Getting last chunk for --resume');
7237
7238
my $sql = "SELECT * FROM $repl_table FORCE INDEX (ts_db_tbl) "
7239
. "WHERE master_cnt IS NOT NULL "
7240
. "ORDER BY ts DESC, db DESC, tbl DESC LIMIT 1";
7241
PTDEBUG && _d($sql);
7242
my $sth = $dbh->prepare($sql);
7243
$sth->execute();
7244
my $last_chunk = $sth->fetchrow_hashref();
7245
$sth->finish();
7246
PTDEBUG && _d('Last chunk:', Dumper($last_chunk));
7247
7248
if ( !$last_chunk || !$last_chunk->{ts} ) {
7249
PTDEBUG && _d('Replicate table is empty; will not resume');
7250
return;
7251
}
7252
7253
return $last_chunk;
7254
}
7255
7256
sub have_more_chunks {
7257
my (%args) = @_;
7258
my @required_args = qw(tbl last_chunk NibbleIterator);
7259
foreach my $arg ( @required_args ) {
7260
die "I need a $arg argument" unless $args{$arg};
7261
}
7262
my ($tbl, $last_chunk, $nibble_iter) = @args{@required_args};
7263
PTDEBUG && _d('Checking for more chunks beyond last chunk');
7264
7265
# If there's no next lower boundary, then this is the last
7266
# chunk of the table.
7267
if ( !$nibble_iter->more_boundaries() ) {
7268
PTDEBUG && _d('No more boundaries');
7269
return 0;
7270
}
7271
7272
# The previous chunk index must match the current chunk index,
7273
# else we don't know what to do.
7274
my $chunk_index = lc($nibble_iter->nibble_index() || '');
7275
if (lc($last_chunk->{chunk_index} || '') ne $chunk_index) {
7276
warn ts("Cannot resume from table $tbl->{db}.$tbl->{tbl} chunk "
7277
. "$last_chunk->{chunk} because the chunk indexes are different: "
7278
. ($last_chunk->{chunk_index} ? $last_chunk->{chunk_index}
7279
: "no index")
7280
. " was used originally but "
7281
. ($chunk_index ? $chunk_index : "no index")
7282
. " is used now. If the table has not changed significantly, "
7283
. "this may be caused by running the tool with different command "
7284
. "line options. This table will be skipped and checksumming "
7285
. "will resume with the next table.\n");
7286
$tbl->{checksum_results}->{errors}++;
7287
return 0;
7288
}
7289
7290
return 1; # more chunks
7291
}
7292
7293
sub wait_for_last_checksum {
7294
my (%args) = @_;
7295
my @required_args = qw(tbl repl_table slaves max_chunk OptionParser);
7296
foreach my $arg ( @required_args ) {
7297
die "I need a $arg argument" unless defined $args{$arg};
7298
}
7299
my ($tbl, $repl_table, $slaves, $max_chunk, $o) = @args{@required_args};
7300
my $check_pr = $args{check_pr};
7301
7302
# Requiring "AND master_crc IS NOT NULL" avoids a race condition
7303
# when the system is fast but replication is slow. In such cases,
7304
# we can select on the slave before the update for $update_sth
7305
# replicates; this causes a false-positive diff.
7306
my $sql = "SELECT MAX(chunk) FROM $repl_table "
7307
. "WHERE db='$tbl->{db}' AND tbl='$tbl->{tbl}' "
7308
. "AND master_crc IS NOT NULL";
7309
PTDEBUG && _d($sql);
7310
7311
my $sleep_time = 0;
7312
my $n_slaves = scalar @$slaves - 1;
7313
my @chunks;
7314
my %skip_slave;
7315
while ( $oktorun && ($chunks[0] || 0) < $max_chunk ) {
7316
@chunks = ();
7317
for my $i ( 0..$n_slaves ) {
7318
my $slave = $slaves->[$i];
7319
if ( $skip_slave{$i} ) {
7320
PTDEBUG && _d('Skipping slave', $slave->name(),
7321
'due to previous error it caused');
7322
next;
7323
}
7324
eval {
7325
my ($chunk) = $slave->dbh()->selectrow_array($sql);
7326
PTDEBUG && _d($slave->name(), 'max chunk:', $chunk);
7327
push @chunks, $chunk || 0;
7328
};
7329
if ($EVAL_ERROR) {
7330
if ( $o->get('quiet') < 2 ) {
7331
warn ts("Error waiting for the last checksum of table "
7332
. "$tbl->{db}.$tbl->{tbl} to replicate to "
7333
. "replica " . $slave->name() . ": $EVAL_ERROR\n"
7334
. "Check that the replica is running and has the "
7335
. "replicate table $repl_table. Checking the replica "
7336
. "for checksum differences will probably cause "
7337
. "another error.\n");
7338
}
7339
$tbl->{checksum_results}->{errors}++;
7340
$skip_slave{$i} = 1;
7341
next;
7342
}
7343
}
7344
@chunks = sort { $a <=> $b } @chunks;
7345
if ( $chunks[0] < $max_chunk ) {
7346
if ( $check_pr ) {
7347
$check_pr->update(sub { return $chunks[0]; });
7348
}
7349
7350
# We shouldn't wait long here because we already waited
7351
# for all slaves to catchup at least until --max-lag.
7352
$sleep_time += 0.25 if $sleep_time <= $o->get('max-lag');
7353
PTDEBUG && _d('Sleep', $sleep_time, 'waiting for chunks');
7354
sleep $sleep_time;
7355
}
7356
}
7357
return;
7358
}
7359
7360
# Catches signals so we can exit gracefully.
7361
sub sig_int {
7362
my ( $signal ) = @_;
7363
if ( $oktorun ) {
7364
print STDERR "# Caught SIG$signal.\n";
7365
$oktorun = 0;
7366
}
7367
else {
7368
print STDERR "# Exiting on SIG$signal.\n";
7369
exit 1;
7370
}
7371
}
7372
7079
7373
sub _d {
7080
7374
my ($package, undef, $line) = caller 0;
7081
7375
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
7098
7392
7099
7393
=head1 NAME
7100
7394
7101
pt-table-checksum - Perform an online replication consistency check, or checksum MySQL tables efficiently on one or many servers.
7395
pt-table-checksum - Verify MySQL replication integrity.
7102
7396
7103
7397
=head1 SYNOPSIS
7104
7398
7105
Usage: pt-table-checksum [OPTION...] DSN [DSN...]
7106
7107
pt-table-checksum checksums MySQL tables efficiently on one or more hosts.
7108
Each host is specified as a DSN and missing values are inherited from the
7109
first host. If you specify multiple hosts, the first is assumed to be the
7110
master.
7111
7112
B<STOP!> Are you checksumming slaves against a master? Then be sure to learn
7113
what L<"--replicate"> does. It is probably the option you want to use.
7114
7115
Checksum all slaves against the master:
7116
7117
pt-table-checksum \
7118
h=master-host \
7119
--replicate mydb.checksums
7120
7121
# Wait for first command to complete and replication to catchup
7122
# on all slaves, then...
7123
7124
pt-table-checksum \
7125
h=master-host \
7126
--replicat mydb.checksums \
7127
--replicate-check 2
7128
7129
Checksum all databases and tables on two servers and print the differences:
7130
7131
pt-table-checksum h=host1,u=user h=host2 | pt-checksum-filter
7132
7133
See L<"SPECIFYING HOSTS"> for more on the syntax of the host arguments.
7399
Usage: pt-table-checksum [OPTION...] [DSN]
7400
7401
pt-table-checksum performs an online replication consistency check by executing
7402
checksum queries on the master, which produces different results on replicas
7403
that are inconsistent with the master. The optional DSN specifies the master
7404
host. The tool's exit status is nonzero if any differences are found, or if any
7405
warnings or errors occur.
7406
7407
The following command will connect to the replication master on localhost,
7408
checksum every table, and report the results on every detected replica:
7409
7410
pt-table-checksum
7411
7412
This tool is focused on finding data differences efficiently. If any data is
7413
different, you can resolve the problem with pt-table-sync.
7134
7414
7135
7415
=head1 RISKS
7136
7416
7139
7419
are those created by the nature of the tool (e.g. read-only tools vs. read-write
7140
7420
tools) and those created by bugs.
7141
7421
7142
pt-table-checksum executes queries that cause the MySQL server to checksum its
7143
data. This can cause significant server load. It is read-only unless you use
7144
the L<"--replicate"> option, in which case it inserts a small amount of data
7145
into the specified table.
7422
pt-table-checksum can add load to the MySQL server, although it has many
7423
safeguards to prevent this. It inserts a small amount of data into a table that
7424
contains checksum results. It has checks that, if disabled, can potentially
7425
cause replication to fail when unsafe replication options are used. In short,
7426
it is safe by default, but it permits you to turn off its safety checks.
7146
7427
7147
At the time of this release, we know of no bugs that could cause serious harm to
7148
users. There are miscellaneous bugs that might be annoying.
7428
At the time of this release, we know of no bugs that could cause harm to
7429
users.
7149
7430
7150
7431
The authoritative source for updated information is always the online issue
7151
7432
tracking system. Issues that affect this tool will be marked as such. You can
7156
7437
7157
7438
=head1 DESCRIPTION
7158
7439
7159
pt-table-checksum generates table checksums for MySQL tables, typically
7160
useful for verifying your slaves are in sync with the master. The checksums
7161
are generated by a query on the server, and there is very little network
7162
traffic as a result.
7163
7164
Checksums typically take about twice as long as COUNT(*) on very large InnoDB
7165
tables in my tests. For smaller tables, COUNT(*) is a good bit faster than
7166
the checksums. See L<"--algorithm"> for more details on performance.
7167
7168
If you specify more than one server, pt-table-checksum assumes the first
7169
server is the master and others are slaves. Checksums are parallelized for
7170
speed, forking off a child process for each table. Duplicate server names are
7171
ignored, but if you want to checksum a server against itself you can use two
7172
different forms of the hostname (for example, "localhost 127.0.0.1", or
7173
"h=localhost,P=3306 h=localhost,P=3307").
7174
7175
If you want to compare the tables in one database to those in another database
7176
on the same server, just checksum both databases:
7177
7178
pt-table-checksum --databases db1,db2
7179
7180
You can then use L<pt-checksum-filter> to compare the results in both databases
7181
easily.
7182
7183
pt-table-checksum examines table structure only on the first host specified,
7184
so if anything differs on the others, it won't notice. It ignores views.
7185
7186
The checksums work on MySQL version 3.23.58 through 6.0-alpha. They will not
7187
necessarily produce the same values on all versions. Differences in
7188
formatting and/or space-padding between 4.1 and 5.0, for example, will cause
7189
the checksums to be different.
7190
7191
=head1 SPECIFYING HOSTS
7192
7193
Each host is specified on the command line as a DSN. A DSN is a comma-separted
7194
list of C<option=value> pairs. The most basic DSN is C<h=host> to specify
7195
the hostname of the server and use default for everything else (port, etc.).
7196
See L<"DSN OPTIONS"> for more information.
7197
7198
DSN options that are listed as C<copy: yes> are copied from the first DSN
7199
to subsequent DSNs that do not specify the DSN option. For example,
7200
C<h=host1,P=12345 h=host2> is equivalent to C<h=host1,P=12345 h=host2,P=12345>.
7201
This allows you to avoid repeating DSN options that have the same value
7202
for all DSNs.
7203
7204
Connection-related command-line options like L<"--user"> and L<"--password">
7205
provide default DSN values for the corresponding DSN options indicated by
7206
the short form of each option. For example, the short form of L<"--user">
7207
is C<-u> which corresponds to the C<u> DSN option, so C<--user bob h=host>
7208
is equivalent to C<h=host,u=bob>. These defaults apply to all DSNs that
7209
do not specify the DSN option.
7210
7211
The DSN option value precedence from higest to lowest is:
7212
7213
* explicit values in each DSN on the command-line
7214
* copied values from the first DSN
7215
* default values from connection-related command-line options
7216
7217
If you are confused about how pt-table-checksum will connect to your servers,
7218
use the L<"--explain-hosts"> option and it will tell you.
7219
7220
=head1 HOW FAST IS IT?
7221
7222
Speed and efficiency are important, because the typical use case is checksumming
7223
large amounts of data.
7224
7225
C<pt-table-checksum> is designed to do very little work itself, and generates
7226
very little network traffic aside from inspecting table structures with C<SHOW
7227
CREATE TABLE>. The results of checksum queries are typically 40-character or
7228
shorter strings.
7229
7230
The MySQL server does the bulk of the work, in the form of the checksum queries.
7231
The following benchmarks show the checksum query times for various checksum
7232
algorithms. The first two results are simply running C<COUNT(col8)> and
7233
C<CHECKSUM TABLE> on the table. C<CHECKSUM TABLE> is just C<CRC32> under the
7234
hood, but it's implemented inside the storage engine layer instead of at the
7235
MySQL layer.
7236
7237
ALGORITHM HASH FUNCTION EXTRA TIME
7238
============== ============= ============== =====
7239
COUNT(col8) 2.3
7240
CHECKSUM TABLE 5.3
7241
BIT_XOR FNV_64 12.7
7242
ACCUM FNV_64 42.4
7243
BIT_XOR MD5 --optimize-xor 80.0
7244
ACCUM MD5 87.4
7245
BIT_XOR SHA1 --optimize-xor 90.1
7246
ACCUM SHA1 101.3
7247
BIT_XOR MD5 172.0
7248
BIT_XOR SHA1 197.3
7249
7250
The tests are entirely CPU-bound. The sample data is an InnoDB table with the
7251
following structure:
7252
7253
CREATE TABLE test (
7254
col1 int NOT NULL,
7255
col2 date NOT NULL,
7256
col3 int NOT NULL,
7257
col4 int NOT NULL,
7258
col5 int,
7259
col6 decimal(3,1),
7260
col7 smallint unsigned NOT NULL,
7261
col8 timestamp NOT NULL,
7262
PRIMARY KEY (col2, col1),
7263
KEY (col7),
7264
KEY (col1)
7265
) ENGINE=InnoDB
7266
7267
The table has 4303585 rows, 365969408 bytes of data and 173457408 bytes of
7268
indexes. The server is a Dell PowerEdge 1800 with dual 32-bit Xeon 2.8GHz
7269
processors and 2GB of RAM. The tests are fully CPU-bound, and the server is
7270
otherwise idle. The results are generally consistent to within a tenth of a
7271
second on repeated runs.
7272
7273
C<CRC32> is the default checksum function to use, and should be enough for most
7274
cases. If you need stronger guarantees that your data is identical, you should
7275
use one of the other functions.
7276
7277
=head1 ALGORITHM SELECTION
7278
7279
The L<"--algorithm"> option allows you to specify which algorithm you would
7280
like to use, but it does not guarantee that pt-table-checksum will use this
7281
algorithm. pt-table-checksum will ultimately select the best algorithm possible
7282
given various factors such as the MySQL version and other command line options.
7283
7284
The three basic algorithms in descending order of preference are CHECKSUM,
7285
BIT_XOR and ACCUM. CHECKSUM cannot be used if any one of these criteria
7286
is true:
7287
7288
* --where is used
7289
* --since is used
7290
* --chunk-size is used
7291
* --replicate is used
7292
* --count is used
7293
* MySQL version less than 4.1.1
7294
7295
The BIT_XOR algorithm also requires MySQL version 4.1.1 or later.
7296
7297
After checking these criteria, if the requested L<"--algorithm"> remains then it
7298
is used, otherwise the first remaining algorithm with the highest preference
7299
is used.
7300
7301
=head1 CONSISTENT CHECKSUMS
7302
7303
If you are using this tool to verify your slaves still have the same data as the
7304
master, which is why I wrote it, you should read this section.
7305
7306
The best way to do this with replication is to use the L<"--replicate"> option.
7307
When the queries are finished running on the master and its slaves, you can go
7308
to the slaves and issue SQL queries to see if any tables are different from the
7309
master. Try the following:
7310
7311
SELECT db, tbl, chunk, this_cnt-master_cnt AS cnt_diff,
7312
this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc)
7313
AS crc_diff
7314
FROM checksum
7315
WHERE master_cnt <> this_cnt OR master_crc <> this_crc
7316
OR ISNULL(master_crc) <> ISNULL(this_crc);
7317
7318
The L<"--replicate-check"> option can do this query for you. If you can't use
7319
this method, try the following:
7320
7321
=over
7322
7323
=item *
7324
7325
If your servers are not being written to, you can just run the tool with no
7326
further ado:
7327
7328
pt-table-checksum server1 server2 ... serverN
7329
7330
=item *
7331
7332
If the servers are being written to, you need some way to make sure they are
7333
consistent at the moment you run the checksums. For situations other than
7334
master-slave replication, you will have to figure this out yourself. You may be
7335
able to use the L<"--where"> option with a date or time column to only checksum
7336
data that's not recent.
7337
7338
=item *
7339
7340
If you are checksumming a master and slaves, you can do a fast parallel
7341
checksum and assume the slaves are caught up to the master. In practice, this
7342
tends to work well except for tables which are constantly updated. You can
7343
use the L<"--slave-lag"> option to see how far behind each slave was when it
7344
checksummed a given table. This can help you decide whether to investigate
7345
further.
7346
7347
=item *
7348
7349
The next most disruptive technique is to lock the table on the master, then take
7350
checksums. This should prevent changes from propagating to the slaves. You can
7351
just lock on the master (with L<"--lock">), or you can both lock on the master
7352
and wait on the slaves till they reach that point in the master's binlog
7353
(L<"--wait">). Which is better depends on your workload; only you know that.
7354
7355
=item *
7356
7357
If you decide to make the checksums on the slaves wait until they're guaranteed
7358
to be caught up to the master, the algorithm looks like this:
7359
7360
For each table,
7361
Master: lock table
7362
Master: get pos
7363
In parallel,
7364
Master: checksum
7365
Slave(s): wait for pos, then checksum
7366
End
7367
Master: unlock table
7368
End
7369
7370
=back
7371
7372
What I typically do when I'm not using the L<"--replicate"> option is simply run
7373
the tool on all servers with no further options. This runs fast, parallel,
7374
non-blocking checksums simultaneously. If there are tables that look different,
7375
I re-run with L<"--wait">=600 on the tables in question. This makes the tool
7376
lock on the master as explained above.
7440
pt-table-checksum is designed to do the right thing by default in almost every
7441
case. When in doubt, use L<"--explain"> to see how the tool will checksum a
7442
table. The following is a high-level overview of how the tool functions.
7443
7444
In contrast to older versions of pt-table-checksum, this tool is focused on a
7445
single purpose, and does not have a lot of complexity or support many different
7446
checksumming techniques. It executes checksum queries on only one server, and
7447
these flow through replication to re-execute on replicas. If you need the older
7448
behavior, you can use Percona Toolkit version 1.0.
7449
7450
pt-table-checksum connects to the server you specify, and finds databases and
7451
tables that match the filters you specify (if any). It works one table at a
7452
time, so it does not accumulate large amounts of memory or do a lot of work
7453
before beginning to checksum. This makes it usable on very large servers. We
7454
have used it on servers with hundreds of thousands of databases and tables, and
7455
trillions of rows. No matter how large the server is, pt-table-checksum works
7456
equally well.
7457
7458
One reason it can work on very large tables is that it divides each table into
7459
chunks of rows, and checksums each chunk with a single REPLACE..SELECT query.
7460
It varies the chunk size to make the checksum queries run in the desired amount
7461
of time. The goal of chunking the tables, instead of doing each table with a
7462
single big query, is to ensure that checksums are unintrusive and don't cause
7463
too much replication lag or load on the server. That's why the target time for
7464
each chunk is 0.5 seconds by default.
7465
7466
The tool keeps track of how quickly the server is able to execute the queries,
7467
and adjusts the chunks as it learns more about the server's performance. It
7468
uses an exponentially decaying weighted average to keep the chunk size stable,
7469
yet remain responsive if the server's performance changes during checksumming
7470
for any reason. This means that the tool will quickly throttle itself if your
7471
server becomes heavily loaded during a traffic spike or a background task, for
7472
example.
7473
7474
Chunking is accomplished by a technique that we used to call "nibbling" in other
7475
tools in Percona Toolkit. It is the same technique used for pt-archiver, for
7476
example. The legacy chunking algorithms used in older versions of
7477
pt-table-checksum are removed, because they did not result in predictably sized
7478
chunks, and didn't work well on many tables. All that is required to divide a
7479
table into chunks is an index of some sort (preferably a primary key or unique
7480
index). If there is no index, and the table contains a suitably small number of
7481
rows, the tool will checksum the table in a single chunk.
7482
7483
pt-table-checksum has many other safeguards to ensure that it does not interfere
7484
with any server's operation, including replicas. To accomplish this,
7485
pt-table-checksum detects replicas and connects to them automatically. (If this
7486
fails, you can give it a hint with the L<"--recursion-method"> option.)
7487
7488
The tool monitors replicas continually. If any replica falls too far behind in
7489
replication, pt-table-checksum pauses to allow it to catch up. If any replica
7490
has an error, or replication stops, pt-table-checksum pauses and waits. In
7491
addition, pt-table-checksum looks for common causes of problems, such as
7492
replication filters, and refuses to operate unless you force it to. Replication
7493
filters are dangerous, because the queries that pt-table-checksum executes could
7494
potentially conflict with them and cause replication to fail.
7495
7496
pt-table-checksum verifies that chunks are not too large to checksum safely. It
7497
performs an EXPLAIN query on each chunk, and skips chunks that might be larger
7498
than the desired number of rows. You can configure the sensitivity of this
7499
safeguard with the L<"--chunk-size-limit"> option. If a table will be
7500
checksummed in a single chunk because it has a small number of rows, then
7501
pt-table-checksum additionally verifies that the table isn't oversized on
7502
replicas. This avoids the following scenario: a table is empty on the master
7503
but is very large on a replica, and is checksummed in a single large query,
7504
which causes a very long delay in replication.
7505
7506
There are several other safeguards. For example, pt-table-checksum sets its
7507
session-level innodb_lock_wait_timeout to 1 second, so that if there is a lock
7508
wait, it will be the victim instead of causing other queries to time out.
7509
Another safeguard checks the load on the database server, and pauses if the load
7510
is too high. There is no single right answer for how to do this, but by default
7511
pt-table-checksum will pause if there are more than 25 concurrently executing
7512
queries. You should probably set a sane value for your server with the
7513
L<"--max-load"> option.
7514
7515
Checksumming usually is a low-priority task that should yield to other work on
7516
the server. However, a tool that must be restarted constantly is difficult to
7517
use. Thus, pt-table-checksum is very resilient to errors. For example, if the
7518
database administrator needs to kill pt-table-checksum's queries for any reason,
7519
that is not a fatal error. Users often run pt-kill to kill any long-running
7520
checksum queries. The tool will retry a killed query once, and if it fails
7521
again, it will move on to the next chunk of that table. The same behavior
7522
applies if there is a lock wait timeout. The tool will print a warning if such
7523
an error happens, but only once per table. If the connection to any server
7524
fails, pt-table-checksum will attempt to reconnect and continue working.
7525
7526
If pt-table-checksum encounters a condition that causes it to stop completely,
7527
it is easy to resume it with the L<"--resume"> option. It will begin from the
7528
last chunk of the last table that it processed. You can also safely stop the
7529
tool with CTRL-C. It will finish the chunk it is currently processing, and then
7530
exit. You can resume it as usual afterwards.
7531
7532
After pt-table-checksum finishes checksumming all of the chunks in a table, it
7533
pauses and waits for all detected replicas to finish executing the checksum
7534
queries. Once that is finished, it checks all of the replicas to see if they
7535
have the same data as the master, and then prints a line of output with the
7536
results. You can see a sample of its output later in this documentation.
7537
7538
The tool prints progress indicators during time-consuming operations. It prints
7539
a progress indicator as each table is checksummed. The progress is computed by
7540
the estimated number of rows in the table. It will also print a progress report
7541
when it pauses to wait for replication to catch up, and when it is waiting to
7542
check replicas for differences from the master. You can make the output less
7543
verbose with the L<"--quiet"> option.
7544
7545
If you wish, you can query the checksum tables manually to get a report of which
7546
tables and chunks have differences from the master. The following query will
7547
report every database and table with differences, along with a summary of the
7548
number of chunks and rows possibly affected:
7549
7550
SELECT db, tbl, SUM(this_cnt) AS total_rows, COUNT(*) AS chunks
7551
FROM percona.checksums
7552
WHERE (
7553
master_cnt <> this_cnt
7554
OR master_crc <> this_crc
7555
OR ISNULL(master_crc) <> ISNULL(this_crc))
7556
GROUP BY db, tbl;
7557
7558
The table referenced in that query is the checksum table, where the checksums
7559
are stored. Each row in the table contains the checksum of one chunk of data
7560
from some table in the server.
7561
7562
Version 2.0 of pt-table-checksum is not backwards compatible with pt-table-sync
7563
version 1.0. In some cases this is not a serious problem. Adding a
7564
"boundaries" column to the table, and then updating it with a manually generated
7565
WHERE clause, may suffice to let pt-table-sync version 1.0 interoperate with
7566
pt-table-checksum version 2.0. Assuming an integer primary key named 'id', You
7567
can try something like the following:
7568
7569
ALTER TABLE checksums ADD boundaries VARCHAR(500);
7570
UPDATE checksums
7571
SET boundaries = COALESCE(CONCAT('id BETWEEN ', lower_boundary,
7572
' AND ', upper_boundary), '1=1');
7377
7573
7378
7574
=head1 OUTPUT
7379
7575
7380
Output is to STDOUT, one line per server and table, with header lines for each
7381
database. I tried to make the output easy to process with awk. For this reason
7382
columns are always present. If there's no value, pt-table-checksum prints
7383
'NULL'.
7384
7385
The default is column-aligned output for human readability, but you can change
7386
it to tab-separated if you want. Use the L<"--tab"> option for this.
7387
7388
Output is unsorted, though all lines for one table should be output together.
7389
For speed, all checksums are done in parallel (as much as possible) and may
7390
complete out of the order in which they were started. You might want to run
7391
them through another script or command-line utility to make sure they are in the
7392
order you want. If you pipe the output through L<pt-checksum-filter>, you
7393
can sort the output and/or avoid seeing output about tables that have no
7394
differences.
7395
7396
The columns in the output are as follows. The database, table, and chunk come
7397
first so you can sort by them easily (they are the "primary key").
7398
7399
Output from L<"--replicate-check"> and L<"--checksum"> are different.
7400
7401
=over
7402
7403
=item DATABASE
7404
7405
The database the table is in.
7406
7407
=item TABLE
7408
7409
The table name.
7576
The tool prints tabular results, one line per table:
7577
7578
TS ERRORS DIFFS ROWS CHUNKS SKIPPED TIME TABLE
7579
10-20T08:36:50 0 0 200 1 0 0.005 db1.tbl1
7580
10-20T08:36:50 0 0 603 7 0 0.035 db1.tbl2
7581
10-20T08:36:50 0 0 16 1 0 0.003 db2.tbl3
7582
10-20T08:36:50 0 0 600 6 0 0.024 db2.tbl4
7583
7584
Errors, warnings, and progress reports are printed to standard error. See also
7585
L<"--quiet">.
7586
7587
Each table's results are printed when the tool finishes checksumming the table.
7588
The columns are as follows:
7589
7590
=over
7591
7592
=item TS
7593
7594
The timestamp (without the year) when the tool finished checksumming the table.
7595
7596
=item ERRORS
7597
7598
The number of errors and warnings that occurred while checksumming the table.
7599
Errors and warnings are printed to standard error while the table is in
7600
progress.
7601
7602
=item DIFFS
7603
7604
The number of chunks that differ from the master on one or more replicas.
7605
If C<--no-replicate-check> is specified, this column will always have zeros.
7606
If L<"--replicate-check-only"> is specified, then only tables with differences
7607
are printed.
7608
7609
=item ROWS
7610
7611
The number of rows selected and checksummed from the table. It might be
7612
different from the number of rows in the table if you use the --where option.
7613
7614
=item CHUNKS
7615
7616
The number of chunks into which the table was divided.
7617
7618
=item SKIPPED
7619
7620
The number of chunks that were skipped due to errors or warnings, or because
7621
they were oversized.
7622
7623
=item TIME
7624
7625
The time elapsed while checksumming the table.
7626
7627
=item TABLE
7628
7629
The database and table that was checksummed.
7630
7631
=back
7632
7633
If L<"--replicate-check-only"> is specified, only checksum differences on
7634
detected replicas are printed. The output is different: one paragraph per
7635
replica, one checksum difference per line, and values are separted by spaces:
7636
7637
Differences on h=127.0.0.1,P=12346
7638
TABLE CHUNK CNT_DIFF CRC_DIFF CHUNK_INDEX LOWER_BOUNDARY UPPER_BOUNDARY
7639
db1.tbl1 1 0 1 PRIMARY 1 100
7640
db1.tbl1 6 0 1 PRIMARY 501 600
7641
7642
Differences on h=127.0.0.1,P=12347
7643
TABLE CHUNK CNT_DIFF CRC_DIFF CHUNK_INDEX LOWER_BOUNDARY UPPER_BOUNDARY
7644
db1.tbl1 1 0 1 PRIMARY 1 100
7645
db2.tbl2 9 5 0 PRIMARY 101 200
7646
7647
The first line of a paragraph indicates the replica with differences.
7648
In this example there are two: h=127.0.0.1,P=12346 and h=127.0.0.1,P=12347.
7649
The columns are as follows:
7650
7651
=over
7652
7653
=item TABLE
7654
7655
The database and table that differs from the master.
7410
7656
7411
7657
=item CHUNK
7412
7658
7413
The chunk (see L<"--chunk-size">). Zero if you are not doing chunked checksums.
7414
7415
=item HOST
7416
7417
The server's hostname.
7418
7419
=item ENGINE
7420
7421
The table's storage engine.
7422
7423
=item COUNT
7424
7425
The table's row count, unless you specified to skip it. If C<OVERSIZE> is
7426
printed, the chunk was skipped because the actual number of rows was greater
7427
than L<"--chunk-size"> times L<"--chunk-size-limit">.
7428
7429
=item CHECKSUM
7430
7431
The table's checksum, unless you specified to skip it or the table has no rows.
7432
some types of checksums will be 0 if there are no rows; others will print NULL.
7433
7434
=item TIME
7435
7436
How long it took to checksum the C<CHUNK>, not including C<WAIT> time.
7437
Total checksum time is C<WAIT + TIME>.
7438
7439
=item WAIT
7440
7441
How long the slave waited to catch up to its master before beginning to
7442
checksum. C<WAIT> is always 0 for the master. See L<"--wait">.
7443
7444
=item STAT
7445
7446
The return value of MASTER_POS_WAIT(). C<STAT> is always C<NULL> for the
7447
master.
7448
7449
=item LAG
7450
7451
How far the slave lags the master, as reported by SHOW SLAVE STATUS.
7452
C<LAG> is always C<NULL> for the master.
7659
The chunk number of the table that differs from the master.
7660
7661
=item CNT_DIFF
7662
7663
The number of chunk rows on the replica minus the number of chunk rows
7664
on the master.
7665
7666
=item CRC_DIFF
7667
7668
1 if the CRC of the chunk on the replica is different than the CRC of the
7669
chunk on the master, else 0.
7670
7671
=item CHUNK_INDEX
7672
7673
The index used to chunk the table.
7674
7675
=item LOWER_BOUNDARY
7676
7677
The index values that define the lower boundary of the chunk.
7678
7679
=item UPPER_BOUNDARY
7680
7681
The index values that define the upper boundary of the chunk.
7453
7682
7454
7683
=back
7455
7684
7456
=head1 REPLICATE TABLE MAINTENANCE
7457
7458
If you use L<"--replicate"> to store and replicate checksums, you may need to
7459
perform maintenance on the replicate table from time to time to remove old
7460
checksums. This section describes when checksums in the replicate table are
7461
deleted automatically by pt-table-checksum and when you must manually delete
7462
them.
7463
7464
Before starting, pt-table-checksum calculates chunks for each table, even
7465
if L<"--chunk-size"> is not specified (in that case there is one chunk: "1=1").
7466
Then, before checksumming each table, the tool deletes checksum chunks in the
7467
replicate table greater than the current number of chunks. For example,
7468
if a table is chunked into 100 chunks, 0-99, then pt-table-checksum does:
7469
7470
DELETE FROM replicate table WHERE db=? AND tbl=? AND chunk > 99
7471
7472
That removes any high-end chunks from previous runs which no longer exist.
7473
Currently, this operation cannot be disabled.
7474
7475
If you use L<"--resume">, L<"--resume-replicate">, or L<"--modulo">, then
7476
you need to be careful that the number of rows in a table does not decrease
7477
so much that the number of chunks decreases too, else some checksum chunks may
7478
be deleted. The one exception is if only rows at the high end of the range
7479
are deleted. In that case, the high-end chunks are deleted and lower chunks
7480
remain unchanged. An increasing number of rows or chunks should not cause
7481
any adverse affects.
7482
7483
Changing the L<"--chunk-size"> between runs with L<"--resume">,
7484
L<"--resume-replicate">, or L<"--modulo"> can cause odd or invalid checksums.
7485
You should not do this. It won't work with the resume options. With
7486
L<"--modulo">, the safest thing to do is manually delete all the rows in
7487
the replicate table for the table in question and start over.
7488
7489
If the replicate table becomes cluttered with old or invalid checksums
7490
and the auto-delete operation is not deleting them, then you will need to
7491
manually clean up the replicate table. Alternatively, if you specify
7492
L<"--empty-replicate-table">, then the tool deletes every row in the
7493
replicate table.
7494
7495
7685
=head1 EXIT STATUS
7496
7686
7497
An exit status of 0 (sometimes also called a return value or return code)
7498
indicates success. If there is an error checksumming any table, the exit status
7499
is 1.
7500
7501
When running L<"--replicate-check">, if any slave has chunks that differ from
7502
the master, the exit status is 1.
7503
7504
=head1 QUERIES
7505
7506
If you are using innotop (see L<http://code.google.com/p/innotop>),
7507
mytop, or another tool to watch currently running MySQL queries, you may see
7508
the checksum queries. They look similar to this:
7509
7510
REPLACE /*test.test_tbl:'2'/'5'*/ INTO test.checksum(db, ...
7511
7512
Since pt-table-checksum's queries run for a long time and tend to be
7513
textually very long, and thus won't fit on one screen of these monitoring
7514
tools, I've been careful to place a comment at the beginning of the query so
7515
you can see what it is and what it's doing. The comment contains the name of
7516
the table that's being checksummed, the chunk it is currently checksumming,
7517
and how many chunks will be checksummed. In the case above, it is
7518
checksumming chunk 2 of 5 in table test.test_tbl.
7687
A non-zero exit status indicates errors, warnings, or checksum differences.
7519
7688
7520
7689
=head1 OPTIONS
7521
7690
7522
L<"--schema"> is restricted to option groups Connection, Filter, Output, Help, Config, Safety.
7523
7524
L<"--empty-replicate-table">, L<"--resume"> and L<"--resume-replicate"> are mutually exclusive.
7525
7526
7691
This tool accepts additional command-line arguments. Refer to the
7527
7692
L<"SYNOPSIS"> and usage information for details.
7528
7693
7529
7694
=over
7530
7695
7531
=item --algorithm
7532
7533
type: string
7534
7535
Checksum algorithm (ACCUM|CHECKSUM|BIT_XOR).
7536
7537
Specifies which checksum algorithm to use. Valid arguments are CHECKSUM,
7538
BIT_XOR and ACCUM. The latter two do cryptographic hash checksums.
7539
See also L<"ALGORITHM SELECTION">.
7540
7541
CHECKSUM is built into MySQL, but has some disadvantages. BIT_XOR and ACCUM are
7542
implemented by SQL queries. They use a cryptographic hash of all columns
7543
concatenated together with a separator, followed by a bitmap of each nullable
7544
column that is NULL (necessary because CONCAT_WS() skips NULL columns).
7545
7546
CHECKSUM is the default. This method uses MySQL's built-in CHECKSUM TABLE
7547
command, which is a CRC32 behind the scenes. It cannot be used before MySQL
7548
4.1.1, and various options disable it as well. It does not simultaneously count
7549
rows; that requires an extra COUNT(*) query. This is a good option when you are
7550
using MyISAM tables with live checksums enabled; in this case both the COUNT(*)
7551
and CHECKSUM queries will run very quickly.
7552
7553
The BIT_XOR algorithm is available for MySQL 4.1.1 and newer. It uses
7554
BIT_XOR(), which is order-independent, to reduce all the rows to a single
7555
checksum.
7556
7557
ACCUM uses a user variable as an accumulator. It reduces each row to a single
7558
checksum, which is concatenated with the accumulator and re-checksummed. This
7559
technique is order-dependent. If the table has a primary key, it will be used
7560
to order the results for consistency; otherwise it's up to chance.
7561
7562
The pathological worst case is where identical rows will cancel each other out
7563
in the BIT_XOR. In this case you will not be able to distinguish a table full
7564
of one value from a table full of another value. The ACCUM algorithm will
7565
distinguish them.
7566
7567
However, the ACCUM algorithm is order-dependent, so if you have two tables
7568
with identical data but the rows are out of order, you'll get different
7569
checksums with ACCUM.
7570
7571
If a given algorithm won't work for some reason, pt-table-checksum falls back to
7572
another. The least common denominator is ACCUM, which works on MySQL 3.23.2 and
7573
newer.
7574
7575
=item --arg-table
7576
7577
type: string
7578
7579
The database.table with arguments for each table to checksum.
7580
7581
This table may be named anything you wish. It must contain at least the
7582
following columns:
7583
7584
CREATE TABLE checksum_args (
7585
db char(64) NOT NULL,
7586
tbl char(64) NOT NULL,
7587
-- other columns as desired
7588
PRIMARY KEY (db, tbl)
7589
);
7590
7591
In addition to the columns shown, it may contain any of the other columns listed
7592
here (Note: this list is used by the code, MAGIC_overridable_args):
7593
7594
algorithm chunk-column chunk-index chunk-size columns count crc function lock
7595
modulo use-index offset optimize-xor chunk-size-limit probability separator
7596
save-since single-chunk since since-column sleep sleep-coef trim wait where
7597
7598
Each of these columns corresponds to the long form of a command-line option.
7599
Each column should be NULL-able. Column names with hyphens should be enclosed
7600
in backticks (e.g. `chunk-size`) when the table is created. The data type does
7601
not matter, but it's suggested you use a sensible data type to prevent garbage
7602
data.
7603
7604
When C<pt-table-checksum> checksums a table, it will look for a matching entry
7605
in this table. Any column that has a defined value will override the
7606
corresponding command-line argument for the table being currently processed.
7607
In this way it is possible to specify custom command-line arguments for any
7608
table.
7609
7610
If you add columns to the table that aren't in the above list of allowable
7611
columns, it's an error. The exceptions are C<db>, C<tbl>, and C<ts>. The C<ts>
7612
column can be used as a timestamp for easy visibility into the last time the
7613
C<since> column was updated with L<"--save-since">.
7614
7615
This table is assumed to be located on the first server given on the
7616
command-line.
7617
7618
7696
=item --ask-pass
7619
7697
7620
7698
group: Connection
7623
7701
7624
7702
=item --check-interval
7625
7703
7626
type: time; group: Throttle; default: 1s
7704
type: time; default: 1; group: Throttle
7627
7705
7628
How often to check for slave lag if L<"--check-slave-lag"> is given.
7706
Sleep time between checks for L<"--max-lag">.
7629
7707
7630
7708
=item --[no]check-replication-filters
7631
7709
7632
7710
default: yes; group: Safety
7633
7711
7634
Do not L<"--replicate"> if any replication filters are set. When
7635
--replicate is specified, pt-table-checksum tries to detect slaves and look
7636
for options that filter replication, such as binlog_ignore_db and
7637
replicate_do_db. If it finds any such filters, it aborts with an error.
7638
Replication filtering makes it impossible to be sure that the checksum
7639
queries won't break replication or simply fail to replicate. If you are sure
7640
that it's OK to run the checksum queries, you can negate this option to
7641
disable the checks. See also L<"--replicate-database">.
7712
Do not checksum if any replication filters are set on any replicas.
7713
The tool looks for server options that filter replication, such as
7714
binlog_ignore_db and replicate_do_db. If it finds any such filters,
7715
it aborts with an error.
7716
7717
If the replicas are configured with any filtering options, you should be careful
7718
not to checksum any databases or tables that exist on the master and not the
7719
replicas. Changes to such tables might normally be skipped on the replicas
7720
because of the filtering options, but the checksum queries modify the contents
7721
of the table that stores the checksums, not the tables whose data you are
7722
checksumming. Therefore, these queries will be executed on the replica, and if
7723
the table or database you're checksumming does not exist, the queries will cause
7724
replication to fail. For more information on replication rules, see
7725
L<http://dev.mysql.com/doc/en/replication-rules.html>.
7726
7727
Replication filtering makes it impossible to be sure that the checksum queries
7728
won't break replication (or simply fail to replicate). If you are sure that
7729
it's OK to run the checksum queries, you can negate this option to disable the
7730
checks. See also L<"--replicate-database">.
7642
7731
7643
7732
=item --check-slave-lag
7644
7733
7645
type: DSN; group: Throttle
7646
7647
Pause checksumming until the specified slave's lag is less than L<"--max-lag">.
7648
7649
If this option is specified and L<"--throttle-method"> is set to C<slavelag>
7650
then L<"--throttle-method"> only checks this slave.
7651
7652
=item --checksum
7653
7654
group: Output
7655
7656
Print checksums and table names in the style of md5sum (disables
7657
L<"--[no]count">).
7658
7659
Makes the output behave more like the output of C<md5sum>. The checksum is
7660
first on the line, followed by the host, database, table, and chunk number,
7661
concatenated with dots.
7662
7663
=item --chunk-column
7664
7665
type: string
7666
7667
Prefer this column for dividing tables into chunks. By default,
7668
pt-table-checksum chooses the first suitable column for each table, preferring
7669
to use the primary key. This option lets you specify a preferred column, which
7670
pt-table-checksum uses if it exists in the table and is chunkable. If not, then
7671
pt-table-checksum will revert to its default behavior. Be careful when using
7672
this option; a poor choice could cause bad performance. This is probably best
7673
to use when you are checksumming only a single table, not an entire server. See
7674
also L<"--chunk-index">.
7734
type: string; group: Throttle
7735
7736
Pause checksumming until this replica's lag is less than L<"--max-lag">. The
7737
value is a DSN that inherits properties from the master host and the connection
7738
options (L<"--port">, L<"--user">, etc.). This option overrides the normal
7739
behavior of finding and continually monitoring replication lag on ALL connected
7740
replicas. If you don't want to monitor ALL replicas, but you want more than
7741
just one replica to be monitored, then use the DSN option to the
7742
L<"--recursion-method"> option instead of this option.
7675
7743
7676
7744
=item --chunk-index
7677
7745
7678
7746
type: string
7679
7747
7680
Prefer this index for chunking tables. By default, pt-table-checksum chooses an
7681
appropriate index for the L<"--chunk-column"> (even if it chooses the chunk
7682
column automatically). This option lets you specify the index you prefer. If
7683
the index doesn't exist, then pt-table-checksum will fall back to its default
7684
behavior. pt-table-checksum adds the index to the checksum SQL statements in a
7685
C<FORCE INDEX> clause. Be careful when using this option; a poor choice of
7686
index could cause bad performance. This is probably best to use when you are
7687
checksumming only a single table, not an entire server.
7688
7689
=item --chunk-range
7690
7691
type: string; default: open
7692
7693
Set which ends of the chunk range are open or closed. Possible values are
7694
one of MAGIC_chunk_range:
7695
7696
VALUE OPENS/CLOSES
7697
========== ======================
7698
open Both ends are open
7699
openclosed Low end open, high end closed
7700
7701
By default pt-table-checksum uses an open range of chunks like:
7702
7703
`id` < '10'
7704
`id` >= '10' AND < '20'
7705
`id` >= '20'
7706
7707
That range is open because the last chunk selects any row with id greater than
7708
(or equal to) 20. An open range can be a problem in cases where a lot of new
7709
rows are inserted with IDs greater than 20 while pt-table-checksumming is
7710
running because the final open-ended chunk will select all the newly inserted
7711
rows. (The less common case of inserting rows with IDs less than 10 would
7712
require a C<closedopen> range but that is not currently implemented.)
7713
Specifying C<openclosed> will cause the final chunk to be closed like:
7714
7715
`id` >= '20' AND `id` <= N
7716
7717
N is the C<MAX(`id`)> that pt-table-checksum used when it first chunked
7718
the rows. Therefore, it will only chunk the range of rows that existed when
7719
the tool started and not any newly inserted rows (unless those rows happen
7720
to be inserted with IDs less than N).
7721
7722
See also L<"--chunk-size-limit">.
7748
Prefer this index for chunking tables. By default, pt-table-checksum chooses
7749
the most appropriate index for chunking. This option lets you specify the index
7750
that you prefer. If the index doesn't exist, then pt-table-checksum will fall
7751
back to its default behavior of choosing an index. pt-table-checksum adds the
7752
index to the checksum SQL statements in a C<FORCE INDEX> clause. Be careful
7753
when using this option; a poor choice of index could cause bad performance.
7754
This is probably best to use when you are checksumming only a single table, not
7755
an entire server.
7723
7756
7724
7757
=item --chunk-size
7725
7758
7726
type: string
7727
7728
Approximate number of rows or size of data to checksum at a time. Allowable
7729
suffixes are k, M, G. Disallows C<--algorithm CHECKSUM>.
7730
7731
If you specify a chunk size, pt-table-checksum will try to find an index that
7732
will let it split the table into ranges of approximately L<"--chunk-size">
7733
rows, based on the table's index statistics. Currently only numeric and date
7734
types can be chunked.
7735
7736
If the table is chunkable, pt-table-checksum will checksum each range separately
7737
with parameters in the checksum query's WHERE clause. If pt-table-checksum
7738
cannot find a suitable index, it will do the entire table in one chunk as though
7739
you had not specified L<"--chunk-size"> at all. Each table is handled
7740
individually, so some tables may be chunked and others not.
7741
7742
The chunks will be approximately sized, and depending on the distribution of
7743
values in the indexed column, some chunks may be larger than the value you
7744
specify.
7745
7746
If you specify a suffix (one of k, M or G), the parameter is treated as a data
7747
size rather than a number of rows. The output of SHOW TABLE STATUS is then used
7748
to estimate the amount of data the table contains, and convert that to a number
7749
of rows.
7759
type: size; default: 1000
7760
7761
Number of rows to select for each checksum query. Allowable suffixes are
7762
k, M, G.
7763
7764
This option can override the default behavior, which is to adjust chunk size
7765
dynamically to try to make chunks run in exactly L<"--chunk-time"> seconds.
7766
When this option isn't set explicitly, its default value is used as a starting
7767
point, but after that, the tool ignores this option's value. If you set this
7768
option explicitly, however, then it disables the dynamic adjustment behavior and
7769
tries to make all chunks exactly the specified number of rows.
7770
7771
There is a subtlety: if the chunk index is not unique, then it's possible that
7772
chunks will be larger than desired. For example, if a table is chunked by an
7773
index that contains 10,000 of a given value, there is no way to write a WHERE
7774
clause that matches only 1,000 of the values, and that chunk will be at least
7775
10,000 rows large. Such a chunk will probably be skipped because of
7776
L<"--chunk-size-limit">.
7750
7777
7751
7778
=item --chunk-size-limit
7752
7779
7753
7780
type: float; default: 2.0; group: Safety
7754
7781
7755
Do not checksum chunks with this many times more rows than L<"--chunk-size">.
7756
7757
When L<"--chunk-size"> is given it specifies an ideal size for each chunk
7758
of a chunkable table (in rows; size values are converted to rows). Before
7759
checksumming each chunk, pt-table-checksum checks how many rows are in the
7760
chunk with EXPLAIN. If the number of rows reported by EXPLAIN is this many
7761
times greater than L<"--chunk-size">, then the chunk is skipped and C<OVERSIZE>
7762
is printed for the C<COUNT> column of the L<"OUTPUT">.
7763
7764
For example, if you specify L<"--chunk-size"> 100 and a chunk has 150 rows,
7765
then it is checksummed with the default L<"--chunk-size-limit"> value 2.0
7766
because 150 is less than 100 * 2.0. But if the chunk has 205 rows, then it
7767
is not checksummed because 205 is greater than 100 * 2.0.
7768
7769
The minimum value for this option is 1 which means that no chunk can be any
7770
larger than L<"--chunk-size">. You probably don't want to specify 1 because
7771
rows reported by EXPLAIN are estimates which can be greater than or less than
7772
the real number of rows in the chunk. If too many chunks are skipped because
7773
they are oversize, you might want to specify a value larger than 2.
7774
7775
You can disable oversize chunk checking by specifying L<"--chunk-size-limit"> 0.
7776
7777
See also L<"--unchunkable-tables">.
7782
Do not checksum chunks this much larger than the desired chunk size.
7783
7784
When a table has no unique indexes, chunk sizes can be inaccurate. This option
7785
specifies a maximum tolerable limit to the inaccuracy. The tool uses <EXPLAIN>
7786
to estimate how many rows are in the chunk. If that estimate exceeds the
7787
desired chunk size times the limit (twice as large, by default), then the tool
7788
skips the chunk.
7789
7790
The minimum value for this option is 1, which means that no chunk can be larger
7791
than L<"--chunk-size">. You probably don't want to specify 1, because rows
7792
reported by EXPLAIN are estimates, which can be different from the real number
7793
of rows in the chunk. If the tool skips too many chunks because they are
7794
oversized, you might want to specify a value larger than the default of 2.
7795
7796
You can disable oversized chunk checking by specifying a value of 0.
7797
7798
=item --chunk-time
7799
7800
type: float; default: 0.5
7801
7802
Adjust the chunk size dynamically so each checksum query takes this long to execute.
7803
7804
The tool tracks the checksum rate (rows per second) for all tables and each
7805
table individually. It uses these rates to adjust the chunk size after each
7806
checksum query, so that the next checksum query takes this amount of time (in
7807
seconds) to execute.
7808
7809
The algorithm is as follows: at the beginning of each table, the chunk size is
7810
initialized from the overall average rows per second since the tool began
7811
working, or the value of L<"--chunk-size"> if the tool hasn't started working
7812
yet. For each subsequent chunk of a table, the tool adjusts the chunk size to
7813
try to make queries run in the desired amount of time. It keeps an
7814
exponentially decaying moving average of queries per second, so that if the
7815
server's performance changes due to changes in server load, the tool adapts
7816
quickly. This allows the tool to achieve predictably timed queries for each
7817
table, and for the server overall.
7818
7819
If this option is set to zero, the chunk size doesn't auto-adjust, so query
7820
checksum times will vary, but query checksum sizes will not. Another way to do
7821
the same thing is to specify a value for L<"--chunk-size"> explicitly, instead
7822
of leaving it at the default.
7778
7823
7779
7824
=item --columns
7780
7825
7789
7834
Read this comma-separated list of config files; if specified, this must be the
7790
7835
first option on the command line.
7791
7836
7792
=item --[no]count
7793
7794
Count rows in tables. This is built into ACCUM and BIT_XOR, but requires an
7795
extra query for CHECKSUM.
7796
7797
This is disabled by default to avoid an extra COUNT(*) query when
7798
L<"--algorithm"> is CHECKSUM. If you have only MyISAM tables and live checksums
7799
are enabled, both CHECKSUM and COUNT will be very fast, but otherwise you may
7800
want to use one of the other algorithms.
7801
7802
=item --[no]crc
7837
=item --[no]create-replicate-table
7803
7838
7804
7839
default: yes
7805
7840
7806
Do a CRC (checksum) of tables.
7807
7808
Take the checksum of the rows as well as their count. This is enabled by
7809
default. If you disable it, you'll just get COUNT(*) queries.
7810
7811
=item --create-replicate-table
7812
7813
Create the replicate table given by L<"--replicate"> if it does not exist.
7814
7815
Normally, if the replicate table given by L<"--replicate"> does not exist,
7816
C<pt-table-checksum> will die. With this option, however, C<pt-table-checksum>
7817
will create the replicate table for you, using the database.table name given to
7818
L<"--replicate">.
7819
7841
Create the L<"--replicate"> database and table if they do not exist.
7820
7842
The structure of the replicate table is the same as the suggested table
7821
mentioned in L<"--replicate">. Note that since ENGINE is not specified, the
7822
replicate table will use the server's default storage engine. If you want to
7823
use a different engine, you need to create the table yourself.
7843
mentioned in L<"--replicate">.
7824
7844
7825
7845
=item --databases
7826
7846
7830
7850
7831
7851
=item --databases-regex
7832
7852
7833
type: string
7853
type: string; group: Filter
7834
7854
7835
7855
Only checksum databases whose names match this Perl regex.
7836
7856
7841
7861
Only read mysql options from the given file. You must give an absolute
7842
7862
pathname.
7843
7863
7844
=item --empty-replicate-table
7845
7846
DELETE all rows in the L<"--replicate"> table before starting.
7847
7848
Issues a DELETE against the table given by L<"--replicate"> before beginning
7849
work. Ignored if L<"--replicate"> is not specified. This can be useful to
7850
remove entries related to tables that no longer exist, or just to clean out the
7851
results of a previous run.
7852
7853
If you want to delete entries for specific databases or tables you must
7854
do this manually.
7864
=item --[no]empty-replicate-table
7865
7866
default: yes
7867
7868
Delete previous checksums for each table before checksumming the table. This
7869
option does not truncate the entire table, it only deletes rows (checksums) for
7870
each table just before checksumming the table. Therefore, if checksumming stops
7871
prematurely and there was preexisting data, there will still be rows for tables
7872
that were not checksummed before the tool was stopped.
7873
7874
If you're resuming from a previous checksum run, then the checksum records for
7875
the table from which the tool resumes won't be emptied.
7855
7876
7856
7877
=item --engines
7857
7878
7858
7879
short form: -e; type: hash; group: Filter
7859
7880
7860
Do only this comma-separated list of storage engines.
7881
Only checksum tables which use these storage engines.
7861
7882
7862
7883
=item --explain
7863
7884
7864
group: Output
7865
7866
Show, but do not execute, checksum queries (disables L<"--empty-replicate-table">).
7867
7868
=item --explain-hosts
7869
7870
group: Help
7871
7872
Print full DSNs for each host and exit. This option allows you to see how
7873
pt-table-checksum parses DSNs from the command-line and how it will connect
7874
to those hosts. See L<"SPECIFYING HOSTS">.
7885
cumulative: yes; default: 0; group: Output
7886
7887
Show, but do not execute, checksum queries (disables
7888
L<"--[no]empty-replicate-table">). If specified twice, the tool actually
7889
iterates through the chunking algorithm, printing the upper and lower boundary
7890
values for each chunk, but not executing the checksum queries.
7875
7891
7876
7892
=item --float-precision
7877
7893
7878
7894
type: int
7879
7895
7880
Precision for C<FLOAT> and C<DOUBLE> number-to-string conversion. Causes FLOAT
7896
Precision for FLOAT and DOUBLE number-to-string conversion. Causes FLOAT
7881
7897
and DOUBLE values to be rounded to the specified number of digits after the
7882
7898
decimal point, with the ROUND() function in MySQL. This can help avoid
7883
7899
checksum mismatches due to different floating-point representations of the same
7892
7908
7893
7909
Hash function for checksums (FNV1A_64, MURMUR_HASH, SHA1, MD5, CRC32, etc).
7894
7910
7895
You can use this option to choose the cryptographic hash function used for
7896
L<"--algorithm">=ACCUM or L<"--algorithm">=BIT_XOR. The default is to use
7897
C<CRC32>, but C<MD5> and C<SHA1> also work, and you can use your own function,
7898
such as a compiled UDF, if you wish. Whatever function you specify is run in
7899
SQL, not in Perl, so it must be available to MySQL.
7911
The default is to use CRC32(), but MD5() and SHA1() also work, and you
7912
can use your own function, such as a compiled UDF, if you wish. The
7913
function you specify is run in SQL, not in Perl, so it must be available
7914
to MySQL.
7900
7915
7901
The C<FNV1A_64> UDF mentioned in the benchmarks is much faster than C<MD5>. The
7902
C++ source code is distributed with Maatkit. It is very simple to compile and
7903
install; look at the header in the source code for instructions. If it is
7904
installed, it is preferred over C<MD5>. You can also use the MURMUR_HASH
7905
function if you compile and install that as a UDF; the source is also
7906
distributed with Maatkit, and it is faster and has better distribution
7907
than FNV1A_64.
7916
MySQL doesn't have good built-in hash functions that are fast. CRC32() is too
7917
prone to hash collisions, and MD5() and SHA1() are very CPU-intensive. The
7918
FNV1A_64() UDF that is distributed with Percona Server is a faster alternative.
7919
It is very simple to compile and install; look at the header in the source code
7920
for instructions. If it is installed, it is preferred over MD5(). You can also
7921
use the MURMUR_HASH() function if you compile and install that as a UDF; the
7922
source is also distributed with Percona Server, and it might be better than
7923
FNV1A_64().
7908
7924
7909
7925
=item --help
7910
7926
7912
7928
7913
7929
Show help and exit.
7914
7930
7931
=item --host
7932
7933
short form: -h; type: string; default: localhost; group: Connection
7934
7935
Host to connect to.
7936
7915
7937
=item --ignore-columns
7916
7938
7917
7939
type: Hash; group: Filter
7918
7940
7919
7941
Ignore this comma-separated list of columns when calculating the checksum.
7920
7942
7921
This option only affects the checksum when using the ACCUM or BIT_XOR
7922
L<"--algorithm">.
7923
7924
7943
=item --ignore-databases
7925
7944
7926
7945
type: Hash; group: Filter
7929
7948
7930
7949
=item --ignore-databases-regex
7931
7950
7932
type: string
7951
type: string; group: Filter
7933
7952
7934
7953
Ignore databases whose names match this Perl regex.
7935
7954
7943
7962
7944
7963
type: Hash; group: Filter
7945
7964
7946
Ignore this comma-separated list of tables.
7947
7948
Table names may be qualified with the database name.
7965
Ignore this comma-separated list of tables. Table names may be qualified with
7966
the database name. The L<"--replicate"> table is always automatically ignored.
7949
7967
7950
7968
=item --ignore-tables-regex
7951
7969
7952
type: string
7970
type: string; group: Filter
7953
7971
7954
7972
Ignore tables whose names match the Perl regex.
7955
7973
7956
=item --lock
7957
7958
Lock on master until done on slaves (implies L<"--slave-lag">).
7959
7960
This option can help you to get a consistent read on a master and many slaves.
7961
If you specify this option, pt-table-checksum will lock the table on the
7962
first server on the command line, which it assumes to be the master. It will
7963
keep this lock until the checksums complete on the other servers.
7964
7965
This option isn't very useful by itself, so you probably want to use L<"--wait">
7966
instead.
7967
7968
Note: if you're checksumming a slave against its master, you should use
7969
L<"--replicate">. In that case, there's no need for locking, waiting, or any of
7970
that.
7974
=item --lock-wait-timeout
7975
7976
type: int; default: 1
7977
7978
Set the session value of the innodb_lock_wait_timeout variable on the master host.
7979
Setting this option dynamically requires the InnoDB plugin, so this works only
7980
on newer InnoDB and MySQL versions. This option helps guard against long lock
7981
waits if the checksum queries become slow for some reason.
7971
7982
7972
7983
=item --max-lag
7973
7984
7974
type: time; group: Throttle; default: 1s
7975
7976
Suspend checksumming if the slave given by L<"--check-slave-lag"> lags.
7977
7978
This option causes pt-table-checksum to look at the slave every time it's about
7979
to checksum a chunk. If the slave's lag is greater than the option's value, or
7980
if the slave isn't running (so its lag is NULL), pt-table-checksum sleeps for
7981
L<"--check-interval"> seconds and then looks at the lag again. It repeats until
7982
the slave is caught up, then proceeds to checksum the chunk.
7983
7984
This option is useful to let you checksum data as fast as the slaves can handle
7985
it, assuming the slave you directed pt-table-checksum to monitor is
7986
representative of all the slaves that may be replicating from this server. It
7987
should eliminate the need for L<"--sleep"> or L<"--sleep-coef">.
7988
7989
=item --modulo
7990
7991
type: int
7992
7993
Do only every Nth chunk on chunked tables.
7994
7995
This option lets you checksum only some chunks of the table. This is a useful
7996
alternative to L<"--probability"> when you want to be sure you get full coverage
7997
in some specified number of runs; for example, you can do only every 7th chunk,
7998
and then use L<"--offset"> to rotate the modulo every day of the week.
7999
8000
Just like with L<"--probability">, a table that cannot be chunked is done every
8001
time.
8002
8003
=item --offset
8004
8005
type: string; default: 0
8006
8007
Modulo offset expression for use with L<"--modulo">.
8008
8009
The argument may be an SQL expression, such as C<WEEKDAY(NOW())> (which returns
8010
a number from 0 through 6). The argument is evaluated by MySQL. The result is
8011
used as follows: if chunk_num % L<"--modulo"> == L<"--offset">, the chunk will
8012
be checksummed.
8013
8014
=item --[no]optimize-xor
8015
8016
default: yes
8017
8018
Optimize BIT_XOR with user variables.
8019
8020
This option specifies to use user variables to reduce the number of times each
8021
row must be passed through the cryptographic hash function when you are using
8022
the BIT_XOR algorithm.
8023
8024
With the optimization, the queries look like this in pseudo-code:
8025
8026
SELECT CONCAT(
8027
BIT_XOR(SLICE_OF(@user_variable)),
8028
BIT_XOR(SLICE_OF(@user_variable)),
8029
...
8030
BIT_XOR(SLICE_OF(@user_variable := HASH(col1, col2... colN))));
8031
8032
The exact positioning of user variables and calls to the hash function is
8033
determined dynamically, and will vary between MySQL versions. Without the
8034
optimization, it looks like this:
8035
8036
SELECT CONCAT(
8037
BIT_XOR(SLICE_OF(MD5(col1, col2... colN))),
8038
BIT_XOR(SLICE_OF(MD5(col1, col2... colN))),
8039
...
8040
BIT_XOR(SLICE_OF(MD5(col1, col2... colN))));
8041
8042
The difference is the number of times all the columns must be mashed together
8043
and fed through the hash function. If you are checksumming really large
8044
columns, such as BLOB or TEXT columns, this might make a big difference.
7985
type: time; default: 1s; group: Throttle
7986
7987
Pause checksumming until all replicas' lag is less than this value. After each
7988
checksum query (each chunk), pt-table-checksum looks at the replication lag of
7989
all replicas to which it connects, using Seconds_Behind_Master. If any replica
7990
is lagging more than the value of this option, then pt-table-checksum will sleep
7991
for L<"--check-interval"> seconds, then check all replicas again. If you
7992
specify L<"--check-slave-lag">, then the tool only examines that server for
7993
lag, not all servers. If you want to control exactly which servers the tool
7994
monitors, use the DSN value to L<"--recursion-method">.
7995
7996
The tool waits forever for replicas to stop lagging. If any replica is
7997
stopped, the tool waits forever until the replica is started. Checksumming
7998
continues once all replicas are running and not lagging too much.
7999
8000
The tool prints progress reports while waiting. If a replica is stopped, it
8001
prints a progress report immediately, then again at every progress report
8002
interval.
8003
8004
=item --max-load
8005
8006
type: Array; default: Threads_running=25; group: Throttle
8007
8008
Examine SHOW GLOBAL STATUS after every chunk, and pause if any status variables
8009
are higher than the threshold. The option accepts a comma-separated list of
8010
MySQL status variables to check for a threshold. An optional C<=MAX_VALUE> (or
8011
C<:MAX_VALUE>) can follow each variable. If not given, the tool determines a
8012
threshold by examining the current value and increasing it by 20%.
8013
8014
For example, if you want the tool to pause when Threads_connected gets too high,
8015
you can specify "Threads_connected", and the tool will check the current value
8016
when it starts working and add 20% to that value. If the current value is 100,
8017
then the tool will pause when Threads_connected exceeds 120, and resume working
8018
when it is below 120 again. If you want to specify an explicit threshold, such
8019
as 110, you can use either "Threads_connected:110" or "Threads_connected=110".
8020
8021
The purpose of this option is to prevent the tool from adding too much load to
8022
the server. If the checksum queries are intrusive, or if they cause lock waits,
8023
then other queries on the server will tend to block and queue. This will
8024
typically cause Threads_running to increase, and the tool can detect that by
8025
running SHOW GLOBAL STATUS immediately after each checksum query finishes. If
8026
you specify a threshold for this variable, then you can instruct the tool to
8027
wait until queries are running normally again. This will not prevent queueing,
8028
however; it will only give the server a chance to recover from the queueing. If
8029
you notice queueing, it is best to decrease the chunk time.
8045
8030
8046
8031
=item --password
8047
8032
8068
8053
8069
8054
Port number to use for connection.
8070
8055
8071
=item --probability
8072
8073
type: int; default: 100
8074
8075
Checksums will be run with this percent probability.
8076
8077
This is an integer between 1 and 100. If 100, every chunk of every table will
8078
certainly be checksummed. If less than that, there is a chance that some chunks
8079
of some tables will be skipped. This is useful for routine jobs designed to
8080
randomly sample bits of tables without checksumming the whole server. By
8081
default, if a table is not chunkable, it will be checksummed every time even
8082
when the probability is less than 100. You can override this with
8083
L<"--single-chunk">.
8084
8085
See also L<"--modulo">.
8086
8087
8056
=item --progress
8088
8057
8089
8058
type: array; default: time,30
8090
8059
8091
Print progress reports to STDERR. Currently, this feature is only for when
8092
L<"--throttle-method"> waits for slaves to catch up.
8060
Print progress reports to STDERR.
8093
8061
8094
8062
The value is a comma-separated list with two parts. The first part can be
8095
8063
percentage, time, or iterations; the second part specifies how often an update
8096
should be printed, in percentage, seconds, or number of iterations.
8064
should be printed, in percentage, seconds, or number of iterations. The tool
8065
prints progress reports for a variety of time-consuming operations, including
8066
waiting for replicas to catch up if they become lagged.
8097
8067
8098
8068
=item --quiet
8099
8069
8100
short form: -q; group: Output
8101
8102
Do not print checksum results.
8103
8104
=item --recheck
8105
8106
Re-checksum chunks that L<"--replicate-check"> found to be different.
8070
short form: -q; cumulative: yes; default: 0
8071
8072
Print only the most important information (disables L<"--progress">).
8073
Specifying this option once causes the tool to print only errors, warnings, and
8074
tables that have checksum differences.
8075
8076
Specifying this option twice causes the tool to print only errors. In this
8077
case, you can use the tool's exit status to determine if there were any warnings
8078
or checksum differences.
8107
8079
8108
8080
=item --recurse
8109
8081
8110
type: int; group: Throttle
8111
8112
Number of levels to recurse in the hierarchy when discovering slaves.
8113
Default is infinite.
8114
8115
See L<"--recursion-method">.
8082
type: int
8083
8084
Number of levels to recurse in the hierarchy when discovering replicas.
8085
Default is infinite. See also L<"--recursion-method">.
8116
8086
8117
8087
=item --recursion-method
8118
8088
8119
8089
type: string
8120
8090
8121
Preferred recursion method for discovering slaves.
8122
8123
Possible methods are:
8091
Preferred recursion method for discovering replicas. Possible methods are:
8124
8092
8125
8093
METHOD USES
8126
=========== ================
8094
=========== ==================
8127
8095
processlist SHOW PROCESSLIST
8128
8096
hosts SHOW SLAVE HOSTS
8129
8130
The processlist method is preferred because SHOW SLAVE HOSTS is not reliable.
8131
However, the hosts method is required if the server uses a non-standard
8132
port (not 3306). Usually pt-table-checksum does the right thing and finds
8133
the slaves, but you may give a preferred method and it will be used first.
8134
If it doesn't find any slaves, the other methods will be tried.
8097
dsn=DSN DSNs from a table
8098
8099
The processlist method is the default, because SHOW SLAVE HOSTS is not
8100
reliable. However, the hosts method can work better if the server uses a
8101
non-standard port (not 3306). The tool usually does the right thing and
8102
finds all replicas, but you may give a preferred method and it will be used
8103
first.
8104
8105
The hosts method requires replicas to be configured with report_host,
8106
report_port, etc.
8107
8108
The dsn method is special: it specifies a table from which other DSN strings
8109
are read. The specified DSN must specify a D and t, or a database-qualified
8110
t. The DSN table should have the following structure:
8111
8112
CREATE TABLE `dsns` (
8113
`id` int(11) NOT NULL AUTO_INCREMENT,
8114
`parent_id` int(11) DEFAULT NULL,
8115
`dsn` varchar(255) NOT NULL,
8116
PRIMARY KEY (`id`)
8117
);
8118
8119
To make the tool monitor only the hosts 10.10.1.16 and 10.10.1.17 for
8120
replication lag and checksum differences, insert the values C<h=10.10.1.16> and
8121
C<h=10.10.1.17> into the table. Currently, the DSNs are ordered by id, but id
8122
and parent_id are otherwise ignored.
8135
8123
8136
8124
=item --replicate
8137
8125
8138
type: string
8139
8140
Replicate checksums to slaves (disallows --algorithm CHECKSUM).
8141
8142
This option enables a completely different checksum strategy for a consistent,
8143
lock-free checksum across a master and its slaves. Instead of running the
8144
checksum queries on each server, you run them only on the master. You specify a
8145
table, fully qualified in db.table format, to insert the results into. The
8146
checksum queries will insert directly into the table, so they will be replicated
8147
through the binlog to the slaves.
8148
8149
When the queries are finished replicating, you can run a simple query on each
8150
slave to see which tables have differences from the master. With the
8151
L<"--replicate-check"> option, pt-table-checksum can run the query for you to
8152
make it even easier. See L<"CONSISTENT CHECKSUMS"> for details.
8153
8154
If you find tables that have differences, you can use the chunk boundaries in a
8155
WHERE clause with L<pt-table-sync> to help repair them more efficiently. See
8156
L<pt-table-sync> for details.
8157
8158
The table must have at least these columns: db, tbl, chunk, boundaries,
8159
this_crc, master_crc, this_cnt, master_cnt. The table may be named anything you
8160
wish. Here is a suggested table structure, which is automatically used for
8161
L<"--create-replicate-table"> (MAGIC_create_replicate):
8162
8163
CREATE TABLE checksum (
8164
db char(64) NOT NULL,
8165
tbl char(64) NOT NULL,
8166
chunk int NOT NULL,
8167
boundaries char(100) NOT NULL,
8168
this_crc char(40) NOT NULL,
8169
this_cnt int NOT NULL,
8170
master_crc char(40) NULL,
8171
master_cnt int NULL,
8172
ts timestamp NOT NULL,
8173
PRIMARY KEY (db, tbl, chunk)
8174
);
8175
8176
Be sure to choose an appropriate storage engine for the checksum table. If you
8177
are checksumming InnoDB tables, for instance, a deadlock will break replication
8178
if the checksum table is non-transactional, because the transaction will still
8179
be written to the binlog. It will then replay without a deadlock on the
8180
slave and break replication with "different error on master and slave." This
8181
is not a problem with pt-table-checksum, it's a problem with MySQL
8182
replication, and you can read more about it in the MySQL manual.
8183
8184
This works only with statement-based replication (pt-table-checksum will switch
8185
the binlog format to STATEMENT for the duration of the session if your server
8186
uses row-based replication).
8187
8188
In contrast to running the tool against multiple servers at once, using this
8189
option eliminates the complexities of synchronizing checksum queries across
8190
multiple servers, which normally requires locking and unlocking, waiting for
8191
master binlog positions, and so on. Thus, it disables L<"--lock">, L<"--wait">,
8192
and L<"--slave-lag"> (but not L<"--check-slave-lag">, which is a way to throttle
8193
the execution speed).
8194
8195
The checksum queries actually do a REPLACE into this table, so existing rows
8196
need not be removed before running. However, you may wish to do this anyway to
8197
remove rows related to tables that don't exist anymore. The
8198
L<"--empty-replicate-table"> option does this for you.
8199
8200
Since the table must be qualified with a database (e.g. C<db.checksums>),
8201
pt-table-checksum will only USE this database. This may be important if any
8202
replication options are set because it could affect whether or not changes
8203
to the table are replicated.
8204
8205
If the slaves have any --replicate-do-X or --replicate-ignore-X options, you
8206
should be careful not to checksum any databases or tables that exist on the
8207
master and not the slaves. Changes to such tables may not normally be executed
8208
on the slaves because of the --replicate options, but the checksum queries
8209
modify the contents of the table that stores the checksums, not the tables whose
8210
data you are checksumming. Therefore, these queries will be executed on the
8211
slave, and if the table or database you're checksumming does not exist, the
8212
queries will cause replication to fail. For more information on replication
8213
rules, see L<http://dev.mysql.com/doc/en/replication-rules.html>.
8214
8215
The table specified by L<"--replicate"> will never be checksummed itself.
8216
8217
=item --replicate-check
8218
8219
type: int
8220
8221
Check results in L<"--replicate"> table, to the specified depth. You must use
8222
this after you run the tool normally; it skips the checksum step and only checks
8223
results.
8224
8225
It recursively finds differences recorded in the table given by
8226
L<"--replicate">. It recurses to the depth you specify: 0 is no recursion
8227
(check only the server you specify), 1 is check the server and its slaves, 2 is
8228
check the slaves of its slaves, and so on.
8229
8230
It finds differences by running the query shown in L<"CONSISTENT CHECKSUMS">,
8231
and prints results, then exits after printing. This is just a convenient way of
8232
running the query so you don't have to do it manually.
8233
8234
The output is one informational line per slave host, followed by the results
8235
of the query, if any. If L<"--quiet"> is specified, there is no output. If
8236
there are no differences between the master and any slave, there is no output.
8237
If any slave has chunks that differ from the master, pt-table-checksum's
8238
exit status is 1; otherwise it is 0.
8239
8240
This option makes C<pt-table-checksum> look for slaves by running C<SHOW
8241
PROCESSLIST>. If it finds connections that appear to be from slaves, it derives
8242
connection information for each slave with the same default-and-override method
8243
described in L<"SPECIFYING HOSTS">.
8244
8245
If C<SHOW PROCESSLIST> doesn't return any rows, C<pt-table-checksum> looks at
8246
C<SHOW SLAVE HOSTS> instead. The host and port, and user and password if
8247
available, from C<SHOW SLAVE HOSTS> are combined into a DSN and used as the
8248
argument. This requires slaves to be configured with C<report-host>,
8249
C<report-port> and so on.
8250
8251
This requires the @@SERVER_ID system variable, so it works only on MySQL
8252
3.23.26 or newer.
8126
type: string; default: percona.checksums
8127
8128
Write checksum results to this table. The replicate table must have this
8129
structure (MAGIC_create_replicate):
8130
8131
CREATE TABLE checksums (
8132
db char(64) NOT NULL,
8133
tbl char(64) NOT NULL,
8134
chunk int NOT NULL,
8135
chunk_time float NULL,
8136
chunk_index varchar(200) NULL,
8137
lower_boundary text NULL,
8138
upper_boundary text NULL,
8139
this_crc char(40) NOT NULL,
8140
this_cnt int NOT NULL,
8141
master_crc char(40) NULL,
8142
master_cnt int NULL,
8143
ts timestamp NOT NULL,
8144
PRIMARY KEY (db, tbl, chunk),
8145
INDEX ts_db_tbl (ts, db, tbl)
8146
) ENGINE=InnoDB;
8147
8148
By default, L<"--[no]create-replicate-table"> is true, so the database and
8149
the table specified by this option are created automatically if they do not
8150
exist.
8151
8152
Be sure to choose an appropriate storage engine for the replicate table. If you
8153
are checksumming InnoDB tables, and you use MyISAM for this table, a deadlock
8154
will break replication, because the mixture of transactional and
8155
non-transactional tables in the checksum statements will cause it to be written
8156
to the binlog even though it had an error. It will then replay without a
8157
deadlock on the replicas, and break replication with "different error on master
8158
and slave." This is not a problem with pt-table-checksum; it's a problem with
8159
MySQL replication, and you can read more about it in the MySQL manual.
8160
8161
The replicate table is never checksummed (the tool automatically adds this
8162
table to L<"--ignore-tables">).
8163
8164
=item --[no]replicate-check
8165
8166
default: yes
8167
8168
Check replicas for data differences after finishing each table. The tool finds
8169
differences by executing a simple SELECT statement on all detected replicas.
8170
The query compares the replica's checksum results to the master's checksum
8171
results. It reports differences in the DIFFS column of the output.
8172
8173
=item --replicate-check-only
8174
8175
Check replicas for consistency without executing checksum queries.
8176
This option is used only with L<"--[no]replicate-check">. If specified,
8177
pt-table-checksum doesn't checksum any tables. It checks replicas for
8178
differences found by previous checksumming, and then exits. It might be useful
8179
if you run pt-table-checksum quietly in a cron job, for example, and later want
8180
a report on the results of the cron job, perhaps to implement a Nagios check.
8253
8181
8254
8182
=item --replicate-database
8255
8183
8256
8184
type: string
8257
8185
8258
C<USE> only this database with L<"--replicate">. By default, pt-table-checksum
8259
executes USE to set its default database to the database that contains the table
8260
it's currently working on. It changes its default database as it works on
8261
different tables. This is is a best effort to avoid problems with replication
8262
filters such as binlog_ignore_db and replicate_ignore_db. However, replication
8263
filters can create a situation where there simply is no one right way to do
8264
things. Some statements might not be replicated, and others might cause
8265
replication to fail on the slaves. In such cases, it is up to the user to
8266
specify a safe default database. This option specifies a default database that
8267
pt-table-checksum selects with USE, and never changes afterwards. See also
8268
<L"--[no]check-replication-filters">.
8186
USE only this database. By default, pt-table-checksum executes USE to select
8187
the database that contains the table it's currently working on. This is is a
8188
best effort to avoid problems with replication filters such as binlog_ignore_db
8189
and replicate_ignore_db. However, replication filters can create a situation
8190
where there simply is no one right way to do things. Some statements might not
8191
be replicated, and others might cause replication to fail. In such cases, you
8192
can use this option to specify a default database that pt-table-checksum selects
8193
with USE, and never changes. See also <L"--[no]check-replication-filters">.
8269
8194
8270
8195
=item --resume
8271
8196
8272
type: string
8273
8274
Resume checksum using given output file from a previously interrupted run.
8275
8276
The given output file should be the literal output from a previous run of
8277
C<pt-table-checksum>. For example:
8278
8279
pt-table-checksum host1 host2 -C 100 > checksum_results.txt
8280
pt-table-checksum host1 host2 -C 100 --resume checksum_results.txt
8281
8282
The command line options given to the first run and the resumed run must
8283
be identical (except, of course, for --resume). If they are not, the result
8284
will be unpredictable and probably wrong.
8285
8286
L<"--resume"> does not work with L<"--replicate">; for that, use
8287
L<"--resume-replicate">.
8288
8289
=item --resume-replicate
8290
8291
Resume L<"--replicate">.
8292
8293
This option resumes a previous checksum operation using L<"--replicate">.
8294
It is like L<"--resume"> but does not require an output file. Instead,
8295
it uses the checksum table given to L<"--replicate"> to determine where to
8296
resume the checksum operation.
8297
8298
=item --save-since
8299
8300
When L<"--arg-table"> and L<"--since"> are given, save the current L<"--since">
8301
value into that table's C<since> column after checksumming. In this way you can
8302
incrementally checksum tables by starting where the last one finished.
8303
8304
The value to be saved could be the current timestamp, or it could be the maximum
8305
existing value of the column given by L<"--since-column">. It depends on what
8306
options are in effect. See the description of L<"--since"> to see how
8307
timestamps are different from ordinary values.
8308
8309
=item --schema
8310
8311
Checksum C<SHOW CREATE TABLE> instead of table data.
8197
Resume checksumming from the last completed chunk (disables
8198
L<"--[no]empty-replicate-table">). If the tool stops before it checksums all
8199
tables, this option makes checksumming resume from the last chunk of the last
8200
table that it finished.
8201
8202
=item --retries
8203
8204
type: int; default: 2
8205
8206
Retry a chunk this many times when there is a nonfatal error. Nonfatal errors
8207
are problems such as a lock wait timeout or the query being killed.
8312
8208
8313
8209
=item --separator
8314
8210
8315
8211
type: string; default: #
8316
8212
8317
The separator character used for CONCAT_WS().
8318
8319
This character is used to join the values of columns when checksumming with
8320
L<"--algorithm"> of BIT_XOR or ACCUM.
8213
The separator character used for CONCAT_WS(). This character is used to join
8214
the values of columns when checksumming.
8321
8215
8322
8216
=item --set-vars
8323
8217
8326
8220
Set these MySQL variables. Immediately after connecting to MySQL, this
8327
8221
string will be appended to SET and executed.
8328
8222
8329
=item --since
8330
8331
type: string
8332
8333
Checksum only data newer than this value.
8334
8335
If the table is chunk-able or nibble-able, this value will apply to the first
8336
column of the chunked or nibbled index.
8337
8338
This is not too different to L<"--where">, but instead of universally applying a
8339
WHERE clause to every table, it selectively finds the right column to use and
8340
applies it only if such a column is found. See also L<"--since-column">.
8341
8342
The argument may be an expression, which is evaluated by MySQL. For example,
8343
you can specify C<CURRENT_DATE - INTERVAL 7 DAY> to get the date of one week
8344
ago.
8345
8346
A special bit of extra magic: if the value is temporal (looks like a date or
8347
datetime), then the table is checksummed only if the create time (or last
8348
modified time, for tables that report the last modified time, such as MyISAM
8349
tables) is newer than the value. In this sense it's not applied as a WHERE
8350
clause at all.
8351
8352
=item --since-column
8353
8354
type: string
8355
8356
The column name to be used for L<"--since">.
8357
8358
The default is for the tool to choose the best one automatically. If you
8359
specify a value, that will be used if possible; otherwise the best
8360
auto-determined one; otherwise none. If the column doesn't exist in the table,
8361
it is just ignored.
8362
8363
=item --single-chunk
8364
8365
Permit skipping with L<"--probability"> if there is only one chunk.
8366
8367
Normally, if a table isn't split into many chunks, it will always be
8368
checksummed regardless of L<"--probability">. This setting lets the
8369
probabilistic behavior apply to tables that aren't divided into chunks.
8370
8371
=item --slave-lag
8372
8373
group: Output
8374
8375
Report replication delay on the slaves.
8376
8377
If this option is enabled, the output will show how many seconds behind the
8378
master each slave is. This can be useful when you want a fast, parallel,
8379
non-blocking checksum, and you know your slaves might be delayed relative to the
8380
master. You can inspect the results and make an educated guess whether any
8381
discrepancies on the slave are due to replication delay instead of corrupt data.
8382
8383
If you're using L<"--replicate">, a slave that is delayed relative to the master
8384
does not invalidate the correctness of the results, so this option is disabled.
8385
8386
=item --sleep
8387
8388
type: int; group: Throttle
8389
8390
Sleep time between checksums.
8391
8392
If this option is specified, pt-table-checksum will sleep the specified
8393
number of seconds between checksums. That is, it will sleep between every
8394
table, and if you specify L<"--chunk-size">, it will also sleep between chunks.
8395
8396
This is a very crude way to throttle checksumming; see L<"--sleep-coef"> and
8397
L<"--check-slave-lag"> for techniques that permit greater control.
8398
8399
=item --sleep-coef
8400
8401
type: float; group: Throttle
8402
8403
Calculate L<"--sleep"> as a multiple of the last checksum time.
8404
8405
If this option is specified, pt-table-checksum will sleep the amount of
8406
time elapsed during the previous checksum, multiplied by the specified
8407
coefficient. This option is ignored if L<"--sleep"> is specified.
8408
8409
This is a slightly more sophisticated way to throttle checksum speed: sleep a
8410
varying amount of time between chunks, depending on how long the chunks are
8411
taking. Even better is to use L<"--check-slave-lag"> if you're checksumming
8412
master/slave replication.
8413
8414
8223
=item --socket
8415
8224
8416
8225
short form: -S; type: string; group: Connection
8417
8226
8418
8227
Socket file to use for connection.
8419
8228
8420
=item --tab
8421
8422
group: Output
8423
8424
Print tab-separated output, not column-aligned output.
8425
8426
8229
=item --tables
8427
8230
8428
8231
short form: -t; type: hash; group: Filter
8429
8232
8430
Do only this comma-separated list of tables.
8431
8233
Checksum only this comma-separated list of tables.
8432
8234
Table names may be qualified with the database name.
8433
8235
8434
8236
=item --tables-regex
8435
8237
8436
type: string
8437
8438
Only checksum tables whose names match this Perl regex.
8439
8440
=item --throttle-method
8441
8442
type: string; default: none; group: Throttle
8443
8444
Throttle checksumming when doing L<"--replicate">.
8445
8446
At present there is only one method: C<slavelag>. When L<"--replicate"> is
8447
used, pt-table-checksum automatically sets L<"--throttle-method"> to
8448
C<slavelag> and discovers every slave and throttles checksumming if any slave
8449
lags more than L<"--max-lag">. Specify C<-throttle-method none> to disable
8450
this behavior completely, or specify L<"--check-slave-lag"> and
8451
pt-table-checksum will only check that slave.
8452
8453
See also L<"--recurse"> and L<"--recursion-method">.
8238
type: string; group: Filter
8239
8240
Checksum only tables whose names match this Perl regex.
8454
8241
8455
8242
=item --trim
8456
8243
8457
Trim C<VARCHAR> columns (helps when comparing 4.1 to >= 5.0).
8458
8459
This option adds a C<TRIM()> to C<VARCHAR> columns in C<BIT_XOR> and C<ACCUM>
8460
modes.
8461
8244
Add TRIM() to VARCHAR columns (helps when comparing 4.1 to >= 5.0).
8462
8245
This is useful when you don't care about the trailing space differences between
8463
MySQL versions which vary in their handling of trailing spaces. MySQL 5.0 and
8464
later all retain trailing spaces in C<VARCHAR>, while previous versions would
8465
remove them.
8466
8467
=item --unchunkable-tables
8468
8469
group: Safety
8470
8471
Checksum tables that cannot be chunked when L<"--chunk-size"> is specified.
8472
8473
By default pt-table-checksum will not checksum a table that cannot be chunked
8474
when L<"--chunk-size"> is specified because this might result in a huge,
8475
non-chunkable table being checksummed in one huge, memory-intensive chunk.
8476
8477
Specifying this option allows checksumming tables that cannot be chunked.
8478
Be careful when using this option! Make sure any non-chunkable tables
8479
are not so large that they will cause the tool to consume too much memory
8480
or CPU.
8481
8482
See also L<"--chunk-size-limit">.
8483
8484
=item --[no]use-index
8485
8486
default: yes
8487
8488
Add FORCE INDEX hints to SQL statements.
8489
8490
By default C<pt-table-checksum> adds an index hint (C<FORCE INDEX> for MySQL
8491
v4.0.9 and newer, C<USE INDEX> for older MySQL versions) to each SQL statement
8492
to coerce MySQL into using the L<"--chunk-index"> (whether the index is
8493
specified by the option or auto-detected). Specifying C<--no-use-index> causes
8494
C<pt-table-checksum> to omit index hints.
8246
MySQL versions that vary in their handling of trailing spaces. MySQL 5.0 and
8247
later all retain trailing spaces in VARCHAR, while previous versions would
8248
remove them. These differences will cause false checksum differences.
8495
8249
8496
8250
=item --user
8497
8251
8499
8253
8500
8254
User for login if not current user.
8501
8255
8502
=item --[no]verify
8503
8504
default: yes
8505
8506
Verify checksum compatibility across servers.
8507
8508
This option runs a trivial checksum on all servers to ensure they have
8509
compatible CONCAT_WS() and cryptographic hash functions.
8510
8511
Versions of MySQL before 4.0.14 will skip empty strings and NULLs in
8512
CONCAT_WS, and others will only skip NULLs. The two kinds of behavior will
8513
produce different results if you have any columns containing the empty string
8514
in your table. If you know you don't (for instance, all columns are
8515
integers), you can safely disable this check and you will get a reliable
8516
checksum even on servers with different behavior.
8517
8518
8256
=item --version
8519
8257
8520
8258
group: Help
8521
8259
8522
8260
Show version and exit.
8523
8261
8524
=item --wait
8525
8526
short form: -w; type: time
8527
8528
Wait this long for slaves to catch up to their master (implies L<"--lock">
8529
L<"--slave-lag">).
8530
8531
Note: the best way to verify that a slave is in sync with its master is to use
8532
L<"--replicate"> instead. The L<"--wait"> option is really only useful if
8533
you're trying to compare masters and slaves without using L<"--replicate">,
8534
which is possible but complex and less efficient in some ways.
8535
8536
This option helps you get a consistent checksum across a master server and its
8537
slaves. It combines locking and waiting to accomplish this. First it locks the
8538
table on the master (the first server on the command line). Then it finds the
8539
master's binlog position. Checksums on slaves will be deferred until they reach
8540
the same binlog position.
8541
8542
The argument to the option is the number of seconds to wait for the slaves to
8543
catch up to the master. It is actually the argument to MASTER_POS_WAIT(). If
8544
the slaves don't catch up to the master within this time, they will unblock
8545
and go ahead with the checksum. You can tell whether this happened by
8546
examining the STAT column in the output, which is the return value of
8547
MASTER_POS_WAIT().
8548
8549
8262
=item --where
8550
8263
8551
8264
type: string
8552
8265
8553
Do only rows matching this C<WHERE> clause (disallows L<"--algorithm"> CHECKSUM).
8554
8555
You can use this option to limit the checksum to only part of the table. This
8556
is particularly useful if you have append-only tables and don't want to
8557
constantly re-check all rows; you could run a daily job to just check
8558
yesterday's rows, for instance.
8266
Do only rows matching this WHERE clause. You can use this option to limit
8267
the checksum to only part of the table. This is particularly useful if you have
8268
append-only tables and don't want to constantly re-check all rows; you could run
8269
a daily job to just check yesterday's rows, for instance.
8559
8270
8560
8271
This option is much like the -w option to mysqldump. Do not specify the WHERE
8561
keyword. You may need to quote the value. Here is an example:
8562
8563
pt-table-checksum --where "foo=bar"
8564
8565
=item --[no]zero-chunk
8566
8567
default: yes
8568
8569
Add a chunk for rows with zero or zero-equivalent values. The only has an
8570
effect when L<"--chunk-size"> is specified. The purpose of the zero chunk
8571
is to capture a potentially large number of zero values that would imbalance
8572
the size of the first chunk. For example, if a lot of negative numbers were
8573
inserted into an unsigned integer column causing them to be stored as zeros,
8574
then these zero values are captured by the zero chunk instead of the first
8575
chunk and all its non-zero values.
8272
keyword. You might need to quote the value. Here is an example:
8273
8274
pt-table-checksum --where "ts > CURRENT_DATE - INTERVAL 1 DAY"
8576
8275
8577
8276
=back
8578
8277
8594
8293
8595
8294
=item * D
8596
8295
8597
dsn: database; copy: yes
8296
copy: no
8598
8297
8599
Default database.
8298
DSN table database.
8600
8299
8601
8300
=item * F
8602
8301
8603
dsn: mysql_read_default_file; copy: yes
8302
dsn: mysql_read_default_file; copy: no
8604
8303
8605
8304
Only read default options from the given file
8606
8305
8624
8323
8625
8324
=item * S
8626
8325
8627
dsn: mysql_socket; copy: yes
8326
dsn: mysql_socket; copy: no
8628
8327
8629
8328
Socket file to use for connection.
8630
8329
8330
=item * t
8331
8332
copy: no
8333
8334
DSN table table.
8335
8631
8336
=item * u
8632
8337
8633
8338
dsn: user; copy: yes
8695
8400
8696
8401
=head1 AUTHORS
8697
8402
8698
Baron Schwartz
8403
Baron Schwartz and Daniel Nichter
8699
8404
8700
8405
=head1 ACKNOWLEDGMENTS
8701
8406
8713
8418
8714
8419
=head1 COPYRIGHT, LICENSE, AND WARRANTY
8715
8420
8716
This program is copyright 2007-2011 Baron Schwartz, 2011 Percona Inc.
8421
This program is copyright 2007-2011 Baron Schwartz, 2011-2012 Percona Inc.
8717
8422
Feedback and improvements are welcome.
8718
8423
8719
8424
THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
8732
8437
8733
8438
=head1 VERSION
8734
8439
8735
Percona Toolkit v0.9.5 released 2011-08-04
8440
pt-table-checksum 2.0.3
8736
8441
8737
8442
=cut
Loggerhead is a web-based interface for Breezy
Version: 2.0.1