~akopytov/percona-xtrabackup/bug950334-2.1

#!/usr/bin/env perl

#

# A script for making backups of InnoDB and MyISAM tables, indexes and .frm

# files.

#

# Copyright 2003, 2009 Innobase Oy and Percona Ireland Ltd 2009-2012.

# All Rights Reserved.

#

use strict;

use Getopt::Long;

use File::Spec;

use Pod::Usage qw(pod2usage);

use POSIX "strftime";

use POSIX ":sys_wait_h";

use FileHandle;

use File::Basename;

use File::Temp;

use File::Find;

use File::Copy;

use English qw(-no_match_vars);

use Time::HiRes qw(usleep);

# version of this script

my $innobackup_version = '1.5.1-xtrabackup';

my $innobackup_script = basename($0);

# copyright notice

my $copyright_notice =

"InnoDB Backup Utility v${innobackup_version}; Copyright 2003, 2009 Innobase Oy

and Percona Ireland Ltd 2009-2012.  All Rights Reserved.

This software is published under

the GNU GENERAL PUBLIC LICENSE Version 2, June 1991.

";

# required Perl version (5.005)

my @required_perl_version = (5, 0, 5);

my $required_perl_version_old_style = 5.005;

# force flush after every write and print

$| = 1;

# disable nlink count optimization, see File::Find documentation for details

$File::Find::dont_use_nlink=1;

######################################################################

# modifiable parameters

######################################################################

# maximum number of files in a database directory which are

# separately printed when a backup is made

my $backup_file_print_limit = 9;

# timeout in seconds for a reply from mysql

my $mysql_response_timeout = 900;

# default compression level (this is an argument to ibbackup)

my $default_compression_level = 1;

# time in seconds after which a dummy query is sent to mysql server

# in order to keep the database connection alive

my $mysql_keep_alive = 5;

######################################################################

# end of modifiable parameters

######################################################################

# command line options

my $option_help = '';

my $option_version = '';

my $option_apply_log = '';

my $option_redo_only = '';

my $option_copy_back = '';

my $option_move_back = '';

my $option_include = '';

my $option_databases = '';

my $option_tables_file = '';

my $option_throttle = '';

my $option_sleep = '';

my $option_compress = 999;

my $option_compress_threads = 1;

my $option_uncompress = '';

my $option_export = '';

my $option_use_memory = '';

my $option_mysql_password = '';

my $option_mysql_user = '';

my $option_mysql_port = '';

my $option_mysql_socket = '';

my $option_mysql_host = '';

my $option_defaults_group = 'mysqld';

my $option_no_timestamp = '';

my $option_slave_info = '';

my $option_galera_info = '';

my $option_no_lock = '';

my $option_ibbackup_binary = 'autodetect';

my $option_defaults_file = '';

my $option_defaults_extra_file = '';

my $option_incremental = '';

my $option_incremental_basedir = '';

my $option_incremental_dir = '';

my $option_incremental_lsn = '';

my $option_extra_lsndir = '';

my $option_remote_host = '';

my $option_rsync = '';

my $option_stream = '';

my $stream_cmd = '';

my $option_tmpdir = '';

my $option_tar4ibd = '';

my $option_scp_opt = '-Cp -c arcfour';

my $option_ssh_opt = '';

my $option_parallel = '';

my $option_safe_slave_backup = '';

my $option_safe_slave_backup_timeout = 300;

# name of the my.cnf configuration file

#my $config_file = '';

# root of the backup directory

my $backup_root = '';

# backup directory pathname

my $backup_dir = '';

# name of the ibbackup suspend-at-end file

my $suspend_file = '';

# name of the temporary transaction log file during the backup

my $tmp_logfile = '';

# home directory of innoDB log files

my $innodb_log_group_home_dir = '';

# backup my.cnf file

my $backup_config_file = '';

# whether slave SQL thread is running when wait_for_safe_slave() is called

my $sql_thread_started = 0;

# options from the options file

my %config;

# options from the backup options file

#my %backup_config;

# list of databases to be included in a backup

my %databases_list;

# list of tables to be included in a backup

my %table_list;

# prefix for output lines

my $prefix = "$innobackup_script:";

# process id of mysql client program (runs as a child process of this script)

my $mysql_pid = '';

# mysql server version string

my $mysql_server_version = '';

# name of the file where stderr of mysql process is directed

my $mysql_stderr_fh = File::Temp->new();

my $mysql_stderr = $mysql_stderr_fh->filename;

# name of the file where stdout of mysql process is directed

my $mysql_stdout_fh = File::Temp->new();

my $mysql_stdout = $mysql_stdout_fh->filename;

# name of the file where binlog position info is written

my $binlog_info;

# name of the file where galera position info is written

my $galera_info;

# name of the file where slave info is written

my $slave_info;

# mysql binlog position as given by "SHOW MASTER STATUS" command

my $mysql_binlog_position = '';

# mysql master's binlog position as given by "SHOW SLAVE STATUS" command

# run on a slave server

my $mysql_slave_position = '';

# process id of ibbackup program (runs as a child process of this script)

my $ibbackup_pid = '';

# a counter for numbering mysql connection checks

my $hello_id = 0;

# the request which has been sent to mysqld, but to which

# mysqld has not yet replied. Empty string denotes that no

# request has been sent to mysqld or that mysqld has replied

# to all requests.

my $current_mysql_request = '';

# escape sequences for options files

my %option_value_escapes = ('b' => "\b",

                            't' => "\t",

                            'n' => "\n",

                            'r' => "\r",

                            "\\" => "\\",

                            's' => ' ');

# signal that is sent to child processes when they are killed

my $kill_signal = 15;

# current local time

my $now;

# incremental backup base directory

my $incremental_basedir = '';

my $src_name;

my $dst_name;

my $win = ($^O eq 'MSWin32' ? 1 : 0);

my $CP_CMD = ($win eq 1 ? "copy /Y" : "cp -p");

my $xtrabackup_binary_file = 'xtrabackup_binary';

my $xtrabackup_pid_file = 'xtrabackup_pid';

my %rsync_files_hash;

my $copy_dir_src;

my $copy_dir_dst;

my $copy_dir_exclude_regexp;

my $copy_dir_overwrite;

######################################################################

# program execution begins here

######################################################################

# check command-line args

check_args();

# print program version and copyright

print_version();

# initialize global variables and perform some checks

if ($option_copy_back || $option_move_back) {

    $option_ibbackup_binary = 'xtrabackup' if ($option_ibbackup_binary eq 'autodetect');

} elsif ($option_apply_log) {

	# Read XtraBackup version from backup dir

	if (-e "$backup_dir/$xtrabackup_binary_file") {

		# Read XtraBackup version from file

		open XTRABACKUP_BINARY, "$backup_dir/$xtrabackup_binary_file"

			or die "Cannot open file $backup_dir/$xtrabackup_binary_file: $!\n";

		$option_ibbackup_binary = <XTRABACKUP_BINARY>;

		close XTRABACKUP_BINARY;

		}

	else {

		if( $option_ibbackup_binary eq "autodetect" ){

			# Try to connect MySQL and get the version

			print STDERR "option_ibbackup_binary is autodetect, trying to connect to MySQL\n";

			my $options = get_mysql_options();

			$mysql_pid = open(*MYSQL_WRITER, "| mysql $options >$mysql_stdout 2>$mysql_stderr ");

			print STDERR "Connected to MySQL with pid $mysql_pid\n";

			sleep 1;

			if ($mysql_pid && $mysql_pid == waitpid($mysql_pid, &WNOHANG)) {

				my $reason = `cat $mysql_stderr`;

				$mysql_pid = '';

				# Failed to connect to MySQL

				die "Failed to connect to MySQL server to detect version.\nYou must set xtrabackup version to use with --ibbackup option.\nPossible values are xtrabackup_51 (for MySQL 5.0 and 5.1), xtrabackup_55 (for MySQL 5.5) or xtrabackup (for MySQL 5.1 with InnoDB plugin or Percona Server)\n";

				}

			else{

				mysql_close();

				print STDERR "Connected successfully\n";

				$option_ibbackup_binary = set_xtrabackup_version();

				}

			}

		}

} elsif ($option_ibbackup_binary eq 'autodetect') {

    $option_ibbackup_binary = set_xtrabackup_version();

}

init();

my $ibbackup_exit_code = 0;

if ($option_copy_back) {

    # copy files from backup directory back to their original locations

    copy_back(0);

} elsif ($option_move_back) {

    # move files from backup directory back to their original locations

    copy_back(1);

} elsif ($option_apply_log) {

    # expand data files in backup directory by applying log files to them

    apply_log();

} else {

    # make a backup of InnoDB and MyISAM tables, indexes and .frm files.

    $ibbackup_exit_code = backup();

    if ($option_remote_host) {

      open(XTRABACKUP_BINARY,

	   "| ssh $option_ssh_opt $option_remote_host 'cat > $backup_dir/$xtrabackup_binary_file'")

	|| die "Failed to open file '$option_remote_host:$backup_dir/$xtrabackup_binary_file': $!";

    } elsif ($option_stream) {

      open XTRABACKUP_BINARY, "> $option_tmpdir/$xtrabackup_binary_file"

    	|| die "Cannot open file $option_tmpdir/$xtrabackup_binary_file: $!\n";

    } else {

      open XTRABACKUP_BINARY, "> $backup_dir/$xtrabackup_binary_file"

    	|| die "Cannot open file $backup_dir/$xtrabackup_binary_file: $!\n";

    }

    print XTRABACKUP_BINARY $option_ibbackup_binary;

    close XTRABACKUP_BINARY;

    if ($option_stream) {

      system("cd $option_tmpdir; $stream_cmd $xtrabackup_binary_file")

	&& die "Failed to stream $xtrabackup_binary_file: $!";

      unlink "$option_tmpdir/$xtrabackup_binary_file";

    }

}

$now = current_time();

if ($option_stream eq 'tar') {

   print STDERR "$prefix You must use -i (--ignore-zeros) option for extraction of the tar stream.\n";

}

if ( $ibbackup_exit_code == 0 ) {

   # program has completed successfully

   print STDERR "$now  $prefix completed OK!\n";

}

else {

   print STDERR "$now  $prefix $option_ibbackup_binary failed! (exit code $ibbackup_exit_code)  The backup may not be complete.\n";

}

exit $ibbackup_exit_code;

######################################################################

# end of program execution

######################################################################

#

# print_version subroutine prints program version and copyright.

#

sub print_version {

    printf(STDERR $copyright_notice);

}

#

# usage subroutine prints instructions of how to use this program to stdout.

#

sub usage {

   my $msg = shift || '';

   pod2usage({ -msg => $msg, -verbose => 1});

   return 0;

}

#

# return current local time as string in form "070816 12:23:15"

#

sub current_time {

    return strftime("%y%m%d %H:%M:%S", localtime());

}

#

# Die subroutine kills all child processes and exits this process.

# This subroutine takes the same argument as the built-in die function.

#    Parameters:

#       message   string which is printed to stdout

#

sub Die {

    my $message = shift;

    my $extra_info = '';

    # kill all child processes of this process

    kill_child_processes();

    if ($current_mysql_request) {

        $extra_info = " while waiting for reply to MySQL request:" .

            " '$current_mysql_request'";

    }

    die "$prefix Error: $message$extra_info";

}

#

# backup subroutine makes a backup of InnoDB and MyISAM tables, indexes and 

# .frm files. It connects to the database server and runs ibbackup as a child

# process.

#

sub backup {

    my $orig_datadir = get_option(\%config, $option_defaults_group, 'datadir');

    # check that we can connect to the database. This done by

    # connecting, issuing a query, and closing the connection.

    mysql_open();

    mysql_close();

    # start ibbackup as a child process

    start_ibbackup();

    # wait for ibbackup to suspend itself

    if (!$option_remote_host) {

        wait_for_ibbackup_suspend();

    }

    # connect to database

    mysql_open();

    if ($option_safe_slave_backup) {

      wait_for_safe_slave();

    }

    # flush tables with read lock

    if (!$option_no_lock) {

        # make a prep copy before locking tables, if using rsync

        backup_files(1);

        # flush tables with read lock

        mysql_lockall();

    }

    if ($option_slave_info) {

        write_slave_info();

    }

    # backup non-InnoDB files and tables

    # (or finalize the backup by syncing changes if using rsync)

    backup_files(0);

    # resume ibbackup and wait till log copying is finished

    resume_ibbackup();

    # release read locks on all tables

    mysql_unlockall() if !$option_no_lock;

    my $ibbackup_exit_code = wait_for_ibbackup_finish();

    if ( $option_safe_slave_backup && $sql_thread_started) {

      print STDERR "$prefix: Starting slave SQL thread\n";

      mysql_send('START SLAVE SQL_THREAD;');

    }

    # Close the DB connection

    mysql_close();

    # copy ib_lru_dump

    if (-e "$orig_datadir/ib_lru_dump") {

        if ($option_remote_host) {

            print STDERR "$prefix Backing up file 'ib_lru_dump'\n";

            system("scp $option_scp_opt '$orig_datadir/ib_lru_dump' '$option_remote_host:$backup_dir/ib_lru_dump'")

                and Die "Failed to scp file 'ib_lru_dump': $!";

        } elsif ($option_stream) {

            print STDERR "$prefix Backing up as tar stream 'ib_lru_dump'\n";

            system("cd $orig_datadir; $stream_cmd ib_lru_dump")

                and Die "Failed to stream 'ib_lru_dump': $!";

        } elsif (!$option_rsync) {

            my $src_name = escape_path("$orig_datadir/ib_lru_dump");

            my $dst_name = escape_path("$backup_dir/ib_lru_dump");

            system("$CP_CMD \"$src_name\" \"$dst_name\"")

                and Die "Failed to copy file 'ib_lru_dump': $!";

        }

    }

    if ($option_remote_host) {

        system("scp $option_scp_opt '$tmp_logfile' '$option_remote_host:$backup_dir/xtrabackup_logfile'")

            and Die "Failed to scp file '$option_remote_host:$backup_dir/xtrabackup_logfile': $!";

        unlink $tmp_logfile || Die "Failed to delete '$tmp_logfile': $!";

        system("scp $option_scp_opt '$option_tmpdir/xtrabackup_checkpoints' '$option_remote_host:$backup_dir/xtrabackup_checkpoints'")

            and Die "Failed to scp file '$option_remote_host:$backup_dir/xtrabackup_checkpoints': $!";

        unlink "$option_tmpdir/xtrabackup_checkpoints" || Die "Failed to delete '$option_tmpdir/xtrabackup_checkpoints': $!";

    }

    print STDERR "\n$prefix Backup created in directory '$backup_dir'\n";

    if ($mysql_binlog_position) {

        print STDERR "$prefix MySQL binlog position: $mysql_binlog_position\n";

    }

    if ($mysql_slave_position && $option_slave_info) {

        print STDERR "$prefix MySQL slave binlog position: $mysql_slave_position\n";

    }

    return $ibbackup_exit_code;

}

#

# are_equal_innodb_data_file_paths subroutine checks if the given

# InnoDB data file option values are equal.

#   Parameters:

#     str1    InnoDB data file path option value

#     str2    InnoDB data file path option value

#   Return value:

#     1  if values are equal

#     0  otherwise

#

sub are_equal_innodb_data_file_paths {

    my $str1 = shift;

    my $str2 = shift;

    my @array1 = split(/;/, $str1);

    my @array2 = split(/;/, $str2);

    if ($#array1 != $#array2) { return 0; }

    for (my $i = 0; $i <= $#array1; $i++) {

        my @def1 = split(/:/, $array1[$i]);

        my @def2 = split(/:/, $array2[$i]);

        if ($#def1 != $#def2) { return 0; }

        for (my $j = 0; $j <= $#def1; $j++) {

            if ($def1[$j] ne $def2[$j]) { return 0; }

        }

    }

    return 1;

}        

#

# copy_if_exists subroutin attempt to copy a file from $src to $dst

# If the copy fails due to the file not existing, the error is ignored

#  Parameters:

#    $src    The source file to copy

#    $dst    The destination to copy to

#  Return value:

#    1  if copy was successful, or unsuccessful but the source file didn't exist

#    0  otherwise

#

sub copy_if_exists {

    my $src = shift;

    my $dst = shift;

    # Copy the file

    if( system("$CP_CMD \"$src\" \"$dst\"") != 0 ) {

        # if the copy failed, check if the file exists

        if( -e "$src" ) {

            # if the file exists, we had a real error

            return 0;

        }

    }

    # Success otherwise

    return 1;

}

#

# is_in_array subroutine checks if the given string is in the array.

#   Parameters:

#     str       a string

#     array_ref a reference to an array of strings

#   Return value:

#     1  if string is in the array

#     0  otherwise

# 

sub is_in_array {

    my $str = shift;

    my $array_ref = shift;

    if (grep { $str eq $_ } @{$array_ref}) {

        return 1;

    }

    return 0;

}

#

# if_directory_exists_and_empty accepts two arguments:

# variable with directory name and comment.

# Sub checks that directory exists and is empty

# usage: is_directory_exists_and_empty($directory,"Comment");

#

sub if_directory_exists_and_empty {

    my $empty_dir = shift;

    my $is_directory_empty_comment = shift;

    if (! -d $empty_dir) {

        die "$is_directory_empty_comment directory '$empty_dir' does not exist!";

    }

    opendir (my $dh, $empty_dir) or die "$is_directory_empty_comment directory '$empty_dir': Not a directory";

    if ( ! scalar( grep { $_ ne "." && $_ ne ".." && $_ ne "my.cnf" && $_ ne "master.info"} readdir($dh)) == 0) {

        die "$is_directory_empty_comment directory '$empty_dir' is not empty!";

    }

    closedir($dh);

}

#

# copy() wrapper with error handling

#

sub copy_file {

    my $src_path = shift;

    my $dst_path = shift;

    print STDERR "$prefix Copying '$src_path' to '$dst_path'\n";

    copy($src_path, $dst_path) or Die "copy failed: $!";

}

#

# move() wrapper with error handling

#

sub move_file {

    my $src_path = shift;

    my $dst_path = shift;

    print STDERR "$prefix Moving '$src_path' to '$dst_path'\n";

    move($src_path, $dst_path) or Die "move failed: $!";

}

#

# Auxiliary function called from find() callbacks to copy or move files and create directories

# when necessary.

#

# SYNOPSIS

#

#     process_file(\&process_func);

#

#       &process_func is a code reference that does the actual file operation

#

sub process_file {

    my $process_func = shift;

    if (/$copy_dir_exclude_regexp/) {

	return;

    }

    (my $dst_path = $File::Find::name) =~ s/^$copy_dir_src/$copy_dir_dst/;

    if (-d "$File::Find::name") {

	# Create the directory in the destination if necessary

	if (! -e "$dst_path") {

	    print STDERR "$prefix Creating directory '$dst_path'\n";

	    mkdir "$dst_path" or Die "mkdir failed: $!";

	} elsif (! -d "$dst_path") {

	    Die "$dst_path exists, but is not a directory";

	}

    } else {

	# Don't overwrite files unless $copy_dir_overwrite is 1

	if (!$copy_dir_overwrite && -e "$copy_dir_dst/$_") {

	    Die "Failed to process file $File::Find::name: " .

		"not overwriting file $copy_dir_dst/$_";

	}

	&$process_func($File::Find::name, $dst_path);

    }

}

#

# find() callback to copy files

#

sub copy_file_callback {

    process_file(\&copy_file);

}

#

# find() callback to move files

#

sub move_file_callback {

    process_file(\&move_file);

}

#

# copy_dir_recursively subroutine does a recursive copy of a specified

# directory excluding files matching a specifies regexp. If $overwrite

# is 1, it overwrites the existing files.

#

# SYNOPSIS 

#

#     copy_dir_recursively($src_dir, $dst_dir, $exclude_regexp,

#                          $overwrite);

#

# TODO

#

#     use rsync when --rsync is specified

#

sub copy_dir_recursively {

    # Clean paths and remove trailing slashes if any

    $copy_dir_src = File::Spec->canonpath(shift);

    $copy_dir_dst = File::Spec->canonpath(shift);

    $copy_dir_exclude_regexp = shift;

    $copy_dir_overwrite = shift;

    find(\&copy_file_callback, $copy_dir_src);

}

#

# Similar to copy_dir_recursively, but moves files instead.

#

# SYNOPSIS

#

#     move_dir_recursively($src_dir, $dst_dir, $exclude_regexp,

#                          $overwrite);

#

# TODO

#

#     use rsync when --rsync is specified

#

sub move_dir_recursively {

    # Clean paths and remove trailing slashes if any

    $copy_dir_src = File::Spec->canonpath(shift);

    $copy_dir_dst = File::Spec->canonpath(shift);

    $copy_dir_exclude_regexp = shift;

    $copy_dir_overwrite = shift;

    find(\&move_file_callback, $copy_dir_src);

}

#

# copy_back subroutine copies data and index files from backup directory 

# back to their original locations.

#

sub copy_back {

    my $move_flag = shift;

    my $orig_datadir = get_option(\%config, $option_defaults_group, 'datadir');

    my $orig_ibdata_dir = 

        get_option(\%config, $option_defaults_group, 'innodb_data_home_dir');

    my $orig_innodb_data_file_path = 

        get_option(\%config, $option_defaults_group, 'innodb_data_file_path');

    my $orig_iblog_dir =

        get_option(\%config, $option_defaults_group, 'innodb_log_group_home_dir');

    my $iblog_files = 'ib_logfile.*';

    my $excluded_files = 

        '\.\.?|backup-my\.cnf|xtrabackup_logfile|' .

	'xtrabackup_binary|xtrabackup_binlog_info|xtrabackup_checkpoints|' .

        '.*\.qp|' .

	$iblog_files;

    my $compressed_data_file = '.*\.ibz$';

    my $file;

    my $backup_innodb_data_file_path;

    if (has_option(\%config, $option_defaults_group, 'innodb_doublewrite_file')) {

        my $doublewrite_file =

                get_option(\%config, $option_defaults_group,

                            'innodb_doublewrite_file');

        $excluded_files = $excluded_files . '|' . $doublewrite_file;

    }

    # check whether files should be copied or moved to dest directory

    my $move_or_copy_file = $move_flag ? \&move_file : \&copy_file;

    my $move_or_copy_dir = $move_flag ?

        \&move_dir_recursively : \&copy_dir_recursively;

    my $operation = $move_flag ? "move" : "copy";

    # check that original data directories exist and they are empty

    if_directory_exists_and_empty($orig_datadir, "Original data");

    if_directory_exists_and_empty($orig_ibdata_dir, "Original InnoDB data");

    if_directory_exists_and_empty($orig_iblog_dir, "Original InnoDB log");

    # check that the original options file and the backup options file have

    # the same value for "innodb_data_file_path" option

    #$backup_innodb_data_file_path = 

    #    get_option(\%backup_config, 'mysqld', 'innodb_data_file_path');

    #if (!are_equal_innodb_data_file_paths($orig_innodb_data_file_path, 

    #                                      $backup_innodb_data_file_path)

    #) {

    #    Die "The value of 'innodb_data_file_path' option in the original "

    #      . "my.cnf file '$config_file' is different from the value "

    #      . "in the backup my.cnf file '$backup_config_file'.\n(original: "

    #      . "'$orig_innodb_data_file_path')\n"

    #      . "(backup:   '$backup_innodb_data_file_path')";

    #}

    # make a list of all ibdata files in the backup directory and all

    # directories in the backup directory under which there are ibdata files

    foreach my $a (split(/;/, $orig_innodb_data_file_path)) {

        my $path = (split(/:/,$a))[0];

        my $filename = (split(/\/+/, $path))[-1];

        # check that the backup data file exists

        if (! -e "$backup_dir/$filename") {

            if (-e "$backup_dir/${filename}.ibz") {

                Die "Backup data file '$backup_dir/$filename' does not exist, but "

                  . "its compressed copy '${path}.ibz' exists. Check "

                  . "that you have run '$innobackup_script --apply-log --uncompress "

                  . "...' before attempting '$innobackup_script --copy-back ...'  "

                  . "or '$innobackup_script --move-back ...' !";

            } else {

                Die "Backup data file '$backup_dir/$filename' does not exist.";

            }

        }

        $excluded_files .= "|\Q$filename\E";

    }

    # copy files from backup dir to their original locations

    # copy files to original data directory

    my $excluded_regexp = '^(' . $excluded_files . ')$';

    print STDERR "$prefix Starting to $operation files in '$backup_dir'\n"; 

    print STDERR "$prefix back to original data directory '$orig_datadir'\n";