~gl-az/+junk/ptb

#!/bin/bash
set -f
set +e

. ./include/ptb_core.inc

######################################################
# Percona Test Bench - XtraBackup performance matrix #
######################################################
declare -A PTB_OPTION_DESCRIPTORS=(\
	[mysql-rootdir]="REQ 1 PATHEXISTS Parent or install directory where mysql client and server binaries can be found. Ex: /usr/bin/mysql" \
	[include]="OPT 0 FILEEXISTS Script(s) to source after options have been processed and before testing starts that will declare and populate the required internal variables (PTB_mysql_matrix variable, PTB_xtrabackup_matrix). Ex: ps55-xb21-bitmap.cfg" \
	[cache-dir]="OPT 1 PATHCREATE Directory where server databases may be cached to save time during prepare phase." \
	[test-rootdir]="OPT 1 PATHEXISTS Directory where test tool binaries can be found (RQG, sysbench, tpcc, etc..)." \
	[test-prepare]="OPT 1 FILEEXISTS Script to call that will prepare the database server prior to each test. Ex: sysbench_oltp_prepare_sh" \
	[test-prepare-opt]="OPT 0 STR Option to be passed to prepare tool. Ex: --test-prepare-opt=--oltp-table-count=16 --test-prepare-opt=--oltp-table-size=100000" \
	[test-load]="OPT 1 FILEEXISTS Script to call that will run load on the server during each test. Ex: sysbench_oltp_load.sh" \
	[test-load-opt]="OPT 0 STR Option to be passed to load tool. Ex: --test-load-opt=--oltp-table-count=16 --test-load-opt=--oltp-table-size=100000" \
	[test-backup-wait]="OPT 1 INT 0 9999 0 Amount of time (in seconds) to wait after load has started before starting backup cycles." \
	[test-backup-delay]="OPT 1 INT 0 9999 0 Amount of time (in seconds) to wait in between each full backup." \
	[test-backup-count]="OPT 1 INT 1 9999 1 Number of full backups to perform." \
	[test-backup-destination]="OPT 1 INT 0 2 2 Defines where to send backup to. 0=xbstream to /dev/null (incompatible with incremental backup); 1=xbstream to local file(incompatible with incremental backup); 2=native to local backup directory." \
	[test-incremental-wait]="OPT 1 INT 0 9999 0 Amount of time (in seconds) to wait before performing each subsequent incremental backups. Can not be used with test-incremental-schedule." \
	[test-incremental-count]="OPT 1 INT 0 9999 0 Number of incremental backups to perform after each full backup. Can not be used with test-incremental-schedule." \
	[test-incremental-schedule]="OPT 1 STR Schedule of time delays to perform incremental backups after each full backup delimited by ':'. Use in place of test-incremental-count and test-incremental-wait. Ex: A value of 0:20:60:120 will perform the first incremental 0 seconds after completing a full backup, the second incremental will be performed 20 seconds after completion of the previous incremental, etc.." \
	[test-cleanup]="OPT 1 FILEEXISTS Script to call that will clean up or analyze the server after each test. Ex: sysbench_oltp_cleanup.sh" \
	[test-retry]="OPT 1 INT 0 9999 0 Number of attempts to retry backup test after initial failure." \
	[test-force]="OPT 1 INT 0 1 0 Force continuation of other tests after test-retry has been exhausted." \
	[vardir]="OPT 1 STRDEF ./var Directory where individual test data and results should be located." \
	[verbosity]="OPT 1 INT 0 4 2 Output verbosity filter: ERROR=4; WARNING=3; INFO=2; DEBUG=1; IDEBUG=0." \
	[xtrabackup-rootdir]="REQ 1 PATHEXISTS Directory where the XtraBackup binaries are located."
)
################################################################################
# Shows usage prolog
function usage()
{
	echo "XtraBackup performance matrix test."
}
################################################################################
# Validation callback for options parsing
# $1 - option name
# $2 - proposed option value
function getoption()
{
	case "$1" in
	"test_incremental_count" | "test_incremental_wait" )
		if [ -n "$PTB_OPT_test_incremental_schedule" ]; then
			ptb_report_error "test-incremental-schedule and $1 are mutually exclusive."
			return 2
		elif [ -n "$PTB_OPT_test_backup_destination" ] && [ $PTB_OPT_test_backup_destination -lt 2 ]; then
			ptb_report_error "test-backup-destination != 2 and $1 are mutually exclusive."
			return 2
		fi
		;;
	"test_incremental_schedule" )
		if [ -n "$PTB_OPT_test_incremental_count" ]; then
			ptb_report_error "test-incremental-count and $1 are mutually exclusive."
			return 2
		elif [ -n "$PTB_OPT_test_incremental_wait" ]; then
			ptb_report_error "test-incremental-wait and $1 are mutually exclusive."
			return 2
		elif [ -n "$PTB_OPT_test_backup_destination" ] && [ "$PTB_OPT_test_backup_destination" != "2" ]; then
			ptb_report_error "test-backup-destination != 2 and $1 are mutually exclusive."
			return 2
		fi
		;;
	"test_backup_destination" )
		if [ "$2" != "2" ]; then
			if [ -n "$PTB_OPT_test_incremental_count" ]; then
				ptb_report_error "test-backup-destination != 2 and test-incremental-count are mutually exclusive."
				return 2
			elif  [ -n "$PTB_OPT_test_incremental_schedule" ]; then
				ptb_report_error "test-backup-destination != 2 and test-incremental-schedule are mutually exclusive."
				return 2
			fi
		fi
		;;
	* )
		;;
	esac
	return 0
}
###############################################################################
# Performs backups
#
# $1 required, test number
# $2 required, test base dir
# $3 required, server id
# $4 required, load log file
# $5 required, result file
# $6 required, server options
# $7 required, xtrabackup options
function backup()
{
	local test_number=$1
	local test_dir=$2
	local server_id=$3
	local load_logfile=$4
	local result_file=$5
	local server_options="$6"
	local xtrabackup_options="$7"
	
	local rpt_prefix="backup($1, $2, $3, $4, $5, $6, $7)"

	# build out the proper path to use for callng innobackupex
	local xtrabackup_path="${S_BINDIR[$server_id]}/bin:${S_BINDIR[$server_id]}/libexec:$PTB_OPT_xtrabackup_rootdir"
	
	#parse the options and build out the correct innobackupex command line
	local encrypt=
	local encrypt_threads=1
	local encrypt_chunk_size=64
	local encrypt_key=
	local compress=0
	local compress_threads=1
	local compress_chunk_size=64
	local throttle=0
	local option=
	for option in $xtrabackup_options; do
		ptb_parse_server_option $option backup_optname backup_optvalue
		case "$backup_optname" in
		"encrypt" )
			encrypt=$backup_optvalue
			case $encrypt in
			"AES128" ) encrypt_key="1234567812345678";;
			"AES192" ) encrypt_key="123456781234567812345678";;
			"AES256" ) encrypt_key="12345678123456781234567812345678";;
			* )
				ptb_report_error "$rpt_prefix - invalid encryption algo. [$encrypt]"
				return 1
				;;
			esac
			;;
		"encrypt-threads" | "encrypt_threads" ) encrypt_threads=$backup_optvalue;;
		"encrypt-chunk-size" | "encrypt_chunk_size" ) encrypt_chunk_size=$backup_optvalue;;
		"compress" ) 
			if [ -z "$backup_optvalue" ]; then
				compress=1
			else
				compress=$backup_optvalue
			fi
			;;
		"compress-threads" | "compress_threads" ) compress_threads=$backup_optvalue;;
		"compress-chunk-size" | "compress_chunk_size" ) compress_chunk_size=$backup_optvalue;;
		"throttle" ) throttle=$backup_optvalue;;
		esac
	done

	local backup_command=
	backup_command="$backup_command --defaults-file=${S_DEFAULTSFILE[$server_id]}"
	backup_command="$backup_command --no-timestamp"
	backup_command="$backup_command --tmpdir=$test_dir"

	if [ -n "$encrypt" ]; then
		backup_command="$backup_command --encrypt=$encrypt"
		backup_command="$backup_command --encrypt-key=$encrypt_key"
		backup_command="$backup_command --encrypt-threads=$encrypt_threads"
		backup_command="$backup_command --encyrpt-chunk-size=$encrypt_chunk_size"
	fi

	if [ -n "$compress" ] && [ "$compress" != "0" ]; then
		backup_command="$backup_command --compress"
		backup_command="$backup_command --compress-threads=$compress_threads"
		backup_command="$backup_command --compress-chunk-size=$compress_chunk_size"
	fi


	# loop through full backups
	local full_cycle=0
	while [ $full_cycle -lt $PTB_OPT_test_backup_count ]; do
		# loop through 1 + incremental count making either a full or incremental backup for each cycle
		local current_cycle=0
		local previous_cycle=0
		while [ $current_cycle -lt ${#PTB_OPT_sleep_schedule[@]} ]; do
			local backup_current_dir=${test_dir}/backup-${full_cycle}.${current_cycle}
			local backup_previous_dir=${test_dir}/backup-${full_cycle}.${previous_cycle}
			local backup_logfile=${test_dir}/backup-${full_cycle}.${current_cycle}.log
	
			local inc_backup_command=""
			if [ $current_cycle -ne 0 ]; then
				inc_backup_command="--incremental --incremental-basedir=${backup_previous_dir}"
			fi

			ptb_report_info "$rpt_prefix - Sleeping for ${PTB_OPT_sleep_schedule[$current_cycle]} seconds before starting cycle $current_cycle"
			sleep ${PTB_OPT_sleep_schedule[$current_cycle]}

			echo "XtraBackup options: $xtrabackup_options" >> $backup_logfile
			echo "XtraBackup command: $backup_command $inc_backup_command" >> $backup_logfile

			if [ -n "$load_logfile" ]; then
				echo "Starting XtraBackup full_cycle[$full_cycle] current_cycle[$current_cycle] with command: $backup_command $inc_backup_command" >> $load_logfile
			fi

			#ptb_sql $serverid "SHOW STATUS LIKE 'innodb_buffer_pool%'"

			local backup_start_time=$SECONDS
			case "$PTB_OPT_test_backup_destination" in
			"0" ) # xbstream > /dev/null
				ptb_report_info "PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir --stream=xbstream 1>/dev/null 2>> $backup_logfile"
				( PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir --stream=xbstream 1>/dev/null 2>> $backup_logfile ) 
				rc=$?
				;;
			"1" ) # xbstream > ${backup_current_dir}/backup
				ptb_runcmd mkdir -p ${backup_current_dir}
				ptb_report_info "PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir --stream=xbstream 1>${backup_current_dir}/backup 2>> $backup_logfile"
				( PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir --stream=xbstream 1>${backup_current_dir}/backup 2>> $backup_logfile ) 
				rc=$?
				;;
			"2" ) # ${backup_current_dir}
				ptb_report_info "PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir &>> $backup_logfile"
				( PATH=${xtrabackup_path}:$PATH; innobackupex $backup_command $inc_backup_command $backup_current_dir 2>> $backup_logfile ) 
				rc=$?
				;;
			* )
				ptb_report_error "$rpt_prefix - invalid backup destination option [$PTB_OPT_test_backup_destination]."
				rc=1
				;;
			esac
			local backup_total_time=`expr $SECONDS - $backup_start_time`

			if [ $rc -ne 0 ]; then
				ptb_report_error "$rpt_prefix - innobackupex($xbopts) failed with $rc"
				break
			fi
				
			if [ -n "$load_logfile" ]; then
				echo "Finished XtraBackup full_cycle[$full_cycle] current_cycle[$current_cycle] with command: $backup_command $inc_backup_command" >> $load_logfile
			fi
				
			echo -n "$test_number,$full_cycle,$current_cycle,$backup_total_time,$rc" >> $result_file
			for option in ${server_options}; do
				ptb_parse_server_option $option backup_optname backup_optvalue
				echo -n ",$backup_optvalue" >> $result_file
			done
			for option in ${xtrabackup_options}; do
				ptb_parse_server_option $option backup_optname backup_optvalue
				echo -n ",$backup_optvalue" >> $result_file
			done
			echo "" >> $result_file

			previous_cycle=$current_cycle
			current_cycle=`expr ${current_cycle} + 1`
		done

		if [ $rc -ne 0 ]; then
			break
		fi

		full_cycle=`expr ${full_cycle} + 1`
	done

	return $rc
}
###############################################################################
# Performs restores
#
# $1 required, test number
# $2 required, test base dir
# $3 required, server id
# $4 required, load log file
# $5 required, result file
# $6 required, server options
# $7 required, xtrabackup options
function restore()
{
	local test_number=$1
	local test_dir=$2
	local server_id=$3
	local load_logfile=$4
	local result_file=$5
	local server_options="$6"
	local xtrabackup_options="$7"
	
	local rpt_prefix="restore($1, $2, $3, $4, $5, $6, $7)"

	# build out the proper path to use for callng innobackupex
	local xtrabackup_path="${S_BINDIR[$server_id]}/bin:${S_BINDIR[$server_id]}/libexec:$PTB_OPT_xtrabackup_rootdir"
}
###############################################################################
# Cleanup
#
# $1 required, test number
# $2 required, test base dir
# $3 required, server id
function cleanup()
{
	local test_number=$1
	local test_dir=$2
	local server_id=$3

	local full_cycle=0
	while [ $full_cycle -lt $PTB_OPT_test_backup_count ]; do
		local current_cycle=0
		while [ $current_cycle -lt ${#PTB_OPT_sleep_schedule[@]} ]; do
			local backup_current_dir=${test_dir}/backup-${full_cycle}.${current_cycle}
			ptb_runcmd rm -rf $backup_current_dir
			current_cycle=`expr ${current_cycle} + 1`
		done
		full_cycle=`expr ${full_cycle} + 1`
	done
	
	# remove data dir if success
	ptb_delete_server_data $server_id
}
###############################################################################
# Runs a single test with a single combination of server and xtrabackup
# options
#
# $1 - required, test number
# $2 - required, test base dir
# $3 - required, base port
# $4 - required, result file
# $5 - required, server options
# $6 - required, xtrabackup options
function runonetest()
{
	local test_number=$1
	local test_dir=$2
	local base_port=$3
	local result_file=$4
	local server_options="$5"
	local xtrabackup_options="$6"

	local rpt_prefix="runonetest($1, $2, $3, $4, $5, $6)"
	local server_id=1
	local load_id=1
	local server_port=$base_port
	local load_logfile=${test_dir}/load.log

	ptb_report_info "$rpt_prefix - starting test"


	# initialize the sandbox and variables
	ptb_init $test_dir $PTB_OPT_verbosity

	# create a server instance
	ptb_create_server_instance $server_id $PTB_OPT_mysql_rootdir $server_port
	rc=$?
	if [ $rc -ne 0 ]; then
		ptb_report_error "$rpt_prefix - ptb_create_server_instance($server_id, $PTB_OPT_mysql_rootdir, $server_port) failed with $rc"
		return $rc
	fi

	# set the server options
	ptb_set_all_server_options $server_id "$server_options"
	rc=$?
	if [ $rc -ne 0 ]; then
		ptb_report_error "$rpt_prefix - ptb_set_all_server_options($server_id, $server_options) failed with $rc"
		return $rc
	fi

	# prepare the database
	ptb_prepare_server_data $server_id "$PTB_OPT_cache_dir" "$PTB_OPT_test_prepare" "$PTB_OPT_test_rootdir" "${PTB_OPT_test_prepare_opt[@]}"
	rc=$?
	if [ $rc -ne 0 ]; then
		ptb_report_error "$rpt_prefix - ptb_prepare_server_data($server_id, \"$PTB_OPT_cache_dir\", \"$PTB_OPT_test_prepare\", $PTB_OPT_test_rootdir) failed with $rc."
		ptb_cleanup 1
		return $rc
	fi

	# start any parallel load
	if [ -n "$PTB_OPT_test_load" ]; then
		ptb_start_test_load $server_id $load_id $PTB_OPT_test_load $PTB_OPT_test_rootdir $load_logfile "${PTB_OPT_test_load_opt[@]}" 
		rc=$?
		if [ $rc -ne 0 ]; then
			ptb_report_error "$rpt_prefix - ptb_start_test_load($server_id, $load_id, $PTB_OPT_test_load, $PTB_OPT_test_rootdir, $load_logfile, ${PTB_OPT_test_load_opt[@]}) failed with $rc"
			ptb_cleanup 1
			return $rc
		fi
	fi

	# sleep while load loads up
	ptb_report_info "$rpt_prefix - Sleeping for ${PTB_OPT_test_backup_wait} seconds before starting backups."
	sleep $PTB_OPT_test_backup_wait

	# do the backup
	backup $test_number $test_dir $server_id $load_logfile $result_file "$server_options" "$xtrabackup_options"
	rc=$?

	# shut 'er down
	if [ -n "$PTB_OPT_test_load" ]; then
		ptb_kill_task $load_id
	fi
	ptb_stop_server $server_id

	# cleanup server data and backups on success
	if [ $rc -eq 0 ]; then
		cleanup $test_number $test_dir $server_id
	fi

	# cleanup the sandbox and variables
	ptb_cleanup 1

	return $rc
}
###############################################################################
# The main event
function main()
{
	local rpt_prefix="main()"

	# set up working directory
	if [ -z "$PTB_OPT_vardir" ]; then
		PTB_OPT_vardir="$PWD/var"
	fi
	if [ -d "$PTB_OPT_vardir" ]; then
		local newdiridx=1
		local newdirname="$PTB_OPT_vardir"
		while [ -d "${newdirname}" ]; do
			newdirname="${PTB_OPT_vardir}.${newdiridx}"
			newdiridx=`expr $newdiridx + 1`
		done
		ptb_runcmd mv "$PTB_OPT_vardir" "$newdirname"
		rc=$?
		if [ $rc -ne 0 ]; then
			ptb_report_error "$rpt_prefix - unable to move old vardir \"${PTB_OPT_vardir}\" out of the way to \"${newdirname}\", failed with $rc."
			return $rc
		fi
	fi
	ptb_runcmd mkdir $PTB_OPT_vardir
	rc=$?
	if [ $rc -ne 0 ]; then
		ptb_report_error "$rpt_prefix - unable to create vardir \"${PTB_OPT_vardir}\", failed with $rc."
		return $rc
	fi

	# set up result csv file
	local result_file=${PTB_OPT_vardir}/result.csv
	echo -n "testnum,full_cycle,inc_cycle,backuptime,rc" > $result_file
	for opt in ${server_options[0]}; do
		ptb_parse_server_option $opt optname optvalue
		echo -n ",$optname" >> $result_file
	done
	for opt in ${xb_options[0]}; do
		ptb_parse_server_option $opt optname optvalue
		echo -n ",$optname" >> $result_file
	done
	echo "" >> $result_file

	# build out the sleep schedule
	PTB_OPT_sleep_schedule[0]=$PTB_OPT_test_backup_delay
	if [ -z "$PTB_OPT_test_incremental_schedule" ]; then
		local i=
		for i in `seq 1 $PTB_OPT_test_incremental_count`; do
			PTB_OPT_sleep_schedule[$i]=$PTB_OPT_test_incremental_wait
		done
	else
		local i=1
		local startpos=0
		local colonpos=0
		while [ 1 -eq 1 ]; do
			colonpos=`expr index ${PTB_OPT_test_incremental_schedule:$startpos} :`
			colonpos=`expr $startpos + $colonpos`
			if [ $colonpos -eq $startpos ]; then
				PTB_OPT_sleep_schedule[$i]=${PTB_OPT_test_incremental_schedule:$startpos}
				break
			else
				PTB_OPT_sleep_schedule[$i]=${PTB_OPT_test_incremental_schedule:$startpos:`expr $colonpos - $startpos - 1`}
			fi
			i=`expr $i + 1`
			startpos=$colonpos
		done
	fi

	# loop through the combinations and run each test
	local server_option=
	local test_number=0
	local base_port=10000
	for server_option in "${server_options[@]}"; do
		local xtrabackup_option=
		for xtrabackup_option in "${xb_options[@]}"; do
			local test_dir=${PTB_OPT_vardir}/test-${test_number}
			local test_logfile="${test_dir}/test.log"
			local trys_remaining=`expr $PTB_OPT_test_retry + 1`
			local rc=1
			ptb_report_info "$rpt_prefix - Starting test $test_number with [$server_option] and [$xtrabackup_option]"
			while [ $rc -ne 0 ] && [ $trys_remaining -gt 0 ]; do
				if [ ! -d "$test_dir" ]; then
					ptb_runcmd mkdir -p $test_dir
					rc=$?
					if [ $rc -ne 0 ]; then
						ptb_report_error "$rpt_prefix - unable to create test directory $test_dir, failed with $rc."
						return $rc
					fi
				fi

				runonetest $test_number $test_dir $base_port $result_file "$server_option" "$xtrabackup_option" > $test_logfile
				rc=$?
				if [ $rc -ne 0 ]; then
					local newdiridx=2
					local newdirname="${test_dir}.failure.1"
					while [ -d "${newdirname}" ]; do
						newdirname="${test_dir}.failure.${newdiridx}"
						newdiridx=`expr $newdiridx + 1`
					done
					ptb_report_info "$rpt_prefix - Test $test_num with [$server_option] and [$xtrabackup_option] FAILED with $rc."
					ptb_runcmd mv "$test_dir" "$newdirname"
					rc=$?
					if [ $rc -ne 0 ]; then
						ptb_report_error "$rpt_prefix - unable to move old testdir \"${test_dir}\" out of the way to \"${newdirname}\", failed with $rc."
						return $rc
					fi
				fi
				trys_remaining=`expr $trys_remaining - 1`
			done
			if [ $rc -ne 0 ] && [ $PTB_OPT_test_force -ne 0 ]; then
				return 1
			fi
			test_number=`expr $test_number + 1`
		done
	done
	return 0
}

###############################################################################
# main
###############################################################################

# parse and validate command line args
ptb_parse_options getoption usage $@
# debugging
#ptb_show_option_values


# source in our include configurations 
for i in "${PTB_OPT_include[@]}"; do
	. $i
done
ptb_report_info "Total: `expr ${#server_options[@]} * ${#xb_options[@]}` combinations"

main

exit 0