~dbpercona/percona-qa/readme

#!/bin/bash -ue

# Author: Raghavendra Prabhu <raghavendra.prabhu@percona.com>

ulimit -c unlimited

export MTR_MAX_SAVE_CORE=5

set +e

echo "Killing existing mysqld"

pgrep -f mysqld

pkill -f mysqld

sleep 10

pgrep mysqld || pkill -9 -f mysqld

set -e

sleep 5

LPATH=${SPATH:-/usr/share/doc/sysbench/tests/db}

WORKDIR=$1

ROOT_FS=$WORKDIR

sst_method=${SST_METHOD:-rsync}

thres=1

if [[ $sst_method == 'xtrabackup' ]];then 

    wget -q  http://www.percona.com/redir/downloads/XtraBackup/LATEST/binary/tarball/percona-xtrabackup-2.2.3-4982-Linux-x86_64.tar.gz

    tar xf percona-xtrabackup-2.2.3-4982-Linux-x86_64.tar.gz

    export PATH="$PATH:$PWD/percona-xtrabackup-2.2.3-Linux-x86_64/bin/"

    sst_method='xtrabackup-v2'

fi

#pushd /tmp

#wget -q http://files.wnohang.net/files/libgalera_smm.so

#popd

cd $WORKDIR

VER2=$LATEST_VER

VER1=$(basename $RELEASE_BIN | cut -d- -f4)

if [[ $VER1 == $VER2 ]];then 

    thres=2

fi

count=$(ls -1ct Percona-XtraDB-Cluster-$VER1*.tar.gz | wc -l)

if [[ $count -gt $thres ]];then 

    for dirs in `ls -1ct Percona-XtraDB-Cluster-$VER1*.tar.gz | tail -n +2`;do 

        rm -rf $dirs || true

    done 

fi

find . -maxdepth 1 -type d -name 'Percona-XtraDB-Cluster-$VER1*' -exec rm -rf {} \+ || true 

count=$(ls -1ct Percona-XtraDB-Cluster-$VER2*.tar.gz | wc -l)

if [[ $count -gt $thres ]];then 

    for dirs in `ls -1ct Percona-XtraDB-Cluster-$VER2*.tar.gz | tail -n +2`;do 

        rm -rf $dirs || true 

    done 

fi

find . -maxdepth 1 -type d -name 'Percona-XtraDB-Cluster-$VER2*' -exec rm -rf {} \+ || true 

echo "Removing older directories"

find . -maxdepth 1 -type d -mtime +10 -exec rm -rf {} \+ || true 

echo "Removing their symlinks"

find . -maxdepth 1 -type l -mtime +10 -delete

TAR=`ls -1ct Percona-XtraDB-Cluster-$VER2*.tar.gz | head -n1`

BASE2="$(tar tf $TAR | head -1 | cut -d '/' -f1)"

tar -xf $TAR

TAR=`ls -1ct Percona-XtraDB-Cluster-$VER1*.tar.gz | head -n1`

BASE1="$(tar tf $TAR | head -1 | cut -d '/' -f1)"

if [[ $BASE1 == $BASE2 ]];then 

    TAR=`ls -1ct Percona-XtraDB-Cluster-$VER1*.tar.gz | tail -1`

    BASE1="$(tar tf $TAR | head -1 | cut -d '/' -f1)"

fi

if [[ $BASE1 == $BASE2 ]];then 

    echo "FATAL: Failed"

    exit 1

fi

tar -xf $TAR

#cp -v /tmp/libgalera_smm.so $ROOT_FS/$BASE2/lib/galera3/

# Parameter of parameterized build 

if [[ -n $SDURATION ]];then 

    export SYSBENCH_DURATION=$SDURATION

else 

    export SYSBENCH_DURATION=300

fi

WORKDIR="${ROOT_FS}/$BUILD_NUMBER"

for l in `seq $((BUILD_NUMBER - 2)) -1 1`;do 

    rm -rf $l || true

done

mkdir -p $WORKDIR/logs

MYSQL_BASEDIR1="${ROOT_FS}/$BASE1"

MYSQL_BASEDIR2="${ROOT_FS}/$BASE2"

GALERA1="${MYSQL_BASEDIR1}/lib/libgalera_smm.so"

GALERA2="${MYSQL_BASEDIR2}/lib/libgalera_smm.so"

if [[ ! -e $GALERA1 ]];then 

    GALERA1="${MYSQL_BASEDIR1}/lib/galera3/libgalera_smm.so"

fi

if [[ ! -e $GALERA2 ]];then 

    GALERA2="${MYSQL_BASEDIR2}/lib/galera3/libgalera_smm.so"

fi

export MYSQL_VARDIR="$WORKDIR/mysqldir"

mkdir -p $MYSQL_VARDIR

SDIR="$LPATH"

SRESULTS="$WORKDIR/sresults"

mkdir -p $SRESULTS

echo "Workdir: $WORKDIR"

echo "Basedirs: $MYSQL_BASEDIR1 $MYSQL_BASEDIR2"

  ADDR="127.0.0.1"

  RPORT=$(( RANDOM%21 + 10 ))

  RBASE1="$(( RPORT*1000 ))"

  echo "Setting RBASE to $RBASE1"

  RADDR1="$ADDR:$(( RBASE1 + 7 ))"

  LADDR1="$ADDR:$(( RBASE1 + 8 ))"

  RBASE2="$(( RBASE1 + 100 ))"

  RADDR2="$ADDR:$(( RBASE2 + 7 ))"

  LADDR2="$ADDR:$(( RBASE2 + 8 ))"

  SUSER=root

  SPASS=

  node1="${MYSQL_VARDIR}/node1"

  mkdir -p $node1

  node2="${MYSQL_VARDIR}/node2"

  mkdir -p $node2

EXTSTATUS=0

if [[ $MEM -eq 1 ]];then 

    MEMOPT="--mem"

else 

    MEMOPT=""

fi

cleanup(){

    rm Percona-XtraDB-Cluster*.tar.gz*  || true

    tar cvzf $ROOT_FS/results-${BUILD_NUMBER}.tar.gz $WORKDIR/logs || true

}

trap cleanup EXIT KILL

if [[ -n ${EXTERNALS:-} ]];then 

    EXTOPTS="$EXTERNALS"

else

    EXTOPTS=""

fi

if [[ $DIR -eq 1 ]];then 

    sockets="$node1/socket.sock,$node2/socket.sock"

elif [[ $DIR -eq 2 ]];then 

    sockets="$node2/socket.sock"

elif [[ $DIR -eq 3 ]];then 

    sockets="$node1/socket.sock"

fi

STABLE="test.sbtest1" 

if [[ $BUILD_SOURCE == 'debug' ]];then 

    EXTOPTS+=" --mysqld=--skip-performance-schema "

fi

GDEBUG=""

if [[ $DEBUG -eq 1 ]];then 

    DBG="--mysqld=--wsrep-debug=1"

elif [[ $DEBUG -eq 2 ]];then 

    DBG=" --mysqld=--wsrep-debug=1 --mysqld=--wsrep-log-conflicts=ON "

elif [[ $DEBUG -eq 3 ]];then 

    DBG=" --mysqld=--wsrep-debug=1 --mysqld=--wsrep-log-conflicts=ON"

    GDEBUG="; debug=1"

else 

    DBG=""

fi

#export LD_LIBRARY_PATH="$MYSQL_BASEDIR1/lib"

SBENCH="sysbench"

common=" --start-and-exit --nowarnings  \

    --mysqld=--wsrep_sst_method=$sst_method \

    --mysqld=--wsrep_sst_auth=$SUSER:$SPASS \

    --mysqld=--wsrep_node_address=$ADDR \

    --nodefault-myisam \

    --mysqld=--innodb_flush_method=O_DIRECT \

    --mysqld=--query_cache_type=0 \

    --mysqld=--query_cache_size=0 \

    --mysqld=--innodb_flush_log_at_trx_commit=0 \

    --mysqld=--innodb_buffer_pool_size=500M \

    --mysqld=--innodb_log_file_size=500M \

    --mysqld=--default-storage-engine=InnoDB \

    --mysqld=--loose-innodb \

    --mysqld=--sql-mode=no_engine_substitution \

    --mysqld=--skip-external-locking \

    --mysqld=--core-file \

    --mysqld=--skip-name-resolve \

    --mysqld=--innodb_file_per_table  \

    --mysqld=--binlog-format=ROW \

    --mysqld=--wsrep-slave-threads=8 \

    --mysqld=--innodb_autoinc_lock_mode=2 "

u_common=" --mysqld=--skip-grant-tables \

        --mysqld=--innodb_file_per_table  \

        --mysqld=--binlog-format=ROW \

        --mysqld=--innodb_autoinc_lock_mode=2 \

        --mysqld=--wsrep-provider=none \

        --mysqld=--innodb_flush_method=O_DIRECT \

        --mysqld=--query_cache_type=0 \

        --mysqld=--query_cache_size=0 \

        --mysqld=--innodb_flush_log_at_trx_commit=0 \

        --mysqld=--innodb_buffer_pool_size=500M \

        --mysqld=--innodb_log_file_size=500M \

        --mysqld=--skip-name-resolve \

        --mysqld=--default-storage-engine=InnoDB \

        --mysqld=--loose-innodb \

        --mysqld=--sql-mode=no_engine_substitution \

        --mysqld=--skip-external-locking \

        --start-and-exit \

        --start-dirty \

        --nowarnings \

        --nodefault-myisam "

ver_and_row(){

    local sock=$1

    $MYSQL_BASEDIR1/bin/mysql -S $sock  -u root -e "show global variables like 'version';"

    $MYSQL_BASEDIR1/bin/mysql -S $sock  -u root -e "select count(*) from $STABLE;"

    $MYSQL_BASEDIR1/bin/mysql -S $sock  -u root -e "select count(*) from $STABLE;"

}

prepare()

{

    local sock=$1

    local log=$2

    echo "Sysbench Run: Prepare stage"

    $SBENCH --test=$LPATH/parallel_prepare.lua --report-interval=10  --oltp-auto-inc=$AUTOINC --mysql-engine-trx=yes --mysql-table-engine=innodb \

        --oltp-table-size=$TSIZE --oltp_tables_count=$TCOUNT --mysql-db=test --mysql-user=root  --num-threads=$NUMT \

        --db-driver=mysql --mysql-socket=$sock prepare 2>&1 | tee $log

    $MYSQL_BASEDIR1/bin/mysql  -S $sock -u root -e "create database testdb;" || true

}

rw_full()

{

    local sock=$1

    local log=$2

    echo "Sysbench Run: OLTP RW testing"

    $SBENCH --mysql-table-engine=innodb --num-threads=$NUMT --report-interval=10 --oltp-auto-inc=$AUTOINC --max-time=$SYSBENCH_DURATION --max-requests=1870000000 \

        --test=$SDIR/$STEST.lua --init-rng=on --oltp_index_updates=10 --oltp_non_index_updates=10 --oltp_distinct_ranges=15 --oltp_order_ranges=15 --oltp_tables_count=$TCOUNT --mysql-db=test \

        --mysql-user=root --db-driver=mysql --mysql-socket=$sock  run 2>&1 | tee $log

}

rw_ist()

{

    local sock=$1

    local log=$2

    echo "Populating for IST"

    echo "Sysbench Run: OLTP RW testing"

        $SBENCH --mysql-table-engine=innodb --num-threads=1 --report-interval=10 --oltp-auto-inc=$AUTOINC --max-time=10 --max-requests=1870000000 \

            --test=$LPATH/oltp.lua --init-rng=on --oltp_index_updates=10 --oltp_non_index_updates=10 --oltp_distinct_ranges=15 --oltp_order_ranges=15 --oltp_tables_count=$TCOUNT --mysql-db=test \

            --mysql-user=root --db-driver=mysql --mysql-socket=$sock  run 2>&1 | tee $log

}

get_script()

{

    if [[ ! -e $SDIR/${STEST}.lua ]];then 

        pushd /tmp

        rm $STEST.lua || true

        wget -O $STEST.lua  http://files.wnohang.net/files/${STEST}.lua

        export SDIR=/tmp/

        popd

    fi

}

reset_compat()

{

    local gpath=$1

    if [[ $gpath == *galera3* ]];then 

        COMPAT="; socket.checksum=1"  

    else 

        COMPAT=""  

    fi

}

  reset_compat $GALERA1

  echo "Starting $VER1 node"

  pushd ${MYSQL_BASEDIR1}/mysql-test/

  perl mysql-test-run.pl \

    --mysqld=--basedir=${MYSQL_BASEDIR1} \

    --port-base=$RBASE1 \

    $common \

    --vardir=$node1 $DBG $MEMOPT $EXTOPTS \

    --mysqld=--wsrep-new-cluster \

    --mysqld=--wsrep-provider=$GALERA1 \

    --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

    --mysqld=--wsrep_sst_receive_address=$RADDR1 \

    --mysqld=--wsrep_node_incoming_address=$ADDR \

    --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR1}$COMPAT$GDEBUG" \

    --mysqld=--socket=$node1/socket.sock \

    --mysqld=--log-error=$WORKDIR/logs/node1.err \

    --mysqld=--log-output=none \

    1st

  popd

  prepare $node1/socket.sock $WORKDIR/logs/sysbench_prepare.txt 

  reset_compat $GALERA2

  echo "Starting $VER2 node for SST"

    pushd ${MYSQL_BASEDIR2}/mysql-test/

    perl mysql-test-run.pl \

        --mysqld=--basedir=${MYSQL_BASEDIR2} \

        --port-base=$RBASE2 \

        $common \

        --vardir=$node2 $DBG $MEMOPT $EXTOPTS \

        --mysqld=--wsrep-provider=$GALERA2 \

        --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

        --mysqld=--wsrep_sst_receive_address=$RADDR2 \

        --mysqld=--wsrep_node_incoming_address=$ADDR \

        --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR2}$COMPAT$GDEBUG" \

        --mysqld=--socket=$node2/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node2-sst.err \

        --mysqld=--log-output=none \

        1st

    popd

    echo "Sleeping till SST is complete"

    sleep 10

    ver_and_row $node2/socket.sock 

    ver_and_row $node1/socket.sock

    echo "Shutting down node2 after SST"

    ${MYSQL_BASEDIR1}/bin/mysqladmin  --socket=$node2/socket.sock -u root shutdown

    rw_ist $node1/socket.sock $WORKDIR/logs/sysbench_for_ist.txt 

    sleep 10

  reset_compat $GALERA2

    pushd ${MYSQL_BASEDIR2}/mysql-test/

    echo "Restarting for IST"

    perl mysql-test-run.pl \

        --mysqld=--basedir=${MYSQL_BASEDIR2} \

        --start-dirty \

        --port-base=$RBASE2 \

        $common \

        --vardir=$node2 $DBG $MEMOPT $EXTOPTS \

        --mysqld=--wsrep-provider=$GALERA2 \

        --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

        --mysqld=--wsrep_sst_receive_address=$RADDR2 \

        --mysqld=--wsrep_node_incoming_address=$ADDR \

        --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR2}$COMPAT$GDEBUG" \

        --mysqld=--socket=$node2/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node2-ist.err \

        --mysqld=--log-output=none \

        1st

    popd

    sleep 10

    ver_and_row $node2/socket.sock 

    ver_and_row $node1/socket.sock

    get_script

    $MYSQL_BASEDIR1/bin/mysql -S $node1/socket.sock  -u root -e "create database testdb;" || true

    rw_full $sockets  $WORKDIR/logs/sysbench_rw_run.txt 

    if [[ $VER1 > $VER2 ]];then 

        node_for_upgrade=$VER2 

        nodeu=$node2 

        pbase=$RBASE2

        baseu=$MYSQL_BASEDIR1

        ugalera="${baseu}/lib/libgalera_smm.so"

        listu=$LADDR2

        recvu=$RADDR2

    else 

        node_for_upgrade=$VER1 

        nodeu=$node1 

        pbase=$RBASE1

        baseu=$MYSQL_BASEDIR2

        ugalera="${baseu}/lib/libgalera_smm.so"

        listu=$LADDR1

        recvu=$RADDR1

    fi

    if [[ ! -e $ugalera ]];then 

        ugalera="${baseu}/lib/galera3/libgalera_smm.so"

    fi

    echo "Shutting down $node_for_upgrade  for upgrade"

    $MYSQL_BASEDIR1/bin/mysqladmin  --socket=$nodeu/socket.sock -u root shutdown

    pushd ${baseu}/mysql-test/

    perl mysql-test-run.pl \

        --mysqld=--basedir=$baseu \

        --port-base=$pbase \

        $u_common \

        --vardir=$nodeu $DBG $MEMOPT $EXTOPTS \

        --mysqld=--socket=$nodeu/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node1-upgrade.err \

        --mysqld=--log-output=none \

        1st

    popd

    $baseu/bin/mysql_upgrade -S $nodeu/socket.sock -u root 2>&1 | tee $WORKDIR/logs/mysql_upgrade.log

    echo "Shutting down $node_for_upgrade after upgrade"

    $MYSQL_BASEDIR1/bin/mysqladmin  --socket=$nodeu/socket.sock -u root shutdown

    echo "Starting $node_for_upgrade after upgrade"

  reset_compat $ugalera

  pushd ${baseu}/mysql-test/

  perl mysql-test-run.pl \

    --mysqld=--basedir=$baseu \

    --port-base=$pbase \

    $common \

    --vardir=$nodeu $DBG $MEMOPT $EXTOPTS \

    --mysqld=--wsrep-provider=$ugalera \

    --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

    --mysqld=--wsrep_sst_receive_address=$recvu \

    --mysqld=--wsrep_node_incoming_address=$ADDR \

    --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${listu}$COMPAT$GDEBUG" \

    --mysqld=--socket=$nodeu/socket.sock \

    --mysqld=--log-error=$WORKDIR/logs/node1-postupgrade.err \

    --mysqld=--log-output=none \

    1st

  popd

    get_script

    rw_full $sockets  $WORKDIR/logs/sysbench_rw_run-2.txt 

    sleep 100

    ver_and_row $node2/socket.sock 

    ver_and_row $node1/socket.sock

    $MYSQL_BASEDIR1/bin/mysql -S $node1/socket.sock  -u root -e "drop database testdb;" || true

    $MYSQL_BASEDIR1/bin/mysql -S $node1/socket.sock  -u root -e "drop database test;"

    echo "Shutting down node2 "

    $MYSQL_BASEDIR2/bin/mysqladmin  --socket=$node2/socket.sock -u root shutdown

    sleep 10 

    echo "Shutting down node1 "

    $MYSQL_BASEDIR2/bin/mysqladmin  --socket=$node1/socket.sock -u root shutdown

    echo "TESTING in reverse" 

  node3="${MYSQL_VARDIR}/node3"

  mkdir -p $node3

  node4="${MYSQL_VARDIR}/node4"

  mkdir -p $node4

  MYSQL_BASEDIR1="${ROOT_FS}/$BASE2"

  MYSQL_BASEDIR2="${ROOT_FS}/$BASE1"

  GALERA1="${MYSQL_BASEDIR1}/lib/libgalera_smm.so"

  GALERA2="${MYSQL_BASEDIR2}/lib/libgalera_smm.so"

    if [[ ! -e $GALERA1 ]];then 

        GALERA1="${MYSQL_BASEDIR1}/lib/galera3/libgalera_smm.so"

    fi

    if [[ ! -e $GALERA2 ]];then 

        GALERA2="${MYSQL_BASEDIR2}/lib/galera3/libgalera_smm.so"

    fi

if [[ $DIR -eq 1 ]];then 

    sockets="$node3/socket.sock,$node4/socket.sock"

elif [[ $DIR -eq 2 ]];then 

    sockets="$node4/socket.sock"

elif [[ $DIR -eq 3 ]];then 

    sockets="$node3/socket.sock"

fi

  VER1=$LATEST_VER

  VER2=$(basename $RELEASE_BIN | cut -d- -f4)

  echo "Starting $VER1 node"

  reset_compat $GALERA1

  pushd ${MYSQL_BASEDIR1}/mysql-test/

  perl mysql-test-run.pl \

    --mysqld=--basedir=${MYSQL_BASEDIR1} \

    --port-base=$RBASE1 \

    $common \

    --vardir=$node3 $DBG $MEMOPT $EXTOPTS \

    --mysqld=--wsrep-new-cluster \

    --mysqld=--wsrep-provider=$GALERA1 \

    --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

    --mysqld=--wsrep_sst_receive_address=$RADDR1 \

    --mysqld=--wsrep_node_incoming_address=$ADDR \

    --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR1}$COMPAT$GDEBUG" \

    --mysqld=--socket=$node3/socket.sock \

    --mysqld=--log-error=$WORKDIR/logs/node3.err \

    --mysqld=--log-output=none \

    1st

  popd

    prepare $node3/socket.sock $WORKDIR/logs/sysbench_prepare_r.txt 

  reset_compat $GALERA2

  echo "Starting $VER2 node for SST"

    pushd ${MYSQL_BASEDIR2}/mysql-test/

    perl mysql-test-run.pl \

        --mysqld=--basedir=${MYSQL_BASEDIR2} \

        --port-base=$RBASE2 \

        $common \

        --vardir=$node4 $DBG $MEMOPT $EXTOPTS \

        --mysqld=--wsrep-provider=$GALERA2 \

        --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

        --mysqld=--wsrep_sst_receive_address=$RADDR2 \

        --mysqld=--wsrep_node_incoming_address=$ADDR \

        --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR2}$COMPAT$GDEBUG" \

        --mysqld=--socket=$node4/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node4-sst.err \

        --mysqld=--log-output=none \

        1st

    popd

    echo "Sleeping till SST is complete"

    sleep 10

    ver_and_row $node3/socket.sock 

    ver_and_row $node4/socket.sock

    echo "Shutting down node4 after SST"

    ${MYSQL_BASEDIR1}/bin/mysqladmin  --socket=$node4/socket.sock -u root shutdown

    rw_ist $node3/socket.sock $WORKDIR/logs/sysbench_for_ist_r.txt 

    sleep 10

  reset_compat $GALERA2

    pushd ${MYSQL_BASEDIR2}/mysql-test/

    echo "Restarting for IST"

    perl mysql-test-run.pl \

        --mysqld=--basedir=${MYSQL_BASEDIR2} \

        --start-dirty \

        --port-base=$RBASE2 \

        $common \

        --vardir=$node4 $DBG $MEMOPT $EXTOPTS \

        --mysqld=--wsrep-provider=$GALERA2 \

        --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

        --mysqld=--wsrep_sst_receive_address=$RADDR2 \

        --mysqld=--wsrep_node_incoming_address=$ADDR \

        --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${LADDR2}$COMPAT$GDEBUG" \

        --mysqld=--socket=$node4/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node4-ist.err \

        --mysqld=--log-output=none \

        1st

    popd

    sleep 10

    ver_and_row $node3/socket.sock 

    ver_and_row $node4/socket.sock

    get_script

    $MYSQL_BASEDIR1/bin/mysql -S $node3/socket.sock  -u root -e "create database testdb;" || true

    rw_full $sockets  $WORKDIR/logs/sysbench_rw_run_r.txt 

    if [[ $VER1 > $VER2 ]];then 

        node_for_upgrade=$VER2 

        nodeu=$node4 

        pbase=$RBASE2

        baseu=$MYSQL_BASEDIR1

        ugalera="${baseu}/lib/libgalera_smm.so"

        listu=$LADDR2

        recvu=$RADDR2

    else 

        node_for_upgrade=$VER1 

        nodeu=$node2 

        pbase=$RBASE1

        baseu=$MYSQL_BASEDIR2

        ugalera="${baseu}/lib/libgalera_smm.so"

        listu=$LADDR1

        recvu=$RADDR1

    fi

    if [[ ! -e $ugalera ]];then 

        ugalera="${baseu}/lib/galera3/libgalera_smm.so"

    fi

    echo "Shutting down node4 for upgrade"

    $baseu/bin/mysqladmin  --socket=$nodeu/socket.sock -u root shutdown

    pushd ${baseu}/mysql-test/

    perl mysql-test-run.pl \

        --mysqld=--basedir=${baseu} \

        --port-base=$pbase \

        $u_common \

        --vardir=$nodeu $DBG $MEMOPT $EXTOPTS \

        --mysqld=--socket=$nodeu/socket.sock \

        --mysqld=--log-error=$WORKDIR/logs/node4-upgrade.err \

        --mysqld=--log-output=none \

        1st

    popd

    $MYSQL_BASEDIR1/bin/mysql_upgrade -S $node4/socket.sock -u root 2>&1 | tee $WORKDIR/logs/mysql_upgrade_r.log

    echo "Shutting down node4 after upgrade"

    $MYSQL_BASEDIR1/bin/mysqladmin  --socket=$node4/socket.sock -u root shutdown

    echo "Starting node4 after upgrade"

  reset_compat $ugalera

  pushd ${baseu}/mysql-test/

  perl mysql-test-run.pl \

    --mysqld=--basedir=${baseu} \

    --port-base=$pbase \

    $common \

    --vardir=$nodeu $DBG $MEMOPT $EXTOPTS \

    --mysqld=--wsrep-provider=$ugalera \

    --mysqld=--wsrep_cluster_address=gcomm://$LADDR1,$LADDR2 \

    --mysqld=--wsrep_sst_receive_address=$recvu \

    --mysqld=--wsrep_node_incoming_address=$ADDR \

    --mysqld=--wsrep_provider_options="gmcast.listen_addr=tcp://${listu}$COMPAT$GDEBUG" \

    --mysqld=--socket=$nodeu/socket.sock \

    --mysqld=--log-error=$WORKDIR/logs/node4-postupgrade.err \

    --mysqld=--log-output=none \

    1st

  popd

    get_script

    rw_full $sockets  $WORKDIR/logs/sysbench_rw_run_r-2.txt 

    sleep 100

    ver_and_row $node3/socket.sock 

    ver_and_row $node4/socket.sock

    $MYSQL_BASEDIR1/bin/mysql -S $node3/socket.sock  -u root -e "drop database testdb;" || true

    $MYSQL_BASEDIR1/bin/mysql -S $node3/socket.sock  -u root -e "drop database test;"

    echo "Shutting down node4 after IST"

    $MYSQL_BASEDIR2/bin/mysqladmin  --socket=$node4/socket.sock -u root shutdown

    sleep 10 

    echo "Shutting down node3 after IST"

    $MYSQL_BASEDIR2/bin/mysqladmin  --socket=$node3/socket.sock -u root shutdown

  exit $EXTSTATUS