# The following test aims at verifying MTS recovery.
# It can be invoked in a standard as well as Multi-source replication environment.
#
# Algorithm of test
# -----------------
#
# A number of concurrently executable transactions are prepared on the
# master.  It's done with adding a "separator" group which is a DML in
# myisam table.  Considering their sizes and a small value of
# max_relay_log_size they are relay-logged in a sequence of files
# possibly being broken into pieces so different pieces of a
# transaction be scattered across few logs.
# At applying a number of transactions  is let to complete until a separator
# is found. At this point a slave local session locks random records in
# the data tables which leads to a timeout error.
# After the slave applier is stopped, the failure conditions are removed,
# and slave resumes with two phases. When the gaps are filled up, a next
# undisturbed round of applying follows.
# Gaps filling is done in two branches: an immediate restart in the
# "hot" server, and the slave server shutdown, restart and the slave
# service restart.  In either case MTS recovery must be completed with
# positive data consistency check.

#------------------
# Parameters:
#

# Number of master connection. For Logical_clock it means
# parallelization window, for the Database scheduler it means a number
# of databases.
--let $conn_cnt=4
# Total number of failures that the slave is subject to by simulation
--let $total_fail_cnt=4
--let $fail_to_shutdown=2
# max number of simulataneous errors by Workers at a time
--let $simul_fail_cnt=4
# total number of transactions to be processed by slave
--let $total_trx_cnt=128
# give up waiting for lock in Innodb at error simulation
--let $innodb_wait_timeout=1
# small value for max relay-log-size
--let $relay_log_size= 4096

--connection slave

CALL mtr.add_suppression("The replica coordinator and worker threads are stopped");

SET @save.innodb_lock_wait_timeout=@@GLOBAL.innodb_lock_wait_timeout;
eval SET @@global.innodb_lock_wait_timeout=$innodb_wait_timeout;
SET @save.replica_transaction_retries= @@global.replica_transaction_retries;
SET @@global.replica_transaction_retries= 0;
SET @save_replica_preserve_commit_order= @@global.replica_preserve_commit_order;
SET @save.max_relay_log_size= @@global.max_relay_log_size;
--eval SET @@global.max_relay_log_size=$relay_log_size

--source include/stop_slave_sql.inc
SET GLOBAL replica_preserve_commit_order= OFF;
--source include/start_slave_sql.inc

--let $assert_text= the max size of relay log must be $relay_log_size
--let $assert_cond= [select @@global.max_relay_log_size] = $relay_log_size
--source include/assert.inc
--let $trx_max_size=`select 3*@@global.max_relay_log_size/2`
#
# End of parameters
# ------------------

--let $index=$conn_cnt
while ($index)
{
  --connect(con$index, localhost, root,,)
    eval CREATE DATABASE db$index;
    eval CREATE TABLE db$index.t (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=innodb;
  --dec $index
}

--connection master
# Inserts into myisam will be caught/blocked on the slave side
# to serve as synchronization tool by which the slave side
# becomes aware of execution/scheduling point in the replicated load.
# So each such insert is made to "trigger" a failure.
CREATE TABLE t1m (a INT) ENGINE=myisam;

if ($rpl_multi_source)
{
  --let $rpl_channel_name= 'channel_1'
}
#--let $sync_slave_connection= server_2
--let $sync_slave_connection= slave
--source include/sync_slave_sql_with_master.inc

--connection slave
LOCK TABLES t1m WRITE;

--connection master
#
# Parallelizable load generator
#
--let $trans_no_failure=`SELECT FLOOR($total_trx_cnt / $total_fail_cnt )`
--let $iter=$total_trx_cnt
--disable_query_log
while ($iter)
{
  --let $index=$conn_cnt
  while ($index)
  {
    --connection con$index
      BEGIN;
        eval INSERT INTO db$index.t values (NULL, REPEAT('a', FLOOR(RAND()*$trx_max_size)));
    --dec $index
  }
  --let $index=$conn_cnt
  while ($index)
  {
    --connection con$index
      COMMIT;
      if (`SELECT  ($total_trx_cnt-$iter) % $trans_no_failure = FLOOR($trans_no_failure/2)`)
     {
	--connection master
          INSERT INTO t1m SET a=1;
     }
    --dec $index
    --dec $iter  # the outer loop counter is decremented on purpose here.
  }
}
--enable_query_log

#
# The slave failures loop. Slave sql threads stop and restarted
# prescribed number of times. After being waited tables are unlocked
# the slave must always recover.
#

--disable_query_log
--disable_result_log
--let $iter=$total_fail_cnt
while ($iter)
{
  --let $iter_err= $simul_fail_cnt
  --connection slave1
  BEGIN;
  while ($iter_err)
  {
    --let $index=`SELECT 1 + FLOOR(RAND()*100) % $conn_cnt`
    --eval SELECT * FROM db$index .t FOR UPDATE
    --dec $iter_err
  }
  --connection slave
  UNLOCK TABLES;
  --let $slave_sql_errno=convert_error(ER_LOCK_WAIT_TIMEOUT)
  --let $show_slave_sql_error= 0
  --source include/wait_for_slave_sql_error.inc

  --connection slave1
  ROLLBACK;

  # first few iterations with a "warm" restart
  if ($iter > $fail_to_shutdown)
  {
    --connection slave
    if ($rpl_multi_source)
    {
      START SLAVE UNTIL SQL_AFTER_MTS_GAPS for channel 'channel_1';
      --let $rpl_channel_name= 'channel_1'
    }
    if (!$rpl_multi_source)
    {
      START SLAVE UNTIL SQL_AFTER_MTS_GAPS;
    }
    --source include/wait_for_slave_sql_to_stop.inc
  }
  if ($iter <= $fail_to_shutdown)
  {
    --let $rpl_server_number= 2
    --let $rpl_skip_start_slave= 1
    --source include/rpl_restart_server.inc
    --connection slave
    SET @save.innodb_lock_wait_timeout=@@GLOBAL.innodb_lock_wait_timeout;
    eval SET @@global.innodb_lock_wait_timeout=$innodb_wait_timeout;
    SET @save.replica_transaction_retries= @@global.replica_transaction_retries;
    SET @@global.replica_transaction_retries= 0;
    SET @save_replica_preserve_commit_order= @@global.replica_preserve_commit_order;
    SET @@global.replica_preserve_commit_order=OFF;
    SET @save.max_relay_log_size= @@global.max_relay_log_size;
    SET @@global.max_relay_log_size=4096;
    if ($rpl_multi_source)
    {
      START SLAVE UNTIL SQL_AFTER_MTS_GAPS for channel 'channel_1';
      --let $rpl_channel_name= 'channel_1'
    }
    if (!$rpl_multi_source)
    {
      START SLAVE UNTIL SQL_AFTER_MTS_GAPS;
    }
    --source include/wait_for_slave_sql_to_stop.inc
  }
  LOCK TABLES t1m WRITE;
  if ($rpl_multi_source)
  {
    --let $rpl_channel_name= 'channel_1'
  }
  --source include/start_slave_sql.inc

  --dec $iter
} #eof of the slave error load
--enable_result_log
--enable_query_log

--connection slave
  UNLOCK TABLES;

--connection master
if ($rpl_multi_source)
{
  --let $rpl_channel_name= 'channel_1'
}
--let $sync_slave_connection= slave
--source include/sync_slave_sql_with_master.inc

#
# Cleanup
#
--connection master
DROP TABLE t1m;
--let $index=$conn_cnt
while ($index)
{
  --eval DROP DATABASE db$index
  --dec $index
}

--source include/sync_slave_sql_with_master.inc
SET @@global.innodb_lock_wait_timeout=@save.innodb_lock_wait_timeout;
SET @@global.replica_transaction_retries= @save.replica_transaction_retries;
SET @@global.max_relay_log_size= @save.max_relay_log_size;

--source include/stop_slave_sql.inc
SET @@global.replica_preserve_commit_order= @save_replica_preserve_commit_order;
--source include/start_slave_sql.inc