--source include/no_valgrind_without_big.inc

#
# Purpose:
#   This test validates that data is consistent between a primary and replica
# in semi-sync mode when the primary is issued `SHUTDOWN WAIT FOR SLAVES`
# during an active communication. More specifically, the primary should not
# kill the Ack_thread until it is sure a replica has received all binlog
# data, i.e. once the primary receives the ACK. If a primary is issued a
# shutdown before receiving an ACK, it should wait until either 1) the ACK is
# received, or 2) the configured timeout (rpl_semi_sync_master_timeout) is
# reached.
#
# Methodology:
#   Using a topology consisting of one primary with two replicas, all in
# semi-sync mode, we use DEBUG_DBUG to simulate an error or delay on the
# replicas during an active communication while the primary is issued
# `SHUTDOWN WAIT FOR SLAVES`. We create four test cases to ensure the
# Ack_thread is not prematurely killed due to the shutdown.
#   Test Case 1) If both replicas simulate a delay that is within the allowed
#                timeout, the primary should delay killing the Ack_thread
#                until an ACK is received.
#   Test Case 2) If both replicas simulate an error before sending an ACK, the
#                primary should delay killing the Ack_thread until the
#                the timeout is reached.
#   Test Case 3) If one replica simulates a delay within the allowed timeout
#                and the other simulates an error before sending an ACK, the
#                primary should delay killing the Ack_thread until it
#                receives an ACK from the delayed slave.
#   Test Case 4) If a replica errors before sending an ACK, it will cause the
#                IO thread to stop and handle the error. During error handling,
#                if semi-sync is active, the replica will form a new connection
#                with the primary to kill the active connection. However, if
#                the primary is shutting down, it may kill the new connection,
#                thereby leaving the active semi-sync connection in-tact. The
#                slave should notice this, and not issue a `QUIT` command to
#                the primary, which would otherwise be sent to kill an active
#                connection. This test case validates that the slave does not
#                send a `QUIT` in this case.
#   Test Case 5) If a waiting-for-ACK user thread is killed (disconnected)
#                during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will
#                still await the ACK from the replica before killing the
#                Ack_thread.
#
# References:
#   MDEV-11853: semisync thread can be killed after sync binlog but before ACK
#               in the sync state
#   MDEV-28114: Semi-sync Master ACK Receiver Thread Can Error on COM_QUIT
#

--echo #############################
--echo # Common setup for all tests
--echo #############################

--echo # Note: Simulated slave delay is hardcoded to 800 milliseconds
--echo # Note: Simulated master shutdown delay is hardcoded to 500 milliseconds

--source include/have_innodb.inc
--source include/have_debug.inc
--let $rpl_topology=1->2, 1->3
--source include/rpl_init.inc

--connection server_1

--echo # Slaves which simulate an error will produce a timeout on the primary
call mtr.add_suppression("Timeout waiting");
call mtr.add_suppression("did not exit");
call mtr.add_suppression("Got an error reading communication packets");

--let $sav_master_timeout= `SELECT @@GLOBAL.rpl_semi_sync_master_timeout`
--let $sav_enabled_master= `SELECT @@GLOBAL.rpl_semi_sync_master_enabled`

--echo # Suppress slave errors related to the simulated error
--connection server_2
call mtr.add_suppression("reply failed");
call mtr.add_suppression("Replication event checksum verification");
call mtr.add_suppression("Relay log write failure");
call mtr.add_suppression("Failed to kill the active semi-sync connection");
set @sav_enabled_server_2= @@GLOBAL.rpl_semi_sync_slave_enabled;
set @sav_server_2_dbug= @@GLOBAL.debug_dbug;

--connection server_3
call mtr.add_suppression("reply failed");
call mtr.add_suppression("Replication event checksum verification");
call mtr.add_suppression("Relay log write failure");
call mtr.add_suppression("Failed to kill the active semi-sync connection");
set @sav_enabled_server_3= @@GLOBAL.rpl_semi_sync_slave_enabled;
set @sav_server_3_dbug= @@GLOBAL.debug_dbug;

--connection server_1
CREATE TABLE t1 (a int) engine=innodb;
--save_master_pos

--let i= 2
--let slave_last= 3
while (`SELECT $i <= $slave_last`)
{
  --connection server_$i
  --sync_with_master
  --inc $i
}

# Set up the connection used to issue the shutdown
--connect(server_1_con2, localhost, root,,)


--echo #############################
--echo # Test cases
--echo #############################

--echo #
--echo # Test Case 1) If both replicas simulate a delay that is within the
--echo # allowed timeout, the primary should delay killing the Ack_thread
--echo # until an ACK is received.
--echo #
--let server_2_sync_slave_reply=1
--let server_2_dbug= "+d,simulate_delay_semisync_slave_reply"
--let server_3_sync_slave_reply=1
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
--let semisync_timeout= 1600
--let server_2_expect_row_count= 1
--let server_3_expect_row_count= 1
--source rpl_semi_sync_shutdown_await_ack.inc

--echo #
--echo # Test Case 2) If both replicas simulate an error before sending an ACK,
--echo # the primary should delay killing the Ack_thread until the
--echo # timeout is reached.
--echo #
--let server_2_sync_slave_reply=0
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
--let server_3_sync_slave_reply=0
--let server_3_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
--let semisync_timeout= 500
--let server_2_expect_row_count= 0
--let server_3_expect_row_count= 0
--source rpl_semi_sync_shutdown_await_ack.inc

--echo #
--echo # Test Case 3) If one replica simulates a delay within the allowed
--echo # timeout and the other simulates an error before sending an ACK, the
--echo # primary should delay killing the Ack_thread until it receives an
--echo # ACK from the delayed slave.
--echo #
--let server_2_sync_slave_reply=0
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
--let server_3_sync_slave_reply=1
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
--let semisync_timeout= 1600
--let server_2_expect_row_count= 0
--let server_3_expect_row_count= 1
--source rpl_semi_sync_shutdown_await_ack.inc

--echo #
--echo # Test Case 4) If a replica errors before sending an ACK, it will cause
--echo # the IO thread to stop and handle the error. During error handling, if
--echo # semi-sync is active, the replica will form a new connection with the
--echo # primary to kill the active connection. However, if the primary is
--echo # shutting down, it may kill the new connection, thereby leaving the
--echo # active semi-sync connection in-tact. The slave should notice this, and
--echo # not issue a `QUIT` command to the primary, which would otherwise be
--echo # sent to kill an active connection. This test case validates that the
--echo # slave does not send a `QUIT` in this case.
--echo #
--let server_2_sync_slave_reply=0
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
--let server_3_sync_slave_reply=1
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
--let semisync_timeout= 1600
--let server_2_expect_row_count= 0
--let server_3_expect_row_count= 1
--source rpl_semi_sync_shutdown_await_ack.inc

#
# Added with MDEV-33551
#
--echo #
--echo # Test Case 5) If a waiting-for-ACK user thread is killed (disconnected)
--echo # during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will still
--echo # await the ACK from the replica before killing the Ack_receiver thread
--echo #
--connection server_1
insert into t1 values (1);
--source include/save_master_gtid.inc

--connection server_2
--source include/sync_with_master_gtid.inc
--source include/stop_slave.inc
SET GLOBAL rpl_semi_sync_slave_enabled= 1;
--source include/start_slave.inc

--connection server_1
SET GLOBAL rpl_semi_sync_master_enabled= 1;
SET GLOBAL rpl_semi_sync_master_timeout= 2000;

--let $status_var= Rpl_semi_sync_master_clients
--let $status_var_value= 1
source include/wait_for_status_var.inc;

show status like 'Rpl_semi_sync_master_status';
show status like 'Rpl_semi_sync_master_clients';

--connection server_2
SET @old_dbug= @@GLOBAL.debug_dbug;
SET GLOBAL debug_dbug="+d,simulate_delay_semisync_slave_reply";

--connect(con1, localhost, root,,)
--connect(con2, localhost, root,,)

--connection con1
--send insert into t1 values (2)

--connection server_1
--echo # Wait for thd to begin semi-sync wait..
--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = 'Waiting for semi-sync ACK from slave'
--source include/wait_condition.inc
--echo # ..done

--connection server_2
set debug_sync= "now wait_for io_thd_at_slave_reply";

--disconnect con1

--connection default
--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
wait
EOF

--connection con2
SHUTDOWN WAIT FOR ALL SLAVES;

--echo # Waitng for shutdown to be delayed..
--connection server_2
--inc $n_logged_delayed_shutdown_notes
let SEARCH_FILE= $log_error_file;
let SEARCH_PATTERN=Delaying shutdown to await semi-sync ACK;
let SEARCH_WAIT=FOUND $n_logged_delayed_shutdown_notes;
source include/search_pattern_in_file.inc;

--connection default
--source include/wait_until_disconnected.inc

--connection server_1
--source include/wait_until_disconnected.inc

--connection con2
--source include/wait_until_disconnected.inc

--connection server_2
set debug_sync= "now signal io_thd_do_reply";
--let $rpl_allow_error= 1
source include/stop_slave.inc;
--connection server_3
source include/stop_slave.inc;
--let $rpl_allow_error=

--connection default
--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
restart
EOF
--enable_reconnect
--source include/wait_until_connected_again.inc

--connection server_1
--enable_reconnect
--source include/wait_until_connected_again.inc


--echo #############################
--echo # Cleanup
--echo #############################

--connection server_2
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_2;
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
source include/start_slave.inc;

--connection server_3
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_3;
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
source include/start_slave.inc;

--connection server_1
let $status_var= Rpl_semi_sync_master_clients;
let $status_var_value= 0;
source include/wait_for_status_var.inc;

--disable_query_log
--eval SET @@GLOBAL.rpl_semi_sync_master_timeout= $sav_master_timeout;
--eval SET @@GLOBAL.rpl_semi_sync_master_enabled= $sav_enabled_master;
--enable_query_log

drop table t1;

--source include/rpl_end.inc