#############################
# Common setup for all tests
#############################
# Note: Simulated slave delay is hardcoded to 800 milliseconds
# Note: Simulated master shutdown delay is hardcoded to 500 milliseconds
include/rpl_init.inc [topology=1->2, 1->3]
connection server_1;
# Slaves which simulate an error will produce a timeout on the primary
call mtr.add_suppression("Timeout waiting");
call mtr.add_suppression("did not exit");
call mtr.add_suppression("Got an error reading communication packets");
# Suppress slave errors related to the simulated error
connection server_2;
call mtr.add_suppression("reply failed");
call mtr.add_suppression("Replication event checksum verification");
call mtr.add_suppression("Relay log write failure");
call mtr.add_suppression("Failed to kill the active semi-sync connection");
set @sav_enabled_server_2= @@GLOBAL.rpl_semi_sync_slave_enabled;
set @sav_server_2_dbug= @@GLOBAL.debug_dbug;
connection server_3;
call mtr.add_suppression("reply failed");
call mtr.add_suppression("Replication event checksum verification");
call mtr.add_suppression("Relay log write failure");
call mtr.add_suppression("Failed to kill the active semi-sync connection");
set @sav_enabled_server_3= @@GLOBAL.rpl_semi_sync_slave_enabled;
set @sav_server_3_dbug= @@GLOBAL.debug_dbug;
connection server_1;
CREATE TABLE t1 (a int) engine=innodb;
connection server_2;
connection server_3;
connect server_1_con2, localhost, root,,;
#############################
# Test cases
#############################
#
# Test Case 1) If both replicas simulate a delay that is within the
# allowed timeout, the primary should delay killing the Ack_thread
# until an ACK is received.
#
connection server_1;
#--
#-- Semi-sync Setup
connection server_1;
#-- Enable semi-sync on slaves
let slave_last= 3
connection server_2;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
connection server_3;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
#-- Enable semi-sync on master
connection server_1;
SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
set @@global.rpl_semi_sync_master_timeout= 1600;
#-- Wait for master to recognize semi-sync slaves
connection server_1;
#-- Master should have semi-sync enabled with 2 connections
show status like 'Rpl_semi_sync_master_status';
Variable_name	Value
Rpl_semi_sync_master_status	ON
show status like 'Rpl_semi_sync_master_clients';
Variable_name	Value
Rpl_semi_sync_master_clients	2
#-- Prepare servers to simulate delay or error
connection server_2;
SET @@GLOBAL.debug_dbug= "+d,simulate_delay_semisync_slave_reply";
connection server_3;
SET @@GLOBAL.debug_dbug= "+d,simulate_delay_semisync_slave_reply";
#--
#-- Test begins
connection server_1_con2;
connection server_1;
#-- Begin semi-sync transaction
INSERT INTO t1 VALUES (1);
connection server_1_con2;
#-- Wait until master recognizes a connection is awaiting semi-sync ACK
connection server_2;
set debug_sync= "now wait_for io_thd_at_slave_reply";
connection server_3;
set debug_sync= "now wait_for io_thd_at_slave_reply";
connection server_1_con2;
#-- Begin master shutdown
SHUTDOWN WAIT FOR ALL SLAVES;
connection server_2;
# Waitng for shutdown to be delayed..
FOUND 1 /Delaying shutdown to await semi-sync ACK/ in mysqld.1.err
connection server_2;
set debug_sync= "now signal io_thd_do_reply";
connection server_3;
set debug_sync= "now signal io_thd_do_reply";
# Reaping transaction..
connection server_1;
ERROR HY000: Lost connection to server during query
connection server_2;
# Reaping shutdown..
connection server_1_con2;
# Validate slave data is in correct state
connection server_2;
select count(*)=1 from t1;
count(*)=1
1
connection server_3;
select count(*)=1 from t1;
count(*)=1
1
#
#-- Re-synchronize slaves with master and disable semi-sync
#-- Stop slaves
connection server_2;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_2;
connection server_3;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_3;
#-- Bring the master back up
connection server_1_con2;
connection default;
connection server_1;
TRUNCATE TABLE t1;
#-- Bring slaves back up
connection server_2;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
connection server_3;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
#
# Test Case 2) If both replicas simulate an error before sending an ACK,
# the primary should delay killing the Ack_thread until the
# timeout is reached.
#
connection server_1;
#--
#-- Semi-sync Setup
connection server_1;
#-- Enable semi-sync on slaves
let slave_last= 3
connection server_2;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
connection server_3;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
#-- Enable semi-sync on master
connection server_1;
SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
set @@global.rpl_semi_sync_master_timeout= 500;
#-- Wait for master to recognize semi-sync slaves
connection server_1;
#-- Master should have semi-sync enabled with 2 connections
show status like 'Rpl_semi_sync_master_status';
Variable_name	Value
Rpl_semi_sync_master_status	ON
show status like 'Rpl_semi_sync_master_clients';
Variable_name	Value
Rpl_semi_sync_master_clients	2
#-- Prepare servers to simulate delay or error
connection server_2;
SET @@GLOBAL.debug_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141";
connection server_3;
SET @@GLOBAL.debug_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141";
#--
#-- Test begins
connection server_1_con2;
connection server_1;
#-- Begin semi-sync transaction
INSERT INTO t1 VALUES (1);
connection server_1_con2;
#-- Wait until master recognizes a connection is awaiting semi-sync ACK
connection server_1_con2;
#-- Begin master shutdown
SHUTDOWN WAIT FOR ALL SLAVES;
connection server_2;
# Waitng for shutdown to be delayed..
FOUND 2 /Delaying shutdown to await semi-sync ACK/ in mysqld.1.err
# Reaping transaction..
connection server_1;
ERROR HY000: Lost connection to server during query
connection server_2;
# Reaping shutdown..
connection server_1_con2;
# Validate slave data is in correct state
connection server_2;
select count(*)=0 from t1;
count(*)=0
1
connection server_3;
select count(*)=0 from t1;
count(*)=0
1
#
#-- Re-synchronize slaves with master and disable semi-sync
#-- FIXME: workaround for MDEV-28141, preventing errored replicas from
# killing their semi-sync connections
connection server_2;
set debug_sync= "now wait_for at_semisync_kill_connection";
set debug_sync= "now signal continue_semisync_kill_connection";
# Wait for debug_sync signal to have been received before issuing RESET
set debug_sync= "reset";
connection server_3;
set debug_sync= "now wait_for at_semisync_kill_connection";
set debug_sync= "now signal continue_semisync_kill_connection";
# Wait for debug_sync signal to have been received before issuing RESET
set debug_sync= "reset";
#-- Stop slaves
connection server_2;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_2;
connection server_3;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_3;
#-- Bring the master back up
connection server_1_con2;
connection default;
connection server_1;
TRUNCATE TABLE t1;
#-- Bring slaves back up
connection server_2;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
connection server_3;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
#
# Test Case 3) If one replica simulates a delay within the allowed
# timeout and the other simulates an error before sending an ACK, the
# primary should delay killing the Ack_thread until it receives an
# ACK from the delayed slave.
#
connection server_1;
#--
#-- Semi-sync Setup
connection server_1;
#-- Enable semi-sync on slaves
let slave_last= 3
connection server_2;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
connection server_3;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
#-- Enable semi-sync on master
connection server_1;
SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
set @@global.rpl_semi_sync_master_timeout= 1600;
#-- Wait for master to recognize semi-sync slaves
connection server_1;
#-- Master should have semi-sync enabled with 2 connections
show status like 'Rpl_semi_sync_master_status';
Variable_name	Value
Rpl_semi_sync_master_status	ON
show status like 'Rpl_semi_sync_master_clients';
Variable_name	Value
Rpl_semi_sync_master_clients	2
#-- Prepare servers to simulate delay or error
connection server_2;
SET @@GLOBAL.debug_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141";
connection server_3;
SET @@GLOBAL.debug_dbug= "+d,simulate_delay_semisync_slave_reply";
#--
#-- Test begins
connection server_1_con2;
connection server_1;
#-- Begin semi-sync transaction
INSERT INTO t1 VALUES (1);
connection server_1_con2;
#-- Wait until master recognizes a connection is awaiting semi-sync ACK
connection server_3;
set debug_sync= "now wait_for io_thd_at_slave_reply";
connection server_1_con2;
#-- Begin master shutdown
SHUTDOWN WAIT FOR ALL SLAVES;
connection server_2;
# Waitng for shutdown to be delayed..
FOUND 3 /Delaying shutdown to await semi-sync ACK/ in mysqld.1.err
connection server_3;
set debug_sync= "now signal io_thd_do_reply";
# Reaping transaction..
connection server_1;
ERROR HY000: Lost connection to server during query
connection server_2;
# Reaping shutdown..
connection server_1_con2;
# Validate slave data is in correct state
connection server_2;
select count(*)=0 from t1;
count(*)=0
1
connection server_3;
select count(*)=1 from t1;
count(*)=1
1
#
#-- Re-synchronize slaves with master and disable semi-sync
#-- FIXME: workaround for MDEV-28141, preventing errored replicas from
# killing their semi-sync connections
connection server_2;
set debug_sync= "now wait_for at_semisync_kill_connection";
set debug_sync= "now signal continue_semisync_kill_connection";
# Wait for debug_sync signal to have been received before issuing RESET
set debug_sync= "reset";
#-- Stop slaves
connection server_2;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_2;
connection server_3;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_3;
#-- Bring the master back up
connection server_1_con2;
connection default;
connection server_1;
TRUNCATE TABLE t1;
#-- Bring slaves back up
connection server_2;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
connection server_3;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
#
# Test Case 4) If a replica errors before sending an ACK, it will cause
# the IO thread to stop and handle the error. During error handling, if
# semi-sync is active, the replica will form a new connection with the
# primary to kill the active connection. However, if the primary is
# shutting down, it may kill the new connection, thereby leaving the
# active semi-sync connection in-tact. The slave should notice this, and
# not issue a `QUIT` command to the primary, which would otherwise be
# sent to kill an active connection. This test case validates that the
# slave does not send a `QUIT` in this case.
#
connection server_1;
#--
#-- Semi-sync Setup
connection server_1;
#-- Enable semi-sync on slaves
let slave_last= 3
connection server_2;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
connection server_3;
include/stop_slave.inc
set global rpl_semi_sync_slave_enabled = 1;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	ON
#-- Enable semi-sync on master
connection server_1;
SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
set @@global.rpl_semi_sync_master_timeout= 1600;
#-- Wait for master to recognize semi-sync slaves
connection server_1;
#-- Master should have semi-sync enabled with 2 connections
show status like 'Rpl_semi_sync_master_status';
Variable_name	Value
Rpl_semi_sync_master_status	ON
show status like 'Rpl_semi_sync_master_clients';
Variable_name	Value
Rpl_semi_sync_master_clients	2
#-- Prepare servers to simulate delay or error
connection server_2;
SET @@GLOBAL.debug_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141";
connection server_3;
SET @@GLOBAL.debug_dbug= "+d,simulate_delay_semisync_slave_reply";
#--
#-- Test begins
connection server_1_con2;
connection server_1;
#-- Begin semi-sync transaction
INSERT INTO t1 VALUES (1);
connection server_1_con2;
#-- Wait until master recognizes a connection is awaiting semi-sync ACK
connection server_3;
set debug_sync= "now wait_for io_thd_at_slave_reply";
connection server_1_con2;
#-- Begin master shutdown
SHUTDOWN WAIT FOR ALL SLAVES;
connection server_2;
# Waitng for shutdown to be delayed..
FOUND 4 /Delaying shutdown to await semi-sync ACK/ in mysqld.1.err
connection server_3;
set debug_sync= "now signal io_thd_do_reply";
# Reaping transaction..
connection server_1;
ERROR HY000: Lost connection to server during query
connection server_2;
# Reaping shutdown..
connection server_1_con2;
# Validate slave data is in correct state
connection server_2;
select count(*)=0 from t1;
count(*)=0
1
connection server_3;
select count(*)=1 from t1;
count(*)=1
1
#
#-- Re-synchronize slaves with master and disable semi-sync
#-- FIXME: workaround for MDEV-28141, preventing errored replicas from
# killing their semi-sync connections
connection server_2;
set debug_sync= "now wait_for at_semisync_kill_connection";
set debug_sync= "now signal continue_semisync_kill_connection";
# Wait for debug_sync signal to have been received before issuing RESET
set debug_sync= "reset";
#-- Stop slaves
connection server_2;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_2;
connection server_3;
include/stop_slave_io.inc
include/stop_slave_sql.inc
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_3;
#-- Bring the master back up
connection server_1_con2;
connection default;
connection server_1;
TRUNCATE TABLE t1;
#-- Bring slaves back up
connection server_2;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
connection server_3;
include/start_slave.inc
show status like 'Rpl_semi_sync_slave_status';
Variable_name	Value
Rpl_semi_sync_slave_status	OFF
SELECT COUNT(*)=0 from t1;
COUNT(*)=0
1
#
# Test Case 5) If a waiting-for-ACK user thread is killed (disconnected)
# during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will still
# await the ACK from the replica before killing the Ack_receiver thread
#
connection server_1;
insert into t1 values (1);
include/save_master_gtid.inc
connection server_2;
include/sync_with_master_gtid.inc
include/stop_slave.inc
SET GLOBAL rpl_semi_sync_slave_enabled= 1;
include/start_slave.inc
connection server_1;
SET GLOBAL rpl_semi_sync_master_enabled= 1;
SET GLOBAL rpl_semi_sync_master_timeout= 2000;
show status like 'Rpl_semi_sync_master_status';
Variable_name	Value
Rpl_semi_sync_master_status	ON
show status like 'Rpl_semi_sync_master_clients';
Variable_name	Value
Rpl_semi_sync_master_clients	1
connection server_2;
SET @old_dbug= @@GLOBAL.debug_dbug;
SET GLOBAL debug_dbug="+d,simulate_delay_semisync_slave_reply";
connect con1, localhost, root,,;
connect con2, localhost, root,,;
connection con1;
insert into t1 values (2);
connection server_1;
# Wait for thd to begin semi-sync wait..
# ..done
connection server_2;
set debug_sync= "now wait_for io_thd_at_slave_reply";
disconnect con1;
connection default;
connection con2;
SHUTDOWN WAIT FOR ALL SLAVES;
# Waitng for shutdown to be delayed..
connection server_2;
FOUND 5 /Delaying shutdown to await semi-sync ACK/ in mysqld.1.err
connection default;
connection server_1;
connection con2;
connection server_2;
set debug_sync= "now signal io_thd_do_reply";
include/stop_slave.inc
connection server_3;
include/stop_slave.inc
connection default;
connection server_1;
#############################
# Cleanup
#############################
connection server_2;
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_2;
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
include/start_slave.inc
connection server_3;
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_3;
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
include/start_slave.inc
connection server_1;
drop table t1;
include/rpl_end.inc