--source include/galera_cluster.inc
--source include/big_test.inc
--source include/force_restart.inc

#
# Testing gtid consistency in 3 node cluster when nodes drop
# and join back to cluster.
# The tests verify that wsrep_gtid_domain_id and gtid_binlog_pos
# remains same across the cluster
# In the configuration, nodes have different wsrep_gtid_domain_id
# but all nodes are supposed to receive effective domain id
# from the bootstrap node (node_1), and use it
#
--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3

# Save original auto_increment_offset values.
--let $node_1=node_1
--let $node_2=node_2
--let $node_3=node_3
--source ../galera/include/auto_increment_offset_save.inc

--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2
set wsrep_sync_wait=0;
--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
set wsrep_sync_wait=0;

--connection node_1
DELIMITER |;
CREATE PROCEDURE insert_row (IN node varchar(10), IN repeat_count int)
BEGIN
  DECLARE current_num int;
  SET current_num = 0;
  WHILE current_num < repeat_count do
    INSERT INTO t1(node, name) VALUES (node, UUID());
    SET current_num = current_num + 1;
  END WHILE;
END|
DELIMITER ;|

CREATE TABLE t1 (id bigint not null primary key auto_increment, node VARCHAR(10), name VARCHAR(64)) ENGINE=innodb;

#
# report initial gtid positions after table t1 is created
#
--echo # node_1
show variables like '%gtid_binlog_pos%';

--connection node_2
--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'
--source include/wait_condition.inc

--echo # node_2
show variables like '%gtid_binlog_pos%';

--connection node_3
--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'
--source include/wait_condition.inc

--echo # node_3
show variables like '%gtid_binlog_pos%';

###########################################
#  scenario: join node 2 by SST
##########################################

#
# start concurrent insert load and stop node2 while the load is on
#
--connection node_1
--send CALL insert_row('node1', 500);

--connection node_2
--send CALL insert_row('node2', 500);

--connection node_3
--send CALL insert_row('node3', 500);

#
# stop load to node 2 and shutdown the node, force SST
#
--connection node_2
--reap

--echo # Shutdown node_2, force SST
--connection node_2b
--source include/shutdown_mysqld.inc
--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat

--echo # Wait until node_2 leaves cluster
--connection node_1b
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc

#
# stop the remaining load to node 1 and 3
#
--connection node_1
--reap
--connection node_3
--reap

#
# some more inserts and DDL to nodes 1 and 3
# while node 2 is absent
#
--connection node_1
CALL insert_row('node1', 100);

--connection node_3
CALL insert_row('node3', 100);

CREATE TABLE t2(i int primary key) engine=innodb;

#
# restart node 2, should join by SST
#
--connection node_2
--echo # Restart node_2
--source include/start_mysqld.inc

--connection node_1b
--echo # Wait until node_2 is back in cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc

--echo # node2 has joined

#
# check gtid positions in all nodes
#
--echo # GTID in node1
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_2
--echo # GTID in node2
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_3
--echo # GTID in node3
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';


###########################################
#  scenario: join node 3 by IST
##########################################

--echo # Shutdown node_3
--connection node_3
SET GLOBAL wsrep_provider_options = 'gmcast.isolate = 1';

--echo # Wait until node_3 leaves cluster
--connection node_1b
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc

#
# do some inserts and DDL to node 1
# while node 3 is absent
#
--connection node_1
CALL insert_row('node1', 50);

CREATE TABLE t3(i int primary key) engine=innodb;

#
# remove isolation in node 3, should join by IST
#
--connection node_3
--echo # Rejoin node_3
SET GLOBAL wsrep_provider_options = 'gmcast.isolate = 0';

--connection node_1b
--echo # Wait until node_3 is back in cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc

--echo # node3 has joined

#
# check gtid positions in all nodes
#
--connection node_1
--echo # GTID in node1
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_2
--echo # GTID in node2
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_3
--echo # GTID in node3
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';


###########################################
#  scenario: restart full custer
##########################################


#
# stop all nodes, one by one
#
--echo # One by one shutdown all nodes
--connection node_3
--echo # shutdown node_3
--source include/shutdown_mysqld.inc
--remove_file $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat

--connection node_2
--echo # wait until node_3 is out of cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc
--echo # shutdown node_2
--source include/shutdown_mysqld.inc
--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat

--connection node_1
--echo # wait until node_2 is out of cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc
--echo # shutdown node_1
--source include/shutdown_mysqld.inc

#
# bootstap cluster in order node1 - node2 - node3
# send some inserts and DDL after each node started
#

--echo # Bootstrap from node_1
--connection node_1
--let $restart_parameters = --wsrep_new_cluster
--source include/start_mysqld.inc

--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc

show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

ANALYZE TABLE t2;
--send CALL insert_row('node1', 100);

--echo # Restart node_2
--connection node_2
--let $restart_parameters =
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
--source include/start_mysqld.inc

--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc

#
# connection node_1b may not be functional anymore, after node was
# shutdown, open node_1c for controlling node 1 state
#
--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
set wsrep_sync_wait=0;
--connection node_1c
--echo # wait until node_1 and node_2 are in cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc

--connection node_2
ALTER TABLE t2 ADD COLUMN (k int);
--send CALL insert_row('node2', 100);


--echo # Restart node_3
--connection node_3
--let $restart_parameters =
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
--source include/start_mysqld.inc

--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc


--connection node_1c
--echo # wait until all nodes are back in cluster
--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc
--echo after cluster restart

# stop load for nodes 1 and 2
--connection node_2
--reap

--connection node_1
--reap

#
# check gtid positions in all nodes
#
--connection node_1
--echo node1 GTID
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_2
--echo node2 GTID
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

--connection node_3
--echo node3 GTID
show variables like 'wsrep_gtid_domain_id';
show variables like '%gtid_binlog_pos%';

#
# check table size in all nodes
#
--connection node_1
--echo table size in node1
SELECT COUNT(*) FROM t1;

--connection node_2
--echo table size in node2
SELECT COUNT(*) FROM t1;

--connection node_3
--echo table size in node3
SELECT COUNT(*) FROM t1;

#
# cleanups
#
--connection node_2
call mtr.add_suppression("WSREP: Ignoring server id .* for non bootstrap node");
call mtr.add_suppression("Sending JOIN failed: ");
call mtr.add_suppression("WSREP: Failed to JOIN the cluster after SST");
call mtr.add_suppression("WSREP: FLOW message from member .* in non-primary configuration");
--connection node_3
call mtr.add_suppression("WSREP: Ignoring server id .* for non bootstrap node");
call mtr.add_suppression("Sending JOIN failed: ");
call mtr.add_suppression("WSREP: Failed to JOIN the cluster after SST");
call mtr.add_suppression("WSREP: FLOW message from member .* in non-primary configuration");

--echo # cleanup
--connection node_1
call mtr.add_suppression("WSREP: FLOW message from member .* in non-primary configuration");

DROP PROCEDURE insert_row;
DROP TABLE t1;
DROP TABLE t2;
DROP TABLE t3;

--connection node_3
--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't3'
--source include/wait_condition.inc
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't3'
--source include/wait_condition.inc

# Restore original auto_increment_offset values.
--let $galera_cluster_size=3
--source ../galera/include/auto_increment_offset_restore.inc

--disconnect node_3
--disconnect node_2b
--disconnect node_1b
--disconnect node_1c