################################################################################
# Multi Master recovery test
# This test focus on the recovery feature of multi master groups.
# The basic functioning of this test consists in loading some data to a group
# and then verify if a new joining member contains also this data after recovery.
# Three specific scenarios are tested:
#   1) A brand new member joins the group and recovers
#   2) The donor on 1) leaves and returns being now a joiner.
#   3) The joiner in 1) leaves and returns, doing a second round of recovery.
#
# Test:
# 0. The test requires three servers: M1, M2 and M3.
# 1. Test the recovery of new members:
#  - Bootstrap start GR on M1. Add some data for recovery.
#  - Start GR on M2 and M3. Check after recovery all members must see each other
#  - Validate data is present on all the members.
#  - Test group responds to requests well.
#  - All members must have the same GTID set.
#  - Check if the data is the same on joiner(M2) and donor(M1).
# 2. Test recovery on a member that left and now returns:
#  - Stop GR on M1, who never went through recovery.
#  - Add some data on M2 for recovery.
#  - Start GR on M1. Validate data.
#  - Test group responds to requests well. Sync group.
#  - Check if the data is the same on M1 and M2.
# 3. Testing a second recovery round on a member:
#  - Stop GR on M2.
#  - Add some data on M1 for recovery.
#  - Start GR on M2. Validate data.
#  - Test group responds to requests well.
#  - Check if the data is the same on M1 and M2.
# 4. Clean up.
################################################################################

--source include/big_test.inc
--source include/have_group_replication_gtid_assignment_block_size_1.inc
--let $group_replication_group_name= 36236980-4307-11e4-916c-0800200c9a66
--source include/have_group_replication_plugin.inc
--let $rpl_skip_group_replication_start= 1
--let $rpl_server_count= 3
--source include/group_replication.inc

--echo #
--echo # Setup a new group
--echo #

--connection server1
--echo server1
--source include/start_and_bootstrap_group_replication.inc

--echo # Add some data for recovery

CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
BEGIN;
INSERT INTO t1 VALUES (1);
INSERT INTO t1 VALUES (2);
COMMIT;
INSERT INTO t1 VALUES (3);

--echo #
--echo # 1) Testing the recovery of a new member
--echo #

--echo #Add 2 more members
--connection server2
--echo server2
--source include/start_group_replication.inc

--connection server3
--echo server3
--source include/start_group_replication.inc

--echo #After recovery all members must see 3 other members
--let $server_count=3
while ($server_count)
{
  --connection server$server_count
  --let $group_replication_number_of_members= 3
  --source include/gr_wait_for_number_of_members.inc

  --dec $server_count
}

--echo #After recovery all members must have the data present in the donor.
--let $server_count=3
while ($server_count)
{
  --connection server$server_count
  --let $assert_text= On all members, the table should exist and have 3 elements
  --let $assert_cond= [select count(*) from t1] = 3;
  --source include/assert.inc
  --dec $server_count
}

--echo #Test if the group responds to requests
--connection server1
--echo server1

INSERT INTO t1 VALUES (4);

--connection server2
--echo server2

INSERT INTO t1 VALUES (5);

--connection server3
--echo server3

--let $wait_condition= select count(*) = 5 from t1;
--source include/wait_condition.inc

--source include/rpl_sync.inc

--echo #All members must have the same GTID set
--let $server_count=3
while ($server_count)
{
  --connection server$server_count
  --let $assert_text= On member $server_count, all executed GTID should belong to the group
  --let $assert_cond= "[SELECT @@GLOBAL.GTID_EXECUTED]" = "36236980-4307-11e4-916c-0800200c9a66:1-8";
  --source include/assert.inc
  --dec $server_count
}

--echo #Check if the data is the same on joiner and donor
--let  $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc

--echo #
--echo # 2) Testing recovery on a member that left and now returns
--echo #

--echo #Stop the member 1, who never went through recovery
--connection server1
--echo server1

--source include/stop_group_replication.inc

--echo #Add some data to the future donor
--connection server2
--echo server2

INSERT INTO t1 VALUES (6);

--connection server1
--echo server1
--source include/start_group_replication.inc

--let $assert_text= On the recovered member, the table should contain 6 elements
--let $assert_cond= [select count(*) from t1] = 6;
--source include/assert.inc

--echo #Test if the group responds to requests
--connection server2
--echo server2

INSERT INTO t1 VALUES (7);

--let $wait_condition= select count(*) = 7 from t1;
--source include/wait_condition.inc

# we are using this because we want to sync a pair of servers instead of the
# whole group. sync_slave_with_master is a legacy name.

--let $sync_slave_connection= server1
--source include/sync_slave_sql_with_master.inc

--let $sync_slave_connection= server3
--source include/sync_slave_sql_with_master.inc

connection server2;
--let  $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc

--echo #
--echo # 3) Testing a second recovery round on a member
--echo #

--echo #Stop the member 2 making it go through recovery again
--source include/stop_group_replication.inc

--echo #Add some data to the future donor
--connection server1
--echo server1

INSERT INTO t1 VALUES (8);

--connection server2
--echo server2

--source include/start_group_replication.inc

--let $assert_text= On the recovered member, the table should contain 8 elements
--let $assert_cond= [select count(*) from t1] = 8;
--source include/assert.inc

--echo #Test if the group responds to requests
INSERT INTO t1 VALUES (9);

--connection server1
--echo server1

--let $wait_condition= select count(*) = 9 from t1;
--source include/wait_condition.inc

--let $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc

--echo #
--echo # Cleaning up
--echo #

DROP TABLE t1;

--source include/group_replication_end.inc