1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
|
################################################################################
# Test to check the behavior of Group Replication members when the crash
# happens on a member followed by a recovery on the same.
#
# Test:
# 0. The test requires two servers: M1 and M2.
# 1. With both the members are ONLINE, crash a member M1 after a transaction but
# before the commit stage using DEBUG POINT.
# 2. Restart the server M1. Data should ROLLBACK post restart.
# 3. Check that no data is replicated on M2.
# 4. Bring M1 ONLINE and check if group is working fine now.
# 5. Crash a member M1 before the data is committed to make crash for
# transaction using DEBUG POINT.
# 6. Restart the server M1. Data should be recovered post restart.
# 7. Check that data is replicated on M2.
# 8. Bring M1 ONLINE and check if group is working fine now.
# 9. Set DEBUG POINT after binlogging half of the transaction and then crash
# the member M1.
# 10. Restart the server M1. Data will not be recovered until GR start.
# 11. Bring M1 ONLINE and check data is recovered from M2.
# 12. Check if all members are running and group is working fine.
################################################################################
--source include/not_crashrep.inc
# Don't test this under valgrind, memory leaks will occur
--source include/not_valgrind.inc
--source include/big_test.inc
--source include/have_debug.inc
--source include/force_restart.inc
--source include/have_group_replication_plugin.inc
--source include/group_replication.inc
SET sql_log_bin=0;
call mtr.add_suppression("Attempting backtrace");
call mtr.add_suppression("allocated tablespace *., old maximum was 0");
call mtr.add_suppression("Error in Log_event::read_log_event()");
call mtr.add_suppression("Buffered warning: Performance schema disabled");
SET sql_log_bin=1;
--let $_group_replication_local_address_server1= `SELECT @@GLOBAL.group_replication_local_address`
--let $_group_replication_group_seeds_server1= `SELECT @@GLOBAL.group_replication_group_seeds`
--let $member1_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
--let $old_debug = `select @@global.debug`
--echo # Test case1: Set DEBUG POINT before binlog to make
--echo # the first member crash for transaction
CREATE TABLE t1(i INT, a LONGBLOB, PRIMARY KEY(i)) ENGINE=INNODB;
BEGIN;
--let $rows= 3
WHILE($rows)
{
--eval INSERT INTO t1 VALUES ($rows, REPEAT('a',2));
dec $rows;
}
# Write file to make mysql-test-run.pl expect crash and restart
--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--disable_query_log
--eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,crash_commit_after_prepare")
--enable_query_log
# Run the crashing query
--error 2013
COMMIT;
--enable_reconnect
--echo # Restart the crashed member
--exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--source include/wait_until_connected_again.inc
--disable_reconnect
--let $rpl_server_number= 1
--source include/rpl_reconnect.inc
--let $assert_cond= `SELECT COUNT(*) = 0 FROM t1`
--let $assert_text= On server1, test the data will be rolled back after restart.
--source include/assert.inc
# Sync the second member with the first one.
--source include/rpl_sync.inc
--echo # On the second member, test replication will work fine, and the data
--echo # is not replicated
--let $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc
# Wait until group has only 1 member, that is, server1 crash is detected by
# the group.
--let $rpl_connection_name= server2
--source include/rpl_connection.inc
--let $group_replication_member_state= UNREACHABLE
--let $group_replication_member_id= $member1_uuid
--source include/gr_wait_for_member_state.inc
# unblock the group, since server1 has gone away and
# there were only two in the group - majority lost
--let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
--disable_query_log
# Reseting the group to server2 only
--eval SET GLOBAL group_replication_force_members= "$local_address_server2"
--enable_query_log
--let $group_replication_number_of_members= 1
--source include/gr_wait_for_number_of_members.inc
--let $rpl_connection_name= server1
--source include/rpl_connection.inc
--source include/start_group_replication.inc
# Check if the group is still working
CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
INSERT INTO temp VALUES (1);
--source include/rpl_sync.inc
--connection server2
INSERT INTO temp VALUES (2);
--source include/rpl_sync.inc
--let $diff_tables= server1:test.temp, server2:test.temp
--source include/diff_tables.inc
DROP TABLE t1;
DROP TABLE temp;
--source include/rpl_sync.inc
--echo # Test case2: Set DEBUG POINT after binlog, and before the data
--echo # is committed to make crash for transaction
--connection server1
CREATE TABLE t1(i INT, a LONGBLOB, PRIMARY KEY(i)) ENGINE=INNODB;
BEGIN;
--let $rows= 3
WHILE($rows)
{
--eval INSERT INTO t1 VALUES ($rows, REPEAT('a',2));
--dec $rows
}
# Write file to make mysql-test-run.pl expect crash and restart
--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--disable_query_log
--eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,crash_commit_after_log")
--enable_query_log
# Run the crashing query
--error 2013
COMMIT;
--source include/wait_until_disconnected.inc
--enable_reconnect
--echo # Restart the crashed member
--exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--source include/wait_until_connected_again.inc
--disable_reconnect
--let $rpl_server_number= 1
--source include/rpl_reconnect.inc
--let $assert_cond= `SELECT COUNT(*) = 3 FROM t1`
--let $assert_text= On server1, test the data will be recovered after the server1 restart.
--source include/assert.inc
--echo # On the second member, test that the replication is working fine, and
--echo # data gets replicated here.
--let $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc
# Wait until group has only 1 member, that is, server1 crash is detected by
# the group.
--let $rpl_connection_name= server2
--source include/rpl_connection.inc
--let $group_replication_member_state= UNREACHABLE
--let $group_replication_member_id= $member1_uuid
--source include/gr_wait_for_member_state.inc
# unblock the group, since server2 has gone away and
# there were only two in the group - majority lost
--let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
--disable_query_log
# Reseting the group to server2 only
--eval SET GLOBAL group_replication_force_members= "$local_address_server2"
--enable_query_log
--let $group_replication_number_of_members= 1
--source include/gr_wait_for_number_of_members.inc
--let $rpl_connection_name= server1
--source include/rpl_connection.inc
--source include/start_group_replication.inc
# Check if the group is still working
CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
INSERT INTO temp VALUES (1);
--source include/rpl_sync.inc
--connection server2
INSERT INTO temp VALUES (2);
--source include/rpl_sync.inc
--let $diff_tables= server1:test.temp, server2:test.temp
--source include/diff_tables.inc
DROP TABLE t1;
DROP TABLE temp;
--source include/rpl_sync.inc
# Test transaction with large data inserted
--connection server1
CREATE TABLE t1(i INT AUTO_INCREMENT PRIMARY KEY, a LONGBLOB) ENGINE=INNODB;
--echo # Test case3: Set DEBUG POINT after binlogging half of the transaction and
--echo # then crash the member and followed by restarting the member.
set AUTOCOMMIT=0;
BEGIN;
--let $rows= 24
WHILE($rows)
{
INSERT INTO t1 (a) VALUES ( REPEAT('a',6144));
--dec $rows
}
# Write file to make mysql-test-run.pl expect crash and restart
--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--disable_query_log
--eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,half_binlogged_transaction")
--enable_query_log
# Run the crashing query
--error 2013
COMMIT;
--source include/wait_until_disconnected.inc
--enable_reconnect
--echo # Restart the crashed member
--exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--source include/wait_until_connected_again.inc
--disable_reconnect
--let $rpl_server_number= 1
--source include/rpl_reconnect.inc
--echo # Test the data will not be recovered successfully
--echo # after the crashed first member restarts.
--let $assert_cond= `SELECT COUNT(*) = 0 FROM t1`
--let $assert_text= On server1, test the data will be recovered after the server1 restart.
--source include/assert.inc
# Wait until group has only 1 member, that is, server1 crash is detected by
# the group.
--let $rpl_connection_name= server2
--source include/rpl_connection.inc
--let $group_replication_member_state= UNREACHABLE
--let $group_replication_member_id= $member1_uuid
--source include/gr_wait_for_member_state.inc
# unblock the group, since server2 has gone away and
# there were only two in the group - majority lost
--let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
--disable_query_log
# Reseting the group to server2 only
--eval SET GLOBAL group_replication_force_members= "$local_address_server2"
--enable_query_log
--let $group_replication_number_of_members= 1
--source include/gr_wait_for_number_of_members.inc
--let $rpl_connection_name= server1
--source include/rpl_connection.inc
--source include/start_group_replication.inc
# On restarting group replication on the first member it will be getting the transactions
# from a donor and try to get in sync with the other members in the group.
--let $diff_tables= server1:test.t1, server2:test.t1
--source include/diff_tables.inc
--connection server2
--let $assert_cond= `SELECT COUNT(*) = 24 FROM t1`
--let $assert_text= The table has 24 rows after the restart.
--source include/assert.inc
# Check if the group is still running
--connection server1
CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
INSERT INTO temp VALUES (1);
--source include/rpl_sync.inc
--connection server2
INSERT INTO temp VALUES (2);
--source include/rpl_sync.inc
--let $diff_tables= server1:test.temp, server2:test.temp
--source include/diff_tables.inc
DROP TABLE t1;
DROP TABLE temp;
--source include/group_replication_end.inc
|
