1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
#
# Check that the node can cleanly shutdown in case of inconsistency
# (no locked up threads)
#
--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_debug.inc
--source include/have_debug_sync.inc
--source include/galera_have_debug_sync.inc
# Save original auto_increment_offset values.
--let $node_1=node_1
--let $node_2=node_2
--let $node_3=node_3
--source ../galera/include/auto_increment_offset_save.inc
--connection node_2
SELECT @@wsrep_slave_threads = 8;
#
# 1. Inconsistency on slave
#
--connection node_1
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INT);
INSERT INTO t1 VALUES (1, 0),(2, 0),(3, 0),(4, 0),(5, 0),(6, 0),(7, 0),(8, 0);
--connection node_2
# Allow 1K slave queue woithout flow control
SET GLOBAL wsrep_provider_options='gcs.fc_limit=1K';
# Introduce 2 inconsistencies
SET wsrep_on=OFF;
DELETE FROM t1 WHERE f1 = 2;
DELETE FROM t1 WHERE f1 = 4;
SET wsrep_on=ON;
# Build up slave queue:
# - first 8 events will be picked by slave threads
# - one moreevent will be waiting in slave queue
LOCK TABLES t1 WRITE;
--connection node_1
UPDATE t1 SET f2 = 1 WHERE f1 = 1;
UPDATE t1 SET f2 = 1 WHERE f1 = 2;
UPDATE t1 SET f2 = 1 WHERE f1 = 3;
UPDATE t1 SET f2 = 1 WHERE f1 = 4;
UPDATE t1 SET f2 = 2 WHERE f1 = 4; /* dependent applier */
UPDATE t1 SET f2 = 3 WHERE f1 = 4; /* dependent applier */
UPDATE t1 SET f2 = 1 WHERE f1 = 5;
UPDATE t1 SET f2 = 1 WHERE f1 = 6;
UPDATE t1 SET f2 = 1 WHERE f1 = 7;
UPDATE t1 SET f2 = 1 WHERE f1 = 8;
--connection node_2
# make sure all events landed to slave queue
SET wsrep_on=OFF;
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_recv_queue';
--source include/wait_condition.inc
SET wsrep_on=ON;
UNLOCK TABLES;
--source include/wsrep_wait_disconnect.inc
# Wait for the node to shutdown replication
--let $members=0
--source include/wsrep_wait_membership.inc
--connection node_1
--let $members=2
--source include/wsrep_wait_membership.inc
--source include/wait_until_ready.inc
SHOW STATUS LIKE 'wsrep_cluster_size';
SELECT * FROM t1;
--connection node_2
#Gracefully restart the node
SET GLOBAL wsrep_on=OFF;
--source include/shutdown_mysqld.inc
--source include/start_mysqld.inc
--source include/galera_wait_ready.inc
--let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
--source include/wait_condition.inc
DROP TABLE t1;
#
# 2. Inconsistency on master
#
--connection node_1
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INT);
START TRANSACTION;
INSERT INTO t1 VALUES (1, 0);
INSERT INTO t1 VALUES (2, 0);
INSERT INTO t1 VALUES (3, 0);
INSERT INTO t1 VALUES (4, 0);
INSERT INTO t1 VALUES (5, 0);
INSERT INTO t1 VALUES (6, 0);
INSERT INTO t1 VALUES (7, 0);
INSERT INTO t1 VALUES (8, 0);
COMMIT;
CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 INT);
--connection node_2
# Allow 1K slave queue without flow control
SET GLOBAL wsrep_provider_options='gcs.fc_limit=1K';
# Introduce inconsistency
SET wsrep_on=OFF;
--let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2';
--source include/wait_condition.inc
DROP TABLE t2;
SET wsrep_on=ON;
# set up sync point to ensure DROP TABLE replication order below
--let galera_sync_point = after_replicate_sync
--source include/galera_set_sync_point.inc
# Build up slave queue:
# - first 8 events will be picked by slave threads
# - one more event will be waiting in slave queue
LOCK TABLES t1 READ;
--connection node_1
UPDATE t1 SET f2 = 1 WHERE f1 = 1;
UPDATE t1 SET f2 = 1 WHERE f1 = 2;
UPDATE t1 SET f2 = 1 WHERE f1 = 3;
UPDATE t1 SET f2 = 1 WHERE f1 = 4;
UPDATE t1 SET f2 = 2 WHERE f1 = 4; /* dependent applier */;
# interleave a failing statement
--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
--connection node_2a
--send DROP TABLE t2;
# make sure DROP TABLE from above has replicated
--connection node_2
SET wsrep_on=OFF;
--echo "Wait for DROP TABLE to replicate"
--source include/galera_wait_sync_point.inc
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc
--echo "DROP TABLE replicated"
SET wsrep_on=ON;
--connection node_1
UPDATE t1 SET f2 = 3 WHERE f1 = 4; /* dependent applier */
UPDATE t1 SET f2 = 1 WHERE f1 = 5;
UPDATE t1 SET f2 = 1 WHERE f1 = 6;
UPDATE t1 SET f2 = 1 WHERE f1 = 7;
UPDATE t1 SET f2 = 1 WHERE f1 = 8;
--connection node_2
# make sure all events landed to slave queue
SET wsrep_on=OFF;
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_recv_queue';
--source include/wait_condition.inc
SET wsrep_on=ON;
UNLOCK TABLES;
--connection node_2a
--error ER_BAD_TABLE_ERROR
--reap
--connection node_1
--let $members=2
--source include/wsrep_wait_membership.inc
--source include/wait_until_ready.inc
SHOW STATUS LIKE 'wsrep_cluster_size';
SELECT * FROM t1;
--connection node_2
--source include/wsrep_wait_disconnect.inc
# Wait for the node to shutdown replication
--let $members=0
--source include/wsrep_wait_membership.inc
# Gracefully restart the node
SET GLOBAL wsrep_on=OFF;
--source include/shutdown_mysqld.inc
--source include/start_mysqld.inc
--source include/galera_wait_ready.inc
--let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
--source include/wait_condition.inc
DROP TABLE t1;
CALL mtr.add_suppression("Can't find record in 't1'");
CALL mtr.add_suppression("Update_rows_v1 apply failed");
CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus on");
CALL mtr.add_suppression("last left .* greater than drain seqno");
CALL mtr.add_suppression("WSREP: Failed to apply write set: ");
# Restore original auto_increment_offset values.
--source ../galera/include/auto_increment_offset_restore.inc
|
