File: ndb_binlog_restart.test

package info (click to toggle)
mysql-8.0 8.0.43-3
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 1,273,924 kB
  • sloc: cpp: 4,684,605; ansic: 412,450; pascal: 108,398; java: 83,641; perl: 30,221; cs: 27,067; sql: 26,594; sh: 24,181; python: 21,816; yacc: 17,169; php: 11,522; xml: 7,388; javascript: 7,076; makefile: 2,194; lex: 1,075; awk: 670; asm: 520; objc: 183; ruby: 97; lisp: 86
file content (267 lines) | stat: -rw-r--r-- 9,615 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
#############################################################
# Test behaviout of the binlog injector during restart of itself.
# A restart should be triggered when the mysqld disconnects from
# the datanodes.
#
#############################################################

-- source include/have_binlog_format_mixed_or_row.inc
-- source include/have_multi_ndb.inc

# Ignore the warning generated by ndbcluster's binlog thread
# when mysqld is disconnected from cluster
--disable_query_log ONCE
call mtr.add_suppression("cluster disconnect An incident event has been written");

##################################################
# Check whether we have a debug compiled binary:
# @@global.debug can't be referred without this.
# However, instead of skipping entire test without debug,
# we run it without setting the specific debug flags.
###################################################
--let $have_debug= `select (version() like '%debug%')`

connection server1;
# Find NodeId of the mysqld we are connected to:
--let $node_id= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
--let $node_id= `SELECT SUBSTRING('$node_id', 20)+0`

connection server2;
# Find NodeId2 of 'server2' we are connected to:
--let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
--let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0`

# Ignore the warning generated by ndbcluster's binlog thread
# when mysqld is disconnected from cluster also on second Server
--disable_query_log ONCE
call mtr.add_suppression("cluster disconnect An incident event has been written");

connection server1;
reset master;

#############################################################
# A TE_FAILURE event will be received by the binlog thread 
# due to the DUMP disconnecting mysqld from datanodes.
# When binlog handles the FAILURE event, mysqld is put into
# readonly mode. A restart of the Binlog thread is required
# to bring it out of RO-mode.
#############################################################
--echo Disconnect mysqld from all datanodes
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT

#############################################################
# The disconnect will create a GAP event in the binlog.
# To avoid 'disconnect failures' while probing for 'not_readonly',
# we use the binlog GAP as an indicator that disconnect has
# already completed.
#############################################################
--echo Wait for binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

#############################################################
# Until the binlog injector has completed the restart all tables
# will temporarily be in a read-only mode.
# (As the binlog will not be able to log any updates)
#############################################################
--echo Wait to get out of read-only mode
--source include/ndb_not_readonly.inc

#############################################################
# During the restart a 'Gap event' should be written to the
# binlog and the log rotated to 'binlog.000002'.
#############################################################
--echo
--echo Check that the binlog was rotated:
--echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002'
--source include/show_binlog_events.inc


#############################################################
# Bug#22204186: NDB REPLICATION : READ ONLY TABLES AND NO BINLOGGING
#
# Tables could get stuck in read only mode due to slow handling 
# of cluster failure in the binlog.
#
# Testcase is identical to the one above, except that
# we use debug inserts to force the binlog injector to take
# a break when handling failures. When not using debug compiled
# binaries, there will be no such breaks inserted, but the
# test should still succeed.
#
# This previously caused a race condition where the Util-thread
# managed to recreate the schema distr. tables before
# the binlog thread had completed the failure handling. Thus,
# the binlog never reached its restart condition of all
# eventOps have been dropped.
#############################################################

--echo
--echo Testing for Bug#22204186: Stuck in Readonly if slow failure handling
--echo ====================================================================

disable_query_log;
if ($have_debug)
{
  set @save_debug= @@global.debug;
  set global debug='+d,ndb_binlog_slow_failure_handling';
}
enable_query_log;

reset master;

--echo Disconnect mysqld from all datanodes
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT

--echo Wait for binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

--echo Wait to get out of read-only mode
--source include/ndb_not_readonly.inc

--echo
--echo Check that the binlog was rotated:
--echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002'
--source include/show_binlog_events.inc

disable_query_log;
if ($have_debug)
{
  set global debug= @save_debug;
}
enable_query_log;

#############################################################
# Bug#22494024
#  SQL NODE FAILS TO RECONNECT AFTER NETWORK OUTAGE; BLOCKS OTHER NODES
#
# If a mysqld completes ndb_binlog_setup() before the others,
# and executes a schema change operation which has to be distributed,
# it can grab the MDL lock, which is also required by the other
# mysqlds to complete ndb_binlog_setup(). Thus we effectively
# deadlocks on the MDL.
#
# This could happen if one of the mysqlds opens the special table
# 'ndb_schema' *without* also completing the ndb_binlog_setup()
# in the same 'atomic' (MDL locked) operation. Opening of this table
# will cause a SUBSCRIBE-event to be sent to all other mysqlds,
# which is the signal used to enroll this node into the 
# schema distribution. However, as it is not yet 'setup', it
# will not yet act on schema distr. events, thus blocking other
# mysqlds waiting on its response.
#
#############################################################

--echo
--echo Testcase1 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock)
--echo    ndb_binlog_setup() fails to fully complete and does not clean up
--echo    its partial setup binlog subscriptions on exit
--echo =============================================================================

--echo Prepare mysqld 'server2' to not entirely complete binlog_setup
disable_query_log;
if ($have_debug)
{
  connection server2;
  set @save_debug= @@global.debug;
  set global debug='+d,ndb_binlog_setup_incomplete';
  connection server1;
}
enable_query_log;

--echo Disconnect both mysqld servers from all datanodes -> binlog restarts
--exec $NDB_MGM -e "ALL DUMP 900 $node_id"  > $NDB_TOOLS_OUTPUT
--exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT

connection server1;
--echo Wait for 'server1' binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

connection server2;
--echo Wait for 'server2' binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

connection server1;
--echo Wait for 'server1' to complete setup and get out of read-only mode
--source include/ndb_not_readonly.inc

--echo Give 'server2' some time to start, and fail, a binlog_setup
sleep 2;

connection server1;
--echo Create table operation on 'server1' grab MDL lock while waiting for
--echo   schema change distribution to all subscribing mysqld.
create table t1 (a int primary key) engine=ndb;
drop table t1;

disable_query_log;
if ($have_debug)
{
  connection server2;
  set global debug= @save_debug;
  connection server1;
}
enable_query_log;

connection server2;
--echo Wait for 'server2' to complete setup and get out of read-only mode
--source include/ndb_not_readonly.inc


--echo
--echo Testcase2 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock)
--echo   Table 'ndb_schema' is accessed from a SQL query. Causing it to be opened
--echo   and a SUBSCRIBE-event sent to other mysqld.
--echo =============================================================================

--echo Prepare mysqld 'server2' to do slow binlog_setup upon restart
disable_query_log;
if ($have_debug)
{
  connection server2;
  set @save_debug= @@global.debug;
  set global debug='+d,ndb_binlog_setup_slow';
  connection server1;
}
enable_query_log;

--echo Disconnect both mysqld servers from all datanodes -> binlog restarts
--exec $NDB_MGM -e "ALL DUMP 900 $node_id"  > $NDB_TOOLS_OUTPUT
--exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT

connection server1;
--echo Wait for 'server1' binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

connection server2;
--echo Wait for 'server2' binlog rotate to indicate disconnect
--let $wait_binlog_event= binlog.000002
--source include/wait_for_binlog_event.inc

connection server1;
--echo Wait for (fast) 'server1' to complete setup and get out of read-only mode
--source include/ndb_not_readonly.inc

connection server1;
--echo Create table operation on 'server1' grab MDL lock while waiting for
--echo schema distribution to all subscribing mysqld -> stuck on 'server2'
create table t1 (a int primary key) engine=ndb;
drop table t1;

disable_query_log;
if ($have_debug)
{
  connection server2;
  set global debug= @save_debug;
  connection server1;
}
enable_query_log;

connection server2;
--echo Wait for 'server2' to complete setup and get out of read-only mode
--source include/ndb_not_readonly.inc
--remove_file $NDB_TOOLS_OUTPUT