1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
###############################################################################
# Bug#17283409 4-WAY DEADLOCK: ZOMBIES, PURGING BINLOGS, SHOW PROCESSLIST,
# SHOW BINLOGS
# Problem: A deadlock was occurring when 4 threads were involved in acquiring
# locks
# Thread 1: Dump thread ( Slave is reconnecting, so on Master, a new dump
# thread is trying kill zombie dump threads. It acquired
# thread's LOCK_thd_data and it is about to acquire
# mysys_var->current_mutex ( which LOCK_log)
# Thread 2: Application thread is executing show binlogs and acquired
# LOCK_log and it is about to acquire LOCK_index.
# Thread 3: Applicaiton thread is executing Purge binary logs and acquired
# LOCK_index and it is about to acquire LOCK_thread_count.
# Thread 4: Application thread is executing show processlist and acquired
# LOCK_thread_count and it is about to acquire zombie dump
# thread's LOCK_thd_data.
# Deadlock : Thread 1 -> Thread 2 -> Thread 3-> Thread 4 -> Thread 1
# The same above deadlock was observed when thread 4 is replaced with another
# thread which is executing 'SELECT * FROM information_schema.processlist'
# command and acquired LOCK_thread_count and it is about to acquire zombie
# dump thread's LOCK_thd_data.
# Fix: Introuced a new lock (Lock_thread_remove) which will guard the thread
# deletion (exit process) and the same lock is used in show processlist
# execution instead of LOCK_thread_count. i.e., After the patch, Server allows
# new threads to join the system but it will not allow any threads to execute
# its 'cleanup' process. Now the new lock order is: LOCK_thread_remove ->
# LOCK_thd_data -> LOCK_log -> LOCK_index -> LOCK_thread_count Both the above
# scenarios are tested with the below test code and it proves that there is no
# deadlock after the fix.
###############################################################################
--source include/have_debug_sync.inc
--source include/have_binlog_format_statement.inc
--source include/master-slave.inc
connect(master2,127.0.0.1,root,,test,$MASTER_MYPORT,);
connect(master3,127.0.0.1,root,,test,$MASTER_MYPORT,);
--connection master
SET @save_debug=@@global.debug;
SET GLOBAL DEBUG='+d,before_dump_thread_acquires_current_mutex,processlist_acquiring_dump_threads_LOCK_thd_data';
# Iteration 1 with "SHOW PROCESSLIST command"
# Iteration 2 with "SELECT * FROM information_schema.processlist"
--let $iter=2
while ($iter)
{
# Create atleast one binary log, otherwise purge logs command, show binary
# logs command do not hit the DEBUG_SYNC points.
FLUSH LOGS;
--connection slave
--source include/stop_slave.inc
--source include/start_slave.inc
# Thread 1
--connection master
--echo "Wait_for dump_thread_signal"
SET DEBUG_SYNC='now WAIT_FOR dump_thread_signal';
# Thread 2
--connection master1
SET DEBUG_SYNC='show_binlogs_after_lock_log_before_lock_index SIGNAL parked1 WAIT_FOR go_parked1';
--send SHOW BINARY LOGS
--connection master
echo "Wait_for parked1";
SET DEBUG_SYNC='now WAIT_FOR parked1';
# Thread 3
--connection master2
SET DEBUG_SYNC='purge_logs_after_lock_index_before_thread_count SIGNAL parked2 WAIT_FOR go_parked2';
--send PURGE BINARY LOGS BEFORE '2038-01-19'
--connection master
echo "Wait_for parked2";
SET DEBUG_SYNC='now WAIT_FOR parked2';
# Thread 4
--connection master3
SET DEBUG_SYNC='processlist_after_LOCK_thd_list_before_LOCK_thd_data SIGNAL parked3 WAIT_FOR go_parked3';
if ($iter == 1)
{
--let $query=SHOW PROCESSLIST
}
if ($iter == 2)
{
--let $query=SELECT * FROM information_schema.processlist
}
--send_eval $query
--connection master
echo "Wait_for parked3";
SET DEBUG_SYNC='now WAIT_FOR parked3';
echo "now signal go to all 4 waiting threads";
SET DEBUG_SYNC='now SIGNAL go_dump_thread';
SET DEBUG_SYNC='now SIGNAL go_parked1';
SET DEBUG_SYNC='now SIGNAL go_parked2';
SET DEBUG_SYNC='now SIGNAL go_parked3';
# Output of each command vary depends on other tests ran in suite(output vary
# in Pb2 runs), so lets ignore output. Anyways, we are interested in seeing
# that there is no deadlock.
--disable_result_log
--connection master1
--reap
--connection master2
--reap
--connection master3
--reap
--enable_result_log
--dec $iter
}
--echo "Cleanup"
--disconnect master2
--disconnect master3
--connection master
SET DEBUG_SYNC='RESET';
SET GLOBAL DEBUG=@save_debug;
source include/rpl_end.inc;
|
