1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
# Populate a table with 1000 records. Allow the replica to sync with the master.
# Run concurrent threads that run OLTP transactions on master.
# Kill the master database server at random points.
# Check the table against the replica.
# Reinvoke the threads.
# Set python interpreter to use (On Windows executable is python.exe)
--let $PYTHON = python3
if (`SELECT CONVERT(@@VERSION_COMPILE_OS USING latin1) IN ('Win32', 'Win64', 'Windows')`)
{
--let $PYTHON = python
}
# create the directory for temporary log files.
--mkdir $MYSQL_TMP_DIR/load_generator
if ($fake_changes)
{
--mkdir $MYSQL_TMP_DIR/load_generator_slave
}
--connection master
CREATE TABLE errors (msg TEXT);
--sync_slave_with_master
--connection master
--let $pid_file = `SELECT @@pid_file`
--let $crash_num = 0
--let $master_host = 127.0.0.1
--let $table = test
--let $user = root
--let $checksum = 0
--let $secondary_index_checks = 0
if ($do_checksum)
{
# populate the table and store its checksum before any load.
let $exec =
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after
$num_records 0 0 $user $master_host $MASTER_MYPORT
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator 0 0 0;
exec $exec >> $MYSQL_TMP_DIR/load_generator/errors.log;
let $checksum=query_get_value(CHECKSUM TABLE t1, Checksum, 1);
# Master needs to be restarted to start with an empty buffer pool so
# that logical read ahead gets used.
let rpl_server_number = 1;
source include/rpl_restart_server.inc;
connection slave;
# Start slave to avoid I/O thread retry errors
disable_warnings;
source include/start_slave.inc;
enable_warnings;
let $num_records = 0;
}
while ($num_crashes)
{
connection master;
exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
if ($crash_num)
{
let $num_records = 0; # do not populate the table except for the first run.
}
if ($use_blob)
{
let $exec =
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after
$num_records $num_workers $num_transactions $user $master_host $MASTER_MYPORT
$table 1 $max_rows $MYSQL_TMP_DIR/load_generator 0 $checksum $secondary_index_checks;
}
if (!$use_blob)
{
let $exec =
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after
$num_records $num_workers $num_transactions $user $master_host $MASTER_MYPORT
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator 0 $checksum $secondary_index_checks;
}
exec $exec >> $MYSQL_TMP_DIR/load_generator/errors.log;
if ($do_crash)
{
--echo Wait for reconnect
enable_reconnect;
# Call script that will poll the server waiting for it to be back online again
source include/wait_until_connected_again.inc;
connection slave;
source include/wait_until_connected_again.inc;
connection master;
}
--echo Checksum master
let $master_checksum = query_get_value(CHECKSUM TABLE t1, Checksum, 1);
# if sync_slave_with_master had a configurable timeout this would not be needed
let $slave_sync_timeout = 7200;
--source include/wait_for_slave_to_sync_with_master.inc
connection slave;
if ($fake_changes)
{
--echo applying fake updates to the slave
let $slave_pid_file = `SELECT @@pid_file`;
let $slave_exec =
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $slave_pid_file $kill_db_after
0 $num_workers $num_transactions $user $master_host $SLAVE_MYPORT
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator_slave 1 $checksum $secondary_index_checks;
exec $slave_exec >> $MYSQL_TMP_DIR/load_generator/errors.log;
}
--echo Checksum slave
let $slave_checksum=query_get_value(CHECKSUM TABLE t1, Checksum, 1);
let $not_same = `SELECT $master_checksum-$slave_checksum`;
if ($not_same)
{
let $msg =
The checksums of table t1 for master and slave do not match for $crash_num th
crash. This may happen if there is a corrupt recovery log or a bug in crash
recovery. You can take a look at the logs in $MYSQL_TMP_DIR/load_generator to see the
queries issued before the crash.;
echo $msg;
connection master;
eval select * into outfile '$MYSQLTEST_VARDIR/tmp/master_all' from t1 order by id;
eval select id into outfile '$MYSQLTEST_VARDIR/tmp/master_id' from t1 order by id;
show master status;
connection slave;
eval select * into outfile '$MYSQLTEST_VARDIR/tmp/slave_all' from t1 order by id;
eval select id into outfile '$MYSQLTEST_VARDIR/tmp/slave_id' from t1 order by id;
show slave status;
die;
}
dec $num_crashes;
inc $crash_num;
}
# final cleanup
--connection master
let $primary=`select count(*) from t1 use index (primary)`;
let $secondary=`select count(*) from t1 use index (msg_i)`;
if ($primary != $secondary)
{
--echo Secondary index inconsistent! $primary != $secondary
--die
}
if (!$do_crash)
{
--cat_file $MYSQL_TMP_DIR/load_generator/errors.log
SELECT * FROM errors;
}
DROP TABLE t1;
DROP TABLE errors;
# if sync_slave_with_master had a configurable timeout this would not be needed
let $slave_sync_timeout = 7200;
--source include/wait_for_slave_to_sync_with_master.inc
--connection slave
--source include/stop_slave.inc
# For stress tests sometimes the replication thread can not connect to master
# temporarily. This is either because the master crashed and it is recovering
# or the master is too busy and could not service the slave's requests.
# mtr's internal check requires that there be no errors in slave status.
# restarting replication clears the errors.
--source include/start_slave.inc
--source include/stop_slave.inc
connection master;
--force-rmdir $MYSQL_TMP_DIR/load_generator
if ($fake_changes)
{
--force-rmdir $MYSQL_TMP_DIR/load_generator_slave
}
# Reconnect client in case master had been killed
if ($kill_db_after)
{
--let $rpl_server_number = 1
--source include/rpl_reconnect.inc
}
--let $rpl_only_running_threads= 1
--source include/rpl_end.inc
|
