1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
|
start_server {tags {"repl network external:skip singledb:skip"} overrides {save {}}} {
start_server { overrides {save {}}} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
test {First server should have role slave after SLAVEOF} {
$slave slaveof $master_host $master_port
wait_for_condition 50 100 {
[s 0 role] eq {slave}
} else {
fail "Replication not started."
}
}
test {Test replication with parallel clients writing in different DBs} {
# Gives the random workloads a chance to add some complex commands.
after 5000
# Make sure all parallel clients have written data.
wait_for_condition 1000 50 {
[$master select 9] == {OK} && [$master dbsize] > 0 &&
[$master select 11] == {OK} && [$master dbsize] > 0 &&
[$master select 12] == {OK} && [$master dbsize] > 0
} else {
fail "Parallel clients are not writing in different DBs."
}
stop_bg_complex_data $load_handle0
stop_bg_complex_data $load_handle1
stop_bg_complex_data $load_handle2
wait_for_condition 100 100 {
[$master debug digest] == [$slave debug digest]
} else {
set csv1 [csvdump r]
set csv2 [csvdump {r -1}]
set fd [open /tmp/repldump1.txt w]
puts -nonewline $fd $csv1
close $fd
set fd [open /tmp/repldump2.txt w]
puts -nonewline $fd $csv2
close $fd
fail "Master - Replica inconsistency, Run diff -u against /tmp/repldump*.txt for more info"
}
}
}
}
start_server {tags {"repl external:skip"}} {
start_server {} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
# Load some functions to be used later
$master FUNCTION load replace {#!lua name=test
redis.register_function{function_name='f_default_flags', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={}}
redis.register_function{function_name='f_no_writes', callback=function(keys, args) return redis.call('get',keys[1]) end, flags={'no-writes'}}
}
test {First server should have role slave after SLAVEOF} {
$slave slaveof $master_host $master_port
wait_replica_online $master
}
test {With min-slaves-to-write (1,3): master should be writable} {
$master config set min-slaves-max-lag 3
$master config set min-slaves-to-write 1
assert_equal OK [$master set foo 123]
assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0]
}
test {With min-slaves-to-write (2,3): master should not be writable} {
$master config set min-slaves-max-lag 3
$master config set min-slaves-to-write 2
assert_error "*NOREPLICAS*" {$master set foo bar}
assert_error "*NOREPLICAS*" {$master eval "redis.call('set','foo','bar')" 0}
}
test {With min-slaves-to-write function without no-write flag} {
assert_error "*NOREPLICAS*" {$master fcall f_default_flags 1 foo}
assert_equal "12345" [$master fcall f_no_writes 1 foo]
}
test {With not enough good slaves, read in Lua script is still accepted} {
$master config set min-slaves-max-lag 3
$master config set min-slaves-to-write 1
$master eval "redis.call('set','foo','bar')" 0
$master config set min-slaves-to-write 2
$master eval "return redis.call('get','foo')" 0
} {bar}
test {With min-slaves-to-write: master not writable with lagged slave} {
$master config set min-slaves-max-lag 2
$master config set min-slaves-to-write 1
assert_equal OK [$master set foo 123]
assert_equal OK [$master eval "return redis.call('set','foo',12345)" 0]
# Killing a slave to make it become a lagged slave.
pause_process [srv 0 pid]
# Waiting for slave kill.
wait_for_condition 100 100 {
[catch {$master set foo 123}] != 0
} else {
fail "Master didn't become readonly"
}
assert_error "*NOREPLICAS*" {$master set foo 123}
assert_error "*NOREPLICAS*" {$master eval "return redis.call('set','foo',12345)" 0}
resume_process [srv 0 pid]
}
}
}
start_server {tags {"repl external:skip"}} {
start_server {} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
test {First server should have role slave after SLAVEOF} {
$slave slaveof $master_host $master_port
wait_for_condition 50 100 {
[s 0 master_link_status] eq {up}
} else {
fail "Replication not started."
}
}
test {Replication of an expired key does not delete the expired key} {
# This test is very likely to do a false positive if the wait_for_ofs_sync
# takes longer than the expiration time, so give it a few more chances.
# Go with 5 retries of increasing timeout, i.e. start with 500ms, then go
# to 1000ms, 2000ms, 4000ms, 8000ms.
set px_ms 500
for {set i 0} {$i < 5} {incr i} {
wait_for_ofs_sync $master $slave
$master debug set-active-expire 0
$master set k 1 px $px_ms
wait_for_ofs_sync $master $slave
pause_process [srv 0 pid]
$master incr k
after [expr $px_ms + 1]
# Stopping the replica for one second to makes sure the INCR arrives
# to the replica after the key is logically expired.
resume_process [srv 0 pid]
wait_for_ofs_sync $master $slave
# Check that k is logically expired but is present in the replica.
set res [$slave exists k]
set errcode [catch {$slave debug object k} err] ; # Raises exception if k is gone.
if {$res == 0 && $errcode == 0} { break }
set px_ms [expr $px_ms * 2]
} ;# for
if {$::verbose} { puts "Replication of an expired key does not delete the expired key test attempts: $i" }
assert_equal $res 0
assert_equal $errcode 0
}
}
}
start_server {tags {"repl external:skip"}} {
start_server {} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
test {First server should have role slave after SLAVEOF} {
$slave slaveof $master_host $master_port
wait_for_condition 50 100 {
[s 0 role] eq {slave}
} else {
fail "Replication not started."
}
}
test {Replication: commands with many arguments (issue #1221)} {
# We now issue large MSET commands, that may trigger a specific
# class of bugs, see issue #1221.
for {set j 0} {$j < 100} {incr j} {
set cmd [list mset]
for {set x 0} {$x < 1000} {incr x} {
lappend cmd [randomKey] [randomValue]
}
$master {*}$cmd
}
set retry 10
while {$retry && ([$master debug digest] ne [$slave debug digest])}\
{
after 1000
incr retry -1
}
assert {[$master dbsize] > 0}
}
test {spopwithcount rewrite srem command} {
$master del myset
set content {}
for {set j 0} {$j < 4000} {} {
lappend content [incr j]
}
$master sadd myset {*}$content
$master spop myset 1023
$master spop myset 1024
$master spop myset 1025
assert_match 928 [$master scard myset]
assert_match {*calls=3,*} [cmdrstat spop $master]
wait_for_condition 50 100 {
[status $slave master_repl_offset] == [status $master master_repl_offset]
} else {
fail "SREM replication inconsistency."
}
assert_match {*calls=4,*} [cmdrstat srem $slave]
assert_match 928 [$slave scard myset]
}
test {Replication of SPOP command -- alsoPropagate() API} {
$master del myset
set size [expr 1+[randomInt 100]]
set content {}
for {set j 0} {$j < $size} {incr j} {
lappend content [randomValue]
}
$master sadd myset {*}$content
set count [randomInt 100]
set result [$master spop myset $count]
wait_for_condition 50 100 {
[$master debug digest] eq [$slave debug digest]
} else {
fail "SPOP replication inconsistency"
}
}
}
}
start_server {tags {"repl external:skip"}} {
start_server {} {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set replica [srv 0 client]
test {First server should have role slave after SLAVEOF} {
$replica slaveof $master_host $master_port
wait_for_condition 50 100 {
[s 0 role] eq {slave}
} else {
fail "Replication not started."
}
wait_for_sync $replica
}
test {Data divergence can happen under default conditions} {
$replica config set propagation-error-behavior ignore
$master debug replicate fake-command-1
# Wait for replication to normalize
$master set foo bar2
$master wait 1 2000
# Make sure we triggered the error, by finding the critical
# message and the fake command.
assert_equal [count_log_message 0 "fake-command-1"] 1
assert_equal [count_log_message 0 "== CRITICAL =="] 1
}
test {Data divergence is allowed on writable replicas} {
$replica config set replica-read-only no
$replica set number2 foo
$master incrby number2 1
$master wait 1 2000
assert_equal [$master get number2] 1
assert_equal [$replica get number2] foo
assert_equal [count_log_message 0 "incrby"] 1
}
}
}
|