1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
#!/usr/bin/env perl
use strict;
use warnings;
use Test::More;
use FindBin qw($Bin);
use lib "$Bin/lib";
use Carp qw(croak);
use MemcachedTest;
use IO::Select;
use IO::Socket qw(AF_INET SOCK_STREAM);
if (!supports_proxy()) {
plan skip_all => 'proxy not enabled';
exit 0;
}
# TODO: getting log lines is unreliable with the method of localhost testing
# that I'm doing.
# Probably need to get more complicated:
# - Loop-close while polling the backend socket for a new connect
# - This is because it'll stop attempting to connect...
# - Once that poll times out, scan all watcher logs for the markedbad line
# - I think there may also be a bug where if a backend fails in readvalidate
# it doesn't get counted against the bad count.
plan skip_all => 'flappy test';
# Set up some server sockets.
sub mock_server {
my $port = shift;
my $srv = IO::Socket->new(
Domain => AF_INET,
Type => SOCK_STREAM,
Proto => 'tcp',
LocalHost => '127.0.0.1',
LocalPort => $port,
ReusePort => 1,
Listen => 5) || die "IO::Socket: $@";
return $srv;
}
sub accept_backend {
my $srv = shift;
my $be = $srv->accept();
$be->autoflush(1);
ok(defined $be, "mock backend created");
like(<$be>, qr/version/, "received version command");
print $be "VERSION 1.0.0-mock\r\n";
return $be;
}
# Put a version command down the pipe to ensure the socket is clear.
# client version commands skip the proxy code
sub check_version {
my $ps = shift;
print $ps "version\r\n";
like(<$ps>, qr/VERSION /, "version received");
}
sub wait_reload {
my $w = shift;
like(<$w>, qr/ts=(\S+) gid=\d+ type=proxy_conf status=start/, "reload started");
like(<$w>, qr/ts=(\S+) gid=\d+ type=proxy_conf status=done/, "reload completed");
}
# We just need a single backend here; there's no logical difference if it's in
# a cluster or not.
note "making mock servers";
my $msrv = mock_server(11799);
# Start up a clean server.
my $p_srv = new_memcached('-o proxy_config=./t/proxyantiflap.lua -t 1');
my $ps = $p_srv->sock;
$ps->autoflush(1);
{
my $watcher = $p_srv->new_sock;
print $watcher "watch proxyevents\n";
is(<$watcher>, "OK\r\n", "watcher enabled");
my $be = accept_backend($msrv);
my $s = IO::Select->new();
$s->add($be);
# Make some backend requests but refuse to answer.
for (1 .. 3) {
print $ps "mg foo\r\n";
$be->close();
# Block until we error and reconnect.
is(scalar <$ps>, "SERVER_ERROR backend failure\r\n", "request cancelled");
like(<$watcher>, qr/error=(disconnected|reading|readvalidate)/, "disconn error log");
$be = accept_backend($msrv);
}
print $ps "mg bar\r\n";
$be->close();
# Block until we error and reconnect.
is(scalar <$ps>, "SERVER_ERROR backend failure\r\n", "request cancelled");
$be = accept_backend($msrv);
like(<$watcher>, qr/error=markedbadflap name=\S+ port=\S+ label=b\d+ retry=1/, "got caught flapping");
print $ps "mg baz\r\n";
is(scalar <$be>, "mg baz\r\n", "backend does reconnect and still works");
print $be "HD\r\n";
is(scalar <$ps>, "HD\r\n", "client still works post-flap");
# clear error logs.
like(<$watcher>, qr/error=(disconnected|reading)/, "disconn error log");
like(<$watcher>, qr/error=markedbadflap name=\S+ port=\S+ label=b\d+ retry=2/, "re-flapped, longer retry");
$p_srv->reload();
wait_reload($watcher);
# Hold the previous backend so its descriptor doesn't log
my $oldbe = $be;
# This should reset the flap counter as the backend description changes.
$be = accept_backend($msrv);
check_version($ps);
# Make some backend requests but refuse to answer.
for (1 .. 3) {
print $ps "mg foo\r\n";
$be->close();
# Block until we error and reconnect.
$be = accept_backend($msrv);
}
print $ps "mg bar\r\n";
$be->close();
# Block until we error and reconnect.
is(scalar <$ps>, "SERVER_ERROR backend failure\r\n", "request cancelled");
my $flapfound = 0;
for (1 .. 10) {
is(scalar <$ps>, "SERVER_ERROR backend failure\r\n", "request cancelled");
my $line = scalar <$watcher>;
if ($line =~ m/markedbadflap/) {
like($line, qr/error=markedbadflap/, "markedbadflap log");
$flapfound = 1;
last;
} else {
like($line, qr/error=(disconnected|reading|readvalidate)/, "disconn error log");
}
}
is($flapfound, 1, "got caught flapping");
$be = accept_backend($msrv);
like(<$watcher>, qr/error=(disconnected|reading)/, "previous backend goes away");
like(<$watcher>, qr/error=markedbadflap/, "still considered flapping");
check_version($ps);
# verify the new backend works.
print $ps "mg baz\r\n";
is(scalar <$be>, "mg baz\r\n", "backend does work");
print $be "HD\r\n";
is(scalar <$ps>, "HD\r\n", "client still works");
# Now wait and see if the next failure causes a flap or not.
sleep 4;
print $ps "mg bar\r\n";
$be->close();
# Block until we error and reconnect.
is(scalar <$ps>, "SERVER_ERROR backend failure\r\n", "request cancelled");
like(<$watcher>, qr/error=(disconnected|reading)/, "normal disconn error");
# This will pause until readvalidate fails, so we can be sure a flap error
# didn't follow the timeout error.
unlike(<$watcher>, qr/error=markedbadflap/, "didn't flap again");
}
done_testing();
|
