1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
# Hey Emacs, this is a -*- shell-script -*- !!! :-)
get_src_socket ()
{
local proto="$1"
local dst_socket="$2"
local pid="$3"
local prog="$4"
local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$"
out=$(netstat -tanp |
egrep "$pat" |
awk '{ print $4 }')
[ -n "$out" ]
}
wait_until_get_src_socket ()
{
local proto="$1"
local dst_socket="$2"
local pid="$3"
local prog="$4"
echo "Waiting for ${prog} to establish connection to ${dst_socket}..."
wait_until 5 get_src_socket "$@"
}
#######################################
check_tickles ()
{
local node="$1"
local test_ip="$2"
local test_port="$3"
local src_socket="$4"
try_command_on_node $node ctdb gettickles $test_ip $test_port
# SRC: 10.0.2.45:49091 DST: 10.0.2.143:445
[ "${out/SRC: ${src_socket} /}" != "$out" ]
}
check_tickles_all ()
{
local numnodes="$1"
local test_ip="$2"
local test_port="$3"
local src_socket="$4"
try_command_on_node all ctdb gettickles $test_ip $test_port
# SRC: 10.0.2.45:49091 DST: 10.0.2.143:445
local t="${src_socket//./\\.}"
local count=$(grep -E -c "SRC: ${t} " <<<"$out" || true)
[ $count -eq $numnodes ]
}
#######################################
# filename will be in $tcpdump_filename, pid in $tcpdump_pid
tcpdump_start ()
{
tcpdump_filter="$1" # global
echo "Running tcpdump..."
tcpdump_filename=$(mktemp)
ctdb_test_exit_hook_add "rm -f $tcpdump_filename"
# The only way of being sure that tcpdump is listening is to send
# some packets that it will see. So we use dummy pings - the -U
# option to tcpdump ensures that packets are flushed to the file
# as they are captured.
local dummy_addr="127.3.2.1"
local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo"
tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" &
ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"
echo "Waiting for tcpdump output file to be ready..."
ping -q "$dummy_addr" >/dev/null 2>&1 &
ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"
tcpdump_listen_for_dummy ()
{
tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1
}
wait_until 10 tcpdump_listen_for_dummy
}
# By default, wait for 1 matching packet.
tcpdump_wait ()
{
local count="${1:-1}"
local filter="${2:-${tcpdump_filter}}"
tcpdump_check ()
{
local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l)
[ $found -ge $count ]
}
echo "Waiting for tcpdump to capture some packets..."
if ! wait_until 30 tcpdump_check ; then
echo "DEBUG AT $(date '+%F %T'):"
local i
for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do
echo "$i"
$i || true
done
return 1
fi
}
tcpdump_show ()
{
local filter="${1:-${tcpdump_filter}}"
tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null
}
tcptickle_sniff_start ()
{
local src="$1"
local dst="$2"
local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}"
local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}"
local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234
local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)"
tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0"
local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})"
tcpdump_start "$filter"
}
tcptickle_sniff_wait_show ()
{
tcpdump_wait 1 "$tcptickle_reset"
echo "GOOD: here are some TCP tickle packets:"
tcpdump_show
}
gratarp_sniff_start ()
{
tcpdump_start "arp host ${test_ip}"
}
gratarp_sniff_wait_show ()
{
tcpdump_wait 2
echo "GOOD: this should be the some gratuitous ARPs:"
tcpdump_show
}
ctdb_test_check_real_cluster ()
{
[ -z "$TEST_LOCAL_DAEMONS" ] || \
die "ERROR: This test must be run against a real/virtual cluster, not local daemons."
local h=$(hostname)
local i
for i in $(onnode -q all hostname) ; do
[ "$h" != "$i" ] || \
die "ERROR: This test must not be run from a cluster node."
done
}
|
