1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
summary: Ensure that ECONNRESET is handled
details: |
ECONNRESET occurs when the server side closes the TCP connection and your request to the
server is not fulfilled. The server responds with the message that the connection,
you are referring to a invalid connection.
This tests checks the snap download retrying mechanism. It uses iptables
to insert a OUTPUT chain rule that drops outgoing TCP packets and triggers
retries. It also checks that a '.partial' file is created for the snap
being downloaded.
# no iptables on core18+
systems: [-ubuntu-core-18-*, -ubuntu-core-2*]
restore: |
echo "Stop the snap download command"
kill -9 "$(pgrep -f 'snap download')" || true
echo "Remove the firewall rule again"
iptables -D OUTPUT -m owner --uid-owner "$(id -u test)" -j REJECT -p tcp --reject-with tcp-reset || true
echo "Remove ingress traffic policing rule"
iptables -D INPUT -p tcp --match hashlimit --hashlimit-mode srcip,dstip,srcport,dstport --hashlimit-above 512kb/s \
--hashlimit-name 'econnreset' -j DROP
debug: |
echo "Partial download status"
ls -lh test-snapd-huge_* || true
echo "other dir content"
ls -lh
echo "download log:"
cat snap-download.log || true
echo "iptables rules and counters"
iptables -L -n -v || true
execute: |
echo "Downloading a large snap in the background"
rm -f test-snapd-huge_*
# what happens in this test is that when running in GCE backend, the
# downloads is very fast, ~100MB/s and it may finish 'before' we insert the
# OUTPUT chain rule that drops outgoing TCP packets and triggers retries, to
# remedy this apply policing of ingress traffic down to 512kB/s
iptables -I INPUT -p tcp --match hashlimit --hashlimit-mode srcip,dstip,srcport,dstport --hashlimit-above 512kb/s \
--hashlimit-name 'econnreset' -j DROP
su -c "/usr/bin/env SNAPD_DEBUG=1 SNAPD_DEBUG_HTTP=3 snap download --edge test-snapd-huge" test 2>snap-download.log &
for _ in $(seq 120); do
partial=$(find . -name 'test-snapd-huge_*.snap.partial' | head -1)
if [ -n "$partial" ]; then
break
fi
sleep 0.2
done
if [ ! -f "$partial" ]; then
echo "Partial file not found, test broken"
kill -9 "$(pgrep -f 'snap download')" || true
exit 1
fi
echo "Block the download using iptables"
iptables -I OUTPUT -m owner --uid-owner "$(id -u test)" -j REJECT -p tcp --reject-with tcp-reset
echo "Check that we retried"
for _ in $(seq 20); do
if MATCH 'Retrying.*\.snap, attempt 2' < snap-download.log; then
break
fi
echo "Attempt 2 not found, retrying..."
sleep .5
done
|
