1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
summary: Ensures that a failed snap auto-refresh will be will be exponentially delayed in future refreshes.
details: |
Test ensures that if a snap auto-refresh failed the next auto-refresh attempt for the
same revision will be delayed. This checks that the auto-refresh backoff algorithm is
implemented properly.
environment:
BLOB_DIR: $(pwd)/fake-store-blobdir
SNAP_ONE: test-snapd-tools
SNAP_ONE_ID: eFe8BTR5L5V9F7yHeMAPxkEr2NdUXMtw
SNAP_TWO: test-snapd-sh
SNAP_TWO_ID: WOc8eDNKuk1POWZIfcCX08smZrUGY0QV
prepare: |
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
# Install snaps as baseline since we want to test what happens in refreshes not installs
echo "Given installed snaps"
snap install "$SNAP_ONE" "$SNAP_TWO"
echo "And the daemon is configured to point to the fake store"
"$TESTSTOOLS"/store-state setup-fake-store "$BLOB_DIR"
echo "Expose the needed assertions through the fakestore"
cp "$TESTSLIB"/assertions/testrootorg-store.account-key "$BLOB_DIR/asserts"
cp "$TESTSLIB"/assertions/developer1.account "$BLOB_DIR/asserts"
cp "$TESTSLIB"/assertions/developer1.account-key "$BLOB_DIR/asserts"
# It is not enough to copy the assertions, we must also ack them otherwise we
# will get an error about not being able to resolve the account key
snap ack "$BLOB_DIR/asserts/testrootorg-store.account-key"
snap ack "$BLOB_DIR/asserts/developer1.account"
snap ack "$BLOB_DIR/asserts/developer1.account-key"
restore: |
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
"$TESTSTOOLS"/store-state teardown-fake-store "$BLOB_DIR"
rm -rf "$BLOB_DIR"
debug: |
snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\")] | sort_by(.id|tonumber)"
execute: |
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
SNAP_MOUNT_DIR="$(os.paths snap-mount-dir)"
SNAP_ONE_GOOD_PATH=$("$TESTSTOOLS"/snaps-state pack-local "$SNAP_ONE")
SNAP_TWO_GOOD_PATH=$("$TESTSTOOLS"/snaps-state pack-local "$SNAP_TWO")
# Make bad version of SNAP_ONE
unsquashfs -d bad-snap "$SNAP_ONE_GOOD_PATH"
mkdir -p ./bad-snap/meta/hooks
cat <<'EOF' > ./bad-snap/meta/hooks/configure
#!/bin/sh
exit 1
EOF
chmod +x ./bad-snap/meta/hooks/configure
snap pack --filename="$SNAP_ONE-bad.snap" bad-snap .
SNAP_ONE_BAD_PATH="$(pwd)/$SNAP_ONE-bad.snap"
# Prevent refreshes until we have right snap revisions
snap set system refresh.hold=forever
# Make snaps refreshable from fakestore
"$TESTSTOOLS"/store-state init-fake-refreshes "$BLOB_DIR" "$SNAP_ONE" --snap-blob="$SNAP_ONE_GOOD_PATH"
"$TESTSTOOLS"/store-state init-fake-refreshes "$BLOB_DIR" "$SNAP_TWO" --snap-blob="$SNAP_TWO_GOOD_PATH"
add_snap_to_fakestore() {
SNAP_FILE="$1"
SNAP_ID="$2"
SNAP_REV="$3"
# Rebuild snap with $SNAP_REV written into a file inside it to force a new snap-sha3-384
# hash in snap-revision assertion
unsquashfs -d /tmp/fake-snap "$SNAP_FILE"
# Force new snap-sha3-384 hash for snap
echo "$SNAP_REV" > /tmp/fake-snap/rev
snap pack --filename="$SNAP_ID-rev-$SNAP_REV.snap" /tmp/fake-snap .
rm -rf /tmp/fake-snap
"$TESTSTOOLS"/store-state make-snap-installable --revision "$SNAP_REV" "$BLOB_DIR" "$(pwd)/$SNAP_ID-rev-$SNAP_REV.snap" "$SNAP_ID"
}
# Record last change id before we start to avoid flakiness due to auto-refreshes in other tests
LAST_CHANGE_ID=$(snap debug api /v2/changes?select=all | gojq '.result | sort_by(.id|tonumber) | .[-1].id')
# -------- FIRST AUTO REFRESH --------
# Clean old snaps in fakestore directory because the fakestore can't distinguish
# multiple snaps files for the same snap
rm "$BLOB_DIR"/*.snap
echo "Make new revisions available to both snaps and break $SNAP_ONE"
add_snap_to_fakestore "$SNAP_ONE_BAD_PATH" "$SNAP_ONE_ID" 11
add_snap_to_fakestore "$SNAP_TWO_GOOD_PATH" "$SNAP_TWO_ID" 11
# Ensure there are no refresh holds, otherwise can't force auto-refresh
snap set system refresh.hold!
echo "Trigger auto-refresh"
systemctl stop snapd.{service,socket}
"$TESTSTOOLS"/snapd-state force-autorefresh
systemctl start snapd.{service,socket}
# Wait until auto-refresh is triggered and completed
retry -n 120 --wait 1 "$(pwd)"/check_auto_refresh_count.sh "$LAST_CHANGE_ID" 1
# Check log is emitted about failed auto-refresh for SNAP_ONE
retry -n 50 --wait 1 sh -c 'journalctl -b -u snapd | MATCH "auto-refresh to revision 11 has failed, next auto-refresh attempt will be delayed by 8 hours"'
echo "Check we have expected revisions for both snaps"
readlink "$SNAP_MOUNT_DIR/$SNAP_ONE/current" | NOMATCH 11
readlink "$SNAP_MOUNT_DIR/$SNAP_TWO/current" | MATCH 11
# -------- SECOND AUTO REFRESH --------
# Clean old SNAP_TWO snap in fakestore directory and keep bad SNAP_ONE
rm "$BLOB_DIR/$SNAP_TWO_ID-rev-11.snap"
echo "Add new revision for $SNAP_TWO while keeping bad revision of $SNAP_ONE"
add_snap_to_fakestore "$SNAP_TWO_GOOD_PATH" "$SNAP_TWO_ID" 22
echo "Trigger auto-refresh a second time"
systemctl stop snapd.{service,socket}
"$TESTSTOOLS"/snapd-state force-autorefresh
systemctl start snapd.{service,socket}
# Wait until auto-refresh is triggered and completed
retry -n 120 --wait 1 "$(pwd)"/check_auto_refresh_count.sh "$LAST_CHANGE_ID" 2
# Check log is emitted about skipping auto-refresh of bad revision for SNAP_ONE
retry -n 50 --wait 1 sh -c 'journalctl -b -u snapd | MATCH "auto-refresh to revision 11 was skipped due to previous failures, next auto-refresh attempt will be delayed by 8 hours"'
echo "Check we have expected revisions for both snaps"
readlink "$SNAP_MOUNT_DIR/$SNAP_ONE/current" | NOMATCH 11
readlink "$SNAP_MOUNT_DIR/$SNAP_TWO/current" | MATCH 22
# -------- THIRD AUTO REFRESH --------
# Clean old snaps in fakestore directory because the fakestore can't distinguish
# multiple snaps files for the same snap
rm "$BLOB_DIR"/*.snap
echo "Fix $SNAP_ONE in new revision"
add_snap_to_fakestore "$SNAP_ONE_GOOD_PATH" "$SNAP_ONE_ID" 33
add_snap_to_fakestore "$SNAP_TWO_GOOD_PATH" "$SNAP_TWO_ID" 33
echo "Trigger auto-refresh a third time"
systemctl stop snapd.{service,socket}
"$TESTSTOOLS"/snapd-state force-autorefresh
systemctl start snapd.{service,socket}
# Wait until auto-refresh is triggered and completed
retry -n 120 --wait 1 "$(pwd)"/check_auto_refresh_count.sh "$LAST_CHANGE_ID" 3
echo "Check we have expected revisions for both snaps"
readlink "$SNAP_MOUNT_DIR/$SNAP_ONE/current" | MATCH 33
readlink "$SNAP_MOUNT_DIR/$SNAP_TWO/current" | MATCH 33
echo "Check auto-refresh behaviour matches expectations for backoff algorithm"
snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | sort_by(.id|tonumber)" > changes.json
# 1st auto-refresh
gojq '.[0].status' < changes.json | MATCH "Error"
gojq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_ONE"
gojq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_TWO"
gojq '.[0].data."refresh-failed"' < changes.json | MATCH "$SNAP_ONE"
gojq '.[0].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_TWO"
# 2nd auto-refresh
gojq '.[1].status' < changes.json | MATCH "Done"
# Broken SNAP_ONE should have been skipped this time
gojq '.[1].data."snap-names"' < changes.json | NOMATCH "$SNAP_ONE"
gojq '.[1].data."snap-names"' < changes.json | MATCH "$SNAP_TWO"
gojq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_ONE"
gojq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_TWO"
# 3rd auto-refresh
gojq '.[2].status' < changes.json | MATCH "Done"
gojq '.[2].data."snap-names"' < changes.json | MATCH "$SNAP_ONE"
gojq '.[2].data."snap-names"' < changes.json | MATCH "$SNAP_TWO"
gojq '.[2].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_ONE"
gojq '.[2].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_TWO"
|
