1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
summary: check that the core and kernel snaps roll back correctly after a failed upgrade
details: |
This test ensures that the system can survive to a failed upgrade of a fundamental
snap, rolling back to the last good known version.
The logic common to all the scenarios unpacks the target snap, injects the failure,
repacks and installs it. Then it checks that all is set for installed the snap with
the failure and executes a reboot. The test checks that after the reboot (in fact two
reboots, one for trying the upgrade and another for rolling back) the installed
fundamental snap is the good one and the boot environment variables are correctly set.
# TODO: enable for UC20 ?
systems: [ubuntu-core-16-*, ubuntu-core-18-*]
# Start early as it takes a long time.
priority: 100
environment:
INJECT_FAILURE/rclocalcrash: inject_rclocalcrash_failure
INJECT_FAILURE/emptysystemd: inject_emptysystemd_failure
# FIXME: disabled until we find what to do!
# fails with: ERROR cannot replace signed kernel snap with an unasserted one
INJECT_FAILURE/emptyinitrd: inject_emptyinitrd_failure
TARGET_SNAP/rclocalcrash: core
TARGET_SNAP/emptysystemd: core
TARGET_SNAP/emptyinitrd: kernel
BUILD_DIR: /home/tmp
# uploading the core or otherwise large snap triggers OOM
SNAPD_NO_MEMORY_LIMIT: 1
BLOB_DIR: $(pwd)/fake-store-blobdir
SNAP_ID_core20: DLqre5XGLbDqg9jPtiAhRRjDuPVa5X1q
SNAP_ID_core18: CSO04Jhav2yK0uz97cr0ipQRyqg0qQL6
SNAP_ID_core: 99T7MUlRhtI3U0QFgl5mXXESAiSwt776
# pc-kernel snap is specific to x86-64
SNAP_ID_pc_kernel: pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza
prepare: |
if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
echo "variant not supported on ARM architectures"
exit 0
fi
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
snap ack "$TESTSLIB/assertions/testrootorg-store.account-key"
snap ack "$TESTSLIB/assertions/developer1.account"
snap ack "$TESTSLIB/assertions/developer1.account-key"
"$TESTSTOOLS"/store-state setup-fake-store "$BLOB_DIR"
mkdir -p "$BUILD_DIR"
restore: |
if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
echo "variant not supported on ARM architectures"
exit 0
fi
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
"$TESTSTOOLS"/store-state teardown-fake-store "$BLOB_DIR"
rm -rf "$BUILD_DIR"
debug: |
snap debug boot-vars || true
snap list || true
snap changes || true
execute: |
if [ "$TARGET_SNAP" = "kernel" ] && os.query is-arm; then
echo "variant not supported on ARM architectures"
exit 0
fi
if [ "$TRUST_TEST_KEYS" = "false" ]; then
echo "This test needs test keys to be trusted"
exit
fi
inject_rclocalcrash_failure(){
chmod a+x "$BUILD_DIR/unpack/etc/rc.local"
cat <<EOF > "$BUILD_DIR/unpack/etc/rc.local"
#!bin/sh
printf c > /proc/sysrq-trigger
EOF
}
inject_emptysystemd_failure(){
truncate -s 0 "$BUILD_DIR/unpack/lib/systemd/systemd"
}
inject_emptyinitrd_failure(){
truncate -s 0 "$BUILD_DIR/unpack/initrd.img"
}
if os.query is-core18 && [ "$SPREAD_VARIANT" = "rclocalcrash" ]; then
# there is no /etc/rc.local on core18
echo "scenario isn't supported on core18"
exit 0
fi
#shellcheck source=tests/lib/snaps.sh
. "$TESTSLIB"/snaps.sh
core_name="core"
core_snap_id="$SNAP_ID_core"
if os.query is-core18; then
core_name="core18"
core_snap_id="$SNAP_ID_core18"
elif os.query is-core20; then
core_name="core20"
core_snap_id="$SNAP_ID_core20"
fi
TARGET_SNAP_NAME="$core_name"
SNAP_ID="$core_snap_id"
if [ "$TARGET_SNAP" = kernel ]; then
TARGET_SNAP_NAME=pc-kernel
SNAP_ID="$SNAP_ID_pc_kernel"
fi
if [ "$SPREAD_REBOOT" = 0 ]; then
# first pass, save current target snap revision
snap list | awk "/^${TARGET_SNAP_NAME} / {print(\$3)}" > prevBoot
# it is possible that the previous revision of the snap was unasserted
START_REVISION=9999
PREV_REV="$(cat prevBoot)"
if [ "${PREV_REV##x}" = "$PREV_REV" ]; then
# the previous revision has no x prefix, meaning the snap was
# asserted, so just bump the revision number
START_REVISION=$((PREV_REV + 1))
fi
# unpack current target snap
unsquashfs -no-progress -d "$BUILD_DIR/unpack" "/var/lib/snapd/snaps/${TARGET_SNAP_NAME}_$(cat prevBoot).snap"
# set failure condition
eval "${INJECT_FAILURE}"
# repack new target snap
mksnap_fast "$BUILD_DIR/unpack" failing.snap
cat <<EOF > decl-headers.json
{"snap-id": "$SNAP_ID", "snap-name": "$TARGET_SNAP_NAME"}
EOF
cat <<EOF > rev-headers.json
{"snap-id": "$SNAP_ID", "snap-revision": "$START_REVISION"}
EOF
p=$(fakestore new-snap-declaration --dir "$BLOB_DIR" failing.snap --snap-decl-json decl-headers.json)
snap ack "$p"
p=$(fakestore new-snap-revision --dir "$BLOB_DIR" failing.snap --snap-rev-json rev-headers.json)
snap ack "$p"
# use journalctl wrapper to grep only the logs collected while the test is running
if "$TESTSTOOLS"/journal-state get-log | MATCH "Waiting for system reboot"; then
echo "Already waiting for system reboot, exiting..."
exit 1
fi
# install new target snap
snap install --no-wait failing.snap
# use journalctl wrapper to grep only the logs collected while the test is running
# waiting up to 100s to reach waiting for reboot
retry -n 20 --wait 2 "$TESTSTOOLS"/journal-state match-log "Waiting for system reboot"
# check boot env vars
readlink "/snap/$TARGET_SNAP_NAME/current" > failBoot
snap debug boot-vars > before-reboot.bootenv
if [ "$TARGET_SNAP" = kernel ]; then
MATCH "snap_kernel=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < before-reboot.bootenv
MATCH "snap_try_kernel=${TARGET_SNAP_NAME}_$(cat failBoot).snap\$" < before-reboot.bootenv
else
MATCH "snap_core=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < before-reboot.bootenv
MATCH "snap_try_core=${TARGET_SNAP_NAME}_$(cat failBoot).snap\$" < before-reboot.bootenv
fi
REBOOT
fi
# since the change fails snap watch will have non-0 exit status too, so we
# need to ignore the exit status
snap watch --last=install? || true
# after rollback, we have the original target snap for a while
# wait until the kernel and core snap revisions are in place
#shellcheck disable=SC2016
retry -n 60 --wait 1 --env TARGET_SNAP_NAME="$TARGET_SNAP_NAME" sh -c 'test $(snap list | awk "/^${TARGET_SNAP_NAME} / {print(\$3)}") = $(cat prevBoot)'
# ensure the last install change failed as expected
snap change --last=install | MATCH "cannot finish $TARGET_SNAP_NAME installation, there was a rollback across reboot"
snap change --last=install | MATCH "^Error.*Automatically connect"
# and the boot env vars are correctly set
echo "Waiting for snapd to clean snap_mode"
#shellcheck disable=SC2148
#shellcheck disable=SC2016
retry -n 200 --wait 1 sh -c 'snap debug boot-vars | MATCH "snap_mode=\$"'
snap debug boot-vars > after-reboot.bootenv
if [ "$TARGET_SNAP" = kernel ]; then
MATCH 'snap_try_kernel=$' < after-reboot.bootenv
MATCH "snap_kernel=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < after-reboot.bootenv
else
MATCH 'snap_try_core=$' < after-reboot.bootenv
MATCH "snap_core=${TARGET_SNAP_NAME}_$(cat prevBoot).snap\$" < after-reboot.bootenv
fi
|
