1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
#!/bin/bash
# Intended to be called from the root.d cloud-image script as follows:
# $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
BASE_IMAGE_FILE=$1
BASE_IMAGE_TAR=$2
IMAGE_LOCATION=$3
CACHED_IMAGE=$4
CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""}
TAR_LOCK=$CACHED_TAR.lock
# GPT GUIDs of interest.
# See https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
# also https://systemd.io/BOOT_LOADER_SPECIFICATION/
GUID_EFI="c12a7328-f81f-11d2-ba4b-00a0c93ec93b"
GUID_LINUX_BOOT="bc13c2ff-59e6-4262-a352-b275fd6f7172"
function extract_image() {
if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then
echo "Not checking freshness of cached $CACHED_TAR."
else
if [ -z "$DIB_LOCAL_IMAGE" ]; then
echo "Fetching Base Image"
# There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
# If the image fails to download due to a 404 we retry once.
set +e
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
RV=$?
set -e
if [ "$RV" == "44" ] ; then
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
elif [ "$RV" != "0" ] ; then
exit 1
fi
fi
if [ ! -f $CACHED_TAR -o \
$CACHED_IMAGE -nt $CACHED_TAR ] ; then
echo "Repacking base image as tarball."
WORKING=$(mktemp --tmpdir=${TMP_DIR:-/tmp} -d)
EACTION="rm -r $WORKING"
trap "$EACTION" EXIT
echo "Working in $WORKING"
RAW_FILE=$(mktemp --tmpdir=$WORKING XXXXXX.raw)
if [ "${CACHED_IMAGE: -3}" == ".xz" ] ; then
QCOW2_FILE=$(mktemp --tmpdir=$WORKING XXXXXX.qcow2)
# This leaves the old image in place so cache-url wont get it again
unxz --stdout $CACHED_IMAGE > $QCOW2_FILE
CACHED_IMAGE=$QCOW2_FILE
fi
qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $RAW_FILE
# kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
# loop device and creates it if it doesn't exist
LOOPDEV_BASE=$(basename $(sudo losetup -f))
# add partition mappings
sudo kpartx -av $RAW_FILE
# If running inside Docker, make our nodes manually, because udev will not be working.
if [ -f /.dockerenv ]; then
sudo dmsetup --noudevsync mknodes
fi
if ! timeout 5 sh -c "while ! ls /dev/mapper/${LOOPDEV_BASE}p* ; do sleep 1; done"; then
echo "Error: Could not find any ${LOOPDEV_BASE} devices"
exit 1
fi
EACTION="sudo kpartx -d $RAW_FILE ; $EACTION"
trap "$EACTION" EXIT
ROOT_LOOPDEV=""
BOOT_LOOPDEV=""
EFI_LOOPDEV=""
LOOPDEVS=$(ls /dev/mapper/${LOOPDEV_BASE}p* | sort -r)
LOOPDEV_COUNT=$(echo $LOOPDEVS | wc -w)
if [ $LOOPDEV_COUNT == "1" ]; then
# if there is one partition device, assume it is the root device
ROOT_LOOPDEV=${LOOPDEVS}
LOOPDEVS=""
fi
for LOOPDEV in ${LOOPDEVS}; do
fstype=$(sudo blkid -o value -s TYPE -p "${LOOPDEV}" 2>/dev/null)
label=$(sudo blkid -o value -s LABEL -p "${LOOPDEV}" 2>/dev/null)
part_type=$(sudo blkid -o value -s PART_ENTRY_TYPE -p "${LOOPDEV}" 2>/dev/null)
if [ -z "${fstype}" ]; then
# Ignore block device with no filesystem type
continue
fi
# look for EFI partition to mount at /boot/efi either by GUID or
# label convention
if [ -z "$EFI_LOOPDEV" ]; then
if [[ ${part_type} == ${GUID_EFI} ]]; then
EFI_LOOPDEV=$LOOPDEV
continue
fi
fi
# look for EFI partition to mount at /boot/efi either by GUID or
# label convention.
if [ -z "$BOOT_LOOPDEV" ]; then
if [[ ${part_type} == ${GUID_LINUX_BOOT} || ${label} == "boot" ]]; then
BOOT_LOOPDEV=$LOOPDEV
continue
fi
fi
if [ -z "$ROOT_LOOPDEV" ]; then
ROOT_LOOPDEV=$LOOPDEV
continue
fi
done
mkdir $WORKING/mnt
ROOT_FSTYPE=$(sudo blkid -o value -s TYPE $ROOT_LOOPDEV)
if [ "xfs" = "$ROOT_FSTYPE" ]; then
# mount xfs with nouuid, just in case that uuid is already mounted
# use ro to avoid/workaround xfs uuid issues on older
# kernels with newer rhel images which seem to set
# flags to generate unique uuid's:
# xfs superblock has incompatible features (0x4)
# we don't need to worry about this, we just want the data
MOUNTOPTS="-o nouuid,ro"
elif [ "btrfs" = "$ROOT_FSTYPE" ]; then
# Fedora has a btrfs filesystem with a subvolume called root.
# For now assume there will be a 'root' subvolume, but in the
# future the subvolume layout may need to be discovered for different
# images
MOUNTOPTS="-o subvol=root"
else
MOUNTOPTS=""
fi
sudo mount $MOUNTOPTS $ROOT_LOOPDEV $WORKING/mnt
EACTION="sudo umount -f $WORKING/mnt ; $EACTION"
trap "$EACTION" EXIT
if [ ! -z "$BOOT_LOOPDEV" ]; then
# mount to /boot
BOOT_FSTYPE=$(sudo blkid -o value -s TYPE $ROOT_LOOPDEV)
if [ "xfs" = "$BOOT_FSTYPE" ]; then
BOOT_MOUNTOPTS="-o nouuid,ro"
# Similar to root filesystem, if the boot filesystem
# is XFS and the base OS is the same as the image being
# rebuilt, we need to pass "nouuid" to bypass UUID safety
# checks and successfully mounts so we can extract the
# contents.
else
BOOT_MOUNTOPTS=""
fi
sudo mount $BOOT_MOUNTOPTS $BOOT_LOOPDEV $WORKING/mnt/boot
EACTION="sudo umount -f $BOOT_LOOPDEV ; $EACTION"
trap "$EACTION" EXIT
fi
if [ ! -z "$EFI_LOOPDEV" ]; then
# mount to /boot/efi
sudo mount $EFI_LOOPDEV $WORKING/mnt/boot/efi
EACTION="sudo umount -f $EFI_LOOPDEV ; $EACTION"
trap "$EACTION" EXIT
fi
# find out if chroot tar has full xattr support
if [ 0 == `sudo chroot $WORKING/mnt bin/tar --help | grep -c xattrs-exclude` ]; then
TAROPTS="--no-xattrs"
else
TAROPTS="--xattrs --xattrs-include=* --xattrs-exclude=security.selinux"
fi
# Chroot in so that we get the correct uid/gid
sudo chroot $WORKING/mnt bin/tar $TAROPTS -cz . > $WORKING/tmp.tar
mv $WORKING/tmp.tar $CACHED_TAR
else
echo "Using cached tar from $CACHED_TAR"
fi
fi
# Extract the base image (use --numeric-owner to avoid UID/GID mismatch between
# image tarball and host OS e.g. when building Fedora image on an openSUSE host)
# Include all xattrs except selinux because the selinux ones cause issues in our
# chroot environment, and we restore all of those at the end of the build anyway.
echo "Extracting base root image from $CACHED_TAR"
sudo tar -C $TARGET_ROOT --numeric-owner --xattrs --xattrs-include='*' --xattrs-exclude='security.selinux' -xzf $CACHED_TAR
}
(
echo "Getting $TAR_LOCK: $(date)"
# Wait up to 20 minutes for another process to download
if ! flock -w 1200 9 ; then
echo "Did not get $TAR_LOCK: $(date)"
exit 1
fi
extract_image
) 9> $TAR_LOCK
|
