1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
#!/bin/sh
# If needed, build Dockerfiles specified by $2..n and upload it to the project
# registry. (This is used by CI jobs, not a Charliecloud test suite image.)
#
# Because of this bootstrap context, we need to run *this* job using an image
# available somewhere else. It also needs to have some image builder
# installed. GitLab recommends images provided by Docker that are very bare
# bones (Alpine with no programming language other than Busybox) [1], which is
# why this is a POSIX shell script.
#
# An alternative would be Podman images, which are Fedora based and do have a
# recent Python 3 [2].
#
# The gotcha here is that the local image builder does not know if the correct
# image is already in the registry, and it doesn’t even know the layer digests
# without building the layers, which we want to avoid. Therefore, we tag the
# images with the MD5 of the Dockerfile plus the base image digest, and build
# a new image if either is not in the registry.
#
# PREREQUISITES:
#
# 1. Log in to the gitlab.com container registry. For testing, do a manual
# “docker login” with your username and a personal access token
# (PAT) [3]. While your username/password will work, don’t do that
# because Docker stores it in a plaintext file.
#
# 2. Set $CI_REGISTRY_IMAGE [4] if the default below isn’t appropriate.
#
# [1]: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker
# [2]: https://quay.io/repository/podman/stable
# [3]: https://docs.gitlab.com/ee/user/packages/container_registry/authenticate_with_container_registry.html
# [4]: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
set -e # needs to be POSIX so can’t use our fancy traps
branch=$1
shift
echo "branch: $branch"
# Convenience function for computing MD5 of a file.
md5 () {
md5sum "$1" | cut -d' ' -f1
}
# Print the digest of the named image.
#shellcheck disable=SC2016
refhash () {
case $1 in
scratch) # scratch is special
printf '0%.0s' $(seq 1 64)
echo
;;
*':$branch') # local file
distro=$(echo "$1" | sed -E 's/^(\$\{.+\})?ci_(.+):\$branch$/\2/g')
(cd "$(dirname "$0")" && md5 "$distro.df")
;;
*) # ask the registry
# sed(1) drama is because we don’t have # any JSON parsing available.
docker manifest inspect "$df_base" \
| tr -d '"[:space:]' \
| sed -E "s/^(.+)(digest:sha256:([0-9a-f]+),platform:\{architecture:${arch},)(.+)$/\3/"
;;
esac
}
# Set up docker(1) alias if needed.
printf 'docker alias: '
if docker info > /dev/null 2>&1; then
echo 'not needed'
else
alias docker='sudo docker'
alias docker
fi
# Function to pass through proxy variables if needed.
build () {
if [ -z "$HTTP_PROXY" ]; then
docker build "$@"
else
docker build --build-arg HTTP_PROXY="${HTTP_PROXY:?}" \
--build-arg HTTPS_PROXY="${HTTPS_PROXY:?}" \
--build-arg NO_PROXY="${NO_PROXY:?}" \
--build-arg http_proxy="${http_proxy:?}" \
--build-arg https_proxy="${https_proxy:?}" \
--build-arg no_proxy="${no_proxy:?}" \
"$@"
fi
}
# Registry path.
if [ -n "$CI_REGISTRY_IMAGE" ]; then
regy=$CI_REGISTRY_IMAGE
else
regy=registry.gitlab.com/charliecloud/charliecloud
fi
echo "registry: $regy"
arch=$(arch.sh)
echo "architecture: $arch"
#echo "nv distro: $nv_distro"
for df in "$@"; do
echo
echo "file: $df"
df_hash=$(md5 "$df")
echo "file digest: $df_hash"
df_base=$(sed -En 's/^FROM (.+)/\1/p' "$df")
echo "base: $df_base"
printf "base digest: "
base_hash=$(refhash "$df_base")
echo "$base_hash"
digest=$(printf '%s%s' "$df_hash" "$base_hash" | md5 -)
echo "full digest: $digest"
name=$(basename "$df" | sed -E 's/^(.+)\.df$/ci_\1/g')
echo "image name: $name"
# Check if image exists in our registry; if not, build and push it.
ref_digest=$regy/$name:$digest
ref_branch=$regy/$name:$branch
ref_latest=$regy/$name:latest
printf 'in registry: '
if out=$(docker manifest inspect "$ref_digest" 2>&1); then
echo 'yes, skipping build'
else
if ! ( echo "$out" | grep -Eq '^no such manifest:' ); then
echo "bad response:" "$out" 1>&2
exit 1
fi
echo 'no, building and pushing ...'
context=$(dirname "$df")
echo "context: $context"
build --pull -t "$ref_digest" -f "$df" \
--build-arg=regy="$regy/" \
--build-arg=branch="$branch" \
"$context"
docker tag "$ref_digest" "$name:$branch" # for local use
docker push "$ref_digest"
fi
# Re-tag in registry. Do this unconditionally because it’s fast and then we
# don’t have to worry about whether the previous iteration of the script
# worked correctly. Use Docker [1] rather than Skopeo because it’s already
# installed.
#
# This does not completely eliminate the race condition in #1981, but now only
# commits within the same branch will be racing, which seems acceptable.
#
# [1]: https://stackoverflow.com/questions/26763427#70526615
echo 'tagging ...'
docker buildx imagetools create "$ref_digest" --tag "$ref_branch"
docker buildx imagetools create "$ref_digest" --tag "$ref_latest"
done
|
