1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
FROM debian:bookworm-slim
ENV PYTHONUNBUFFERED=1
ENV PYTHONIOENCODING=UTF-8
ARG DEBIAN_FRONTEND=noninteractive
# Pypy3 is installed from a package manager because it takes so long to build.
RUN apt-get update && apt-get install -y build-essential \
libcurl4-openssl-dev \
apt-utils \
debconf \
libffi-dev \
tk-dev \
xz-utils \
ca-certificates \
curl \
lsb-release \
git \
libmemcached-dev \
make \
liblzma-dev \
libreadline-dev \
libbz2-dev \
llvm \
libncurses5-dev \
libsqlite3-dev \
wget \
pypy3 \
pypy3-lib \
python3-openssl \
libncursesw5-dev \
zlib1g-dev \
pkg-config \
libssl-dev \
sudo
# Setup variables. Even though changing these may cause unnecessary invalidation of
# unrelated elements, grouping them together makes the Dockerfile read better.
ENV PROVISIONING=/provisioning
ENV PIP_NO_CACHE_DIR=off
ENV PYTHONDONTWRITEBYTECODE=1
ENV PIP_PREFER_BINARY=1
ARG CELERY_USER=developer
# Check for mandatory build arguments
RUN : "${CELERY_USER:?CELERY_USER build argument needs to be set and non-empty.}"
ENV HOME=/home/$CELERY_USER
ENV PATH="$HOME/.pyenv/bin:$PATH"
# Copy and run setup scripts
WORKDIR $PROVISIONING
#COPY docker/scripts/install-couchbase.sh .
# Scripts will lose their executable flags on copy. To avoid the extra instructions
# we call the shell directly.
#RUN sh install-couchbase.sh
RUN useradd -m -s /bin/bash $CELERY_USER
# Swap to the celery user so packages and celery are not installed as root.
USER $CELERY_USER
# Install pyenv
RUN curl https://pyenv.run | bash
# Install required Python versions
RUN pyenv install 3.13 && \
pyenv install 3.12 && \
pyenv install 3.11 && \
pyenv install 3.10 && \
pyenv install 3.9 && \
pyenv install pypy3.11
# Set global Python versions
RUN pyenv global 3.13 3.12 3.11 3.10 3.9 pypy3.11
# Install celery
WORKDIR $HOME
COPY --chown=1000:1000 requirements $HOME/requirements
COPY --chown=1000:1000 docker/entrypoint /entrypoint
RUN chmod gu+x /entrypoint
# Define the local pyenvs
RUN pyenv local 3.13 3.12 3.11 3.10 3.9 pypy3.11
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \
pyenv exec python3.12 -m pip install --upgrade pip setuptools wheel && \
pyenv exec python3.11 -m pip install --upgrade pip setuptools wheel && \
pyenv exec python3.10 -m pip install --upgrade pip setuptools wheel && \
pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel && \
pyenv exec pypy3.11 -m pip install --upgrade pip setuptools wheel
# Install requirements first to leverage Docker layer caching
# Split into separate RUN commands to reduce memory pressure and improve layer caching
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.13 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.12 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.11 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.10 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.9 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec pypy3.11 -m pip install -r requirements/default.txt \
-r requirements/dev.txt \
-r requirements/docs.txt \
-r requirements/pkgutils.txt \
-r requirements/test-ci-base.txt \
-r requirements/test-ci-default.txt \
-r requirements/test-integration.txt \
-r requirements/test.txt
COPY --chown=1000:1000 . $HOME/celery
# Install celery in editable mode (dependencies already installed above)
RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \
pyenv exec python3.13 -m pip install --no-deps -e $HOME/celery && \
pyenv exec python3.12 -m pip install --no-deps -e $HOME/celery && \
pyenv exec python3.11 -m pip install --no-deps -e $HOME/celery && \
pyenv exec python3.10 -m pip install --no-deps -e $HOME/celery && \
pyenv exec python3.9 -m pip install --no-deps -e $HOME/celery && \
pyenv exec pypy3.11 -m pip install --no-deps -e $HOME/celery
WORKDIR $HOME/celery
RUN git config --global --add safe.directory /home/developer/celery
# Setup the entrypoint, this ensures pyenv is initialized when a container is started
# and that any compiled files from earlier steps or from mounts are removed to avoid
# pytest failing with an ImportMismatchError
ENTRYPOINT ["/entrypoint"]
|
