1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
|
# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
# Manage the creation of a configure-time virtual environment
#
# .. cmake_module::
#
# This module manages the creation of virtual python environment during
# configuration. The purpose of this virtual environment is to be able to run
# python code from cmake in situations such as python-based code generation,
# running postprocessing in python during testing etc.
# If enabled the Python bindings are also installed (editable) into the
# internal virtual environment.
#
# The internal virtual environment is only generated if the
# configuration is not run with an active virtual environment. If an
# external environment is detected this will be used instead of the
# internal environment.
#
# Although designed for internal use, the internal (or detected external)
# virtualenv can also be manually inspected. A symlink to the activation script is
# placed in the top level build directory of all Dune modules in the stack.
# To directly execute a command in the virtualenv, you can use the script
# :code:`run-in-dune-env <command>`, which is also placed into every build directory.
#
# All packages installed with :ref:`dune_python_configure_bindings` or
# :ref:`dune_python_configure_package` are automatically installed into the virtualenv.
#
# After execution of this module, the following are available for use in
# downstream modules:
#
# * :code:`DUNE_PYTHON_VIRTUALENV_PATH` The path of the virtual environment
# * :code:`DUNE_PYTHON_VIRTUALENV_EXECUTABLE` The python interpreter in the virtual environment
#
# By default, the created virtualenv resides in the first non-installed Dune module of
# the module stack (if no installation is performed: dune-common). Be aware
# that mixing installed and non-installed modules may result in a situation,
# where multiple such environments are created, although only one should.
# You can change this behavior by either specifying a fixed path for the virtualenv
# using :ref:`DUNE_PYTHON_VIRTUALENV_PATH` or by enabling
# :ref:`DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR` if you are using an
# absolute build directory with dunecontrol. Note that this flag is enabled by default
# starting from Dune 2.7.
#
# .. cmake_variable:: DUNE_PYTHON_VIRTUALENV_PATH
#
# When the Dune build system has setup a virtualenv, this variable will contain its location.
# You can also set this variable to a fixed path when calling CMake, and the virtualenv will
# be placed at that location.
#
# .. cmake_variable:: DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR
#
# Before Dune 2.7, the virtualenv was always placed inside the build directory of the first
# non-installed Dune module that the current module depends on. When using installed core modules
# or a multi-stage installation process, this can lead to situations where there are multiple
# virtualenvs, making it impossible to find all Python modules installed by upstream modules.
# In order to avoid this problem at least for builds using an absolute build directory (i.e., the
# :code:`--builddir` option of dunecontrol refers to an absolute path), the build system will
# place the virtualenv in a dedicated directory :code:`dune-python-env` inside that absolute
# build directory, where it will be found by all Dune modules. If you want to disable this
# behavior, set :code:`DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR=0`.
#
# .. cmake_variable:: DUNE_PYTHON_ALLOW_GET_PIP
#
# The Dune build system will try to build a virtualenv with pip installed into it,
# but this can fail in some situations, in particular on Debian and Ubuntu distributions.
# In this case, you will see a warning message in the CMake output. If you are on Debian
# or Ubuntu, try installing the :code:`python3-venv` (for Python 3) and / or
# :code:`python-virtualenv` packages, delete your build directory and try configuring
# again.
#
# If that still does not help, set this variable to allow the Dune build system to download
# :code:`get-pip.py` from https://bootstrap.pypa.io/get-pip.py at configure time and execute
# it to install pip into the freshly set up virtual environment. While this should normally
# not be necessary anymore, see https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
# for more information about the underlying distribution bug.
#
# .. cmake_variable:: DUNE_PYTHON_WHEELHOUSE
#
# The place where python wheels are stored. Notice that this wheelhouse directory shall be
# the same for all dune installations.
#
include_guard(GLOBAL)
# pre-populate DUNE_PYTHON_USE_VENV
set(DUNE_PYTHON_USE_VENV ON CACHE BOOL
"Using virtual environments when configuring Dune Python bindings.\
When ON any Python binding packages are installed into an auto-generated\
or any activated virtual environment in editable mode at configure time.\
When OFF users have to take care of installing Python binding packages.\
When OFF a script 'set-dune-pythonpath' is generated that may be used to enable the\
Python binding packages by means of the PYTHONPATH variable."
)
if(NOT DUNE_PYTHON_USE_VENV AND DUNE_ENABLE_PYTHONBINDINGS)
message(STATUS "Found DUNE_PYTHON_USE_VENV=FALSE: Python bindings are configured but Python packages and dependencies are not installed")
endif()
# pre-populate DUNE_PYTHON_SYSTEM_IS_VIRTUALENV
set(DUNE_PYTHON_SYSTEM_IS_VIRTUALENV "" CACHE PATH
"Running in an external activated virtual environment"
)
# pre-populate DUNE_PYTHON_VIRTUALENV_PATH
set(DUNE_PYTHON_VIRTUALENV_PATH "" CACHE PATH
"Location of Python virtualenv created by the Dune build system"
)
# pre-populate DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR
set(DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR ON CACHE BOOL
"Place Python virtualenv in top-level directory \"dune-python-env\" when using an absolute build directory"
)
option(DUNE_RUNNING_IN_CI "This is turned on if running in dune gitlab ci" OFF)
if(DUNE_RUNNING_IN_CI)
set(DUNE_PIP_INDEX "--index-url=https://gitlab.dune-project.org/api/v4/projects/133/packages/pypi/simple")
else()
set(DUNE_PIP_INDEX "")
endif()
# Construct the wheel house installation option string
# First set the path to a Dune wheelhouse that is to be used during installation
# NB: Right now, the same logic is used to retrieve the location of the
# wheelhouse (which means that you have to use the same CMAKE_INSTALL_PREFIX
# when *using* installed modules, you used when *installing* them.
# TODO: Replace this with a better mechanism (like writing the location into
# dune-commons package config file)
set(DUNE_PYTHON_WHEELHOUSE "unset" CACHE PATH "The place where the wheels will be stored")
if(DUNE_PYTHON_WHEELHOUSE STREQUAL "unset")
set(DUNE_PYTHON_WHEELHOUSE "${CMAKE_INSTALL_PREFIX}/share/dune/wheelhouse")
endif()
# if use of venv is enabled
if(DUNE_PYTHON_USE_VENV)
# Determine whether the given interpreter is running inside a virtualenv
dune_execute_process(COMMAND "${Python3_EXECUTABLE}" "${scriptdir}/venvpath.py"
RESULT_VARIABLE DUNE_PYTHON_SYSTEM_IS_VIRTUALENV
OUTPUT_VARIABLE DUNE_PYTHON_VIRTUALENV_PATH
OUTPUT_STRIP_TRAILING_WHITESPACE
)
# If the user has not specified an absolute, we look through the dependency tree of this module
# for a build directory that already contains a virtual environment.
# if we haven't found it yet, check in the current build directory - this might be a reconfigure
if(NOT DUNE_PYTHON_VIRTUALENV_PATH)
foreach(mod ${DUNE_FOUND_DEPENDENCIES} CMAKE_BINARY)
if(IS_DIRECTORY ${${mod}_DIR}/dune-env)
set(DUNE_PYTHON_VIRTUALENV_PATH ${${mod}_DIR}/dune-env)
break()
endif()
endforeach()
endif()
# We didn't find anything, so figure out the correct location for building the virtualenv
if(NOT DUNE_PYTHON_VIRTUALENV_PATH)
if(DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR AND DUNE_BUILD_DIRECTORY_ROOT_PATH)
# Use a dedicated directory not associated with any module
set(DUNE_PYTHON_VIRTUALENV_PATH "${DUNE_BUILD_DIRECTORY_ROOT_PATH}/dune-python-env")
else()
# Create the virtualenv inside our build directory
set(DUNE_PYTHON_VIRTUALENV_PATH ${CMAKE_BINARY_DIR}/dune-env)
endif()
endif()
# If it does not yet exist, set it up!
if(NOT IS_DIRECTORY "${DUNE_PYTHON_VIRTUALENV_PATH}")
# Check for presence of the virtualenv/venv package
dune_python_find_package(PACKAGE virtualenv)
dune_python_find_package(PACKAGE venv)
if(NOT(DUNE_PYTHON_virtualenv_FOUND OR DUNE_PYTHON_venv_FOUND))
message(WARNING "None of the Python packages virtualenv/venv was found. "
"Python bindings are therefore disabled. "
"If you are using Debian or Ubuntu, consider installing python3-venv "
"and/or python-virtualenv. "
"If you know what you are doing, you may also choose to use "
"Python bindings without a virtual environment by configuring "
"Dune with DUNE_PYTHON_USE_VENV=FALSE.")
# TODO: For now we stay with the solution to disable Python
# bindings entirely if venv was not actively disabled
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
# Set some options depending on which virtualenv package is used
if(DUNE_PYTHON_venv_FOUND)
set(VIRTUALENV_PACKAGE_NAME venv)
set(NOPIP_OPTION --without-pip)
set(INTERPRETER_OPTION "")
endif()
if(DUNE_PYTHON_virtualenv_FOUND)
set(VIRTUALENV_PACKAGE_NAME virtualenv)
set(NOPIP_OPTION --no-pip)
set(INTERPRETER_OPTION -p "${Python3_EXECUTABLE}")
endif()
if(DUNE_PYTHON_venv_FOUND AND DUNE_PYTHON_SYSTEM_IS_VIRTUALENV)
message(WARNING "You are using a system python interpreter which is a virtualenv and the venv "
"package. You might want to consider installing the virtualenv package if you "
"experience inconveniences.")
endif()
# Set up the env itself
message(STATUS "Building a virtualenv in ${DUNE_PYTHON_VIRTUALENV_PATH}")
# First, try to build it with pip installed, but only if the user has not set DUNE_PYTHON_ALLOW_GET_PIP
if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
dune_execute_process(COMMAND ${Python3_EXECUTABLE}
-m ${VIRTUALENV_PACKAGE_NAME}
${INTERPRETER_OPTION}
"${DUNE_PYTHON_VIRTUALENV_PATH}"
RESULT_VARIABLE venv_install_result
)
endif()
if(NOT "${venv_install_result}" STREQUAL "0")
if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
# we attempted the default installation before, so issue a warning
message(WARNING "Failed to build a virtual env with pip installed, trying again without "
"pip. If you are using Debian or Ubuntu, consider installing python3-venv "
"and/or python-virtualenv")
endif()
# remove the remainder of a potential first attempt
file(REMOVE_RECURSE "${DUNE_PYTHON_VIRTUALENV_PATH}")
# try to build the env without pip
dune_execute_process(COMMAND ${Python3_EXECUTABLE}
-m ${VIRTUALENV_PACKAGE_NAME}
${INTERPRETER_OPTION}
${NOPIP_OPTION}
"${DUNE_PYTHON_VIRTUALENV_PATH}"
RESULT_VARIABLE venv_install_result2)
if(NOT "${venv_install_result2}" STREQUAL "0")
message(WARNING "Failed to build a virtual env without pip.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
endif()
else()
message(STATUS "Using existing virtualenv in ${DUNE_PYTHON_VIRTUALENV_PATH}")
endif()
# Also store the virtual env interpreter directly
set(DUNE_PYTHON_VIRTUALENV_EXECUTABLE ${DUNE_PYTHON_VIRTUALENV_PATH}/bin/python
CACHE INTERNAL "Python virtual environment executable used by Dune")
# Write a symlink for activation of the environment into all the
# build directories of the Dune stack
dune_execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${DUNE_PYTHON_VIRTUALENV_PATH}/bin/activate ${CMAKE_BINARY_DIR}/activate)
endif()
# Also write a small wrapper script 'run-in-dune-env' into the build directory
# This is necessary to execute installed python scripts (the bin path of a virtualenv
# is *not* in the sys path, so a simple `python scriptname` does not work.
if(UNIX)
find_package(UnixCommands QUIET)
dune_module_path(MODULE dune-common
RESULT scriptdir
SCRIPT_DIR)
configure_file(${scriptdir}/run-in-dune-env.sh.in
${CMAKE_BINARY_DIR}/run-in-dune-env
@ONLY)
else()
message(WARNING "Writing script 'run-in-dune-env' not implemented on your platform!")
endif()
if(DUNE_PYTHON_USE_VENV)
# The virtualenv might not contain pip due to the distribution bug described in
# https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
# We need to install pip, so if pip is missing, we offer to download and run the get-pip
# script. We ask users for permission to do so, or we allow them to set it up themselves.
dune_python_find_package(PACKAGE pip
RESULT pippresent
INTERPRETER ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE}
)
if(NOT pippresent)
if(DUNE_PYTHON_ALLOW_GET_PIP)
# Fetch the get-pip.py script
message(STATUS "Installing pip using https://bootstrap.pypa.io/get-pip.py...")
file(DOWNLOAD https://bootstrap.pypa.io/get-pip.py ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py)
# Verify that the script was successfully fetched
file(READ ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py verify LIMIT 1)
if(NOT verify)
message(WARNING "Fetching get-pip.py failed. This often happens when CMake is built from "
"source without SSL/TLS support. Consider using a different cmake version or "
"fall back to manually installing pip into the virtualenv.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
# Execute the script
dune_execute_process(COMMAND ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py RESULT_VARIABLE pip_install_result)
if(NOT "${pip_install_result}" STREQUAL "0")
message(WARNING "Fatal error when installing pip into the virtualenv.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
else()
message(WARNING "dune-common set up a virtualenv, but needs pip to be installed into it. "
"You can either install it yourself manually activating the virtualenv with "
"the activate script in your build directory ${CMAKE_BINARY_DIR} or you set "
"the CMake variable DUNE_PYTHON_ALLOW_GET_PIP to allow Dune to use get-pip.py "
"from https://bootstrap.pypa.io/get-pip.py")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
endif()
# if pip was not found before then we can set it here since it was now found
set(DUNE_PYTHON_pip_FOUND ON CACHE INTERNAL "Whether dune has found python pip")
# install setuptools into the venv (needed to find dependencies later on)
dune_execute_process(COMMAND ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE} -m pip install
"${DUNE_PIP_INDEX}"
setuptools>=41 ninja
WARNING_MESSAGE "python 'setuptools' package could not be installed - possibly connection to the python package index failed"
)
else()
# Write a small wrapper script 'run-in-dune-env' into the build directory
# This is necessary to execute installed python scripts (the bin path of a virtualenv
# is *not* in the sys path, so a simple `python scriptname` does not work.
if(UNIX)
# create list with python module paths
foreach( mod ${DUNE_FOUND_DEPENDENCIES} )
if( NOT "_${${mod}_PYTHON_WHEELHOUSE}_" STREQUAL "__")
set(MODULE_DEP_PYTHON_PATH "${MODULE_DEP_PYTHON_PATH}\n ${${mod}_PYTHON_WHEELHOUSE}")
endif()
endforeach()
# for dune-common this variable is not set but needed by the script
if( NOT dune-common_DIR )
set( dune-common_DIR ${CMAKE_BINARY_DIR} )
endif()
find_package(UnixCommands QUIET)
dune_module_path(MODULE dune-common
RESULT scriptdir
SCRIPT_DIR)
configure_file(${scriptdir}/set-dune-pythonpath.sh.in
${CMAKE_BINARY_DIR}/set-dune-pythonpath
@ONLY)
else()
message(WARNING "Writing script 'set-dune-pythonpath' not implemented on your platform!")
endif()
endif()
|
