1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
|
# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
# SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
# Manage the creation of a configure-time virtual environment
#
# .. cmake_module::
#
# This module manages the creation of virtual python environment during
# configuration. The purpose of this virtual environment is to be able to run
# python code from cmake in situations such as python-based code generation,
# running postprocessing in python during testing etc.
# If enabled the Python bindings are also installed (editable) into the
# internal virtual environment.
#
# The internal virtual environment is only generated if the
# configuration is not run with an active virtual environment. If an
# external environment is detected this will be used instead of the
# internal environment.
#
# Although designed for internal use, the internal (or detected external)
# virtualenv can also be manually inspected. A symlink to the activation script is
# placed in the top level build directory of all Dune modules in the stack.
# To directly execute a command in the virtualenv, you can use the script
# :code:`run-in-dune-env <command>`, which is also placed into every build directory.
#
# All packages installed with :ref:`dune_python_configure_bindings` or
# :ref:`dune_python_configure_package` are automatically installed into the virtualenv.
#
# After execution of this module, the following are available for use in
# downstream modules:
#
# * :code:`DUNE_PYTHON_VIRTUALENV_PATH` The path of the virtual environment
# * :code:`DUNE_PYTHON_VIRTUALENV_EXECUTABLE` The python interpreter in the virtual environment
#
# By default, the created virtualenv resides in the first non-installed Dune module of
# the module stack (if no installation is performed: dune-common). Be aware
# that mixing installed and non-installed modules may result in a situation,
# where multiple such environments are created, although only one should.
# You can change this behavior by either specifying a fixed path for the virtualenv
# using :ref:`DUNE_PYTHON_VIRTUALENV_PATH` or by enabling
# :ref:`DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR` if you are using an
# absolute build directory with dunecontrol. Note that this flag is enabled by default
# starting from Dune 2.7.
#
# .. cmake_variable:: DUNE_PYTHON_VIRTUALENV_PATH
#
# When the Dune build system has setup a virtualenv, this variable will contain its location.
# You can also set this variable to a fixed path when calling CMake, and the virtualenv will
# be placed at that location.
#
# .. cmake_variable:: DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR
#
# Before Dune 2.7, the virtualenv was always placed inside the build directory of the first
# non-installed Dune module that the current module depends on. When using installed core modules
# or a multi-stage installation process, this can lead to situations where there are multiple
# virtualenvs, making it impossible to find all Python modules installed by upstream modules.
# In order to avoid this problem at least for builds using an absolute build directory (i.e., the
# :code:`--builddir` option of dunecontrol refers to an absolute path), the build system will
# place the virtualenv in a dedicated directory :code:`dune-python-env` inside that absolute
# build directory, where it will be found by all Dune modules. If you want to disable this
# behavior, set :code:`DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR=0`.
#
# .. cmake_variable:: DUNE_PYTHON_ALLOW_GET_PIP
#
# The Dune build system will try to build a virtualenv with pip installed into it,
# but this can fail in some situations, in particular on Debian and Ubuntu distributions.
# In this case, you will see a warning message in the CMake output. If you are on Debian
# or Ubuntu, try installing the :code:`python3-venv` (for Python 3) and / or
# :code:`python-virtualenv` packages, delete your build directory and try configuring
# again.
#
# If that still does not help, set this variable to allow the Dune build system to download
# :code:`get-pip.py` from https://bootstrap.pypa.io/get-pip.py at configure time and execute
# it to install pip into the freshly set up virtual environment. While this should normally
# not be necessary anymore, see https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
# for more information about the underlying distribution bug.
#
# .. cmake_variable:: DUNE_PYTHON_WHEELHOUSE
#
# The place where python wheels are stored. Notice that this wheelhouse directory shall be
# the same for all dune installations.
#
include_guard(GLOBAL)
# pre-populate DUNE_PYTHON_USE_VENV
set(DUNE_PYTHON_USE_VENV ON CACHE BOOL
"Using virtual environments when configuring Dune Python bindings.\
When ON any Python binding packages are installed into an auto-generated\
or any activated virtual environment in editable mode at configure time.\
When OFF users have to take care of installing Python binding packages.\
When OFF a script 'set-dune-pythonpath' is generated that may be used to enable the\
Python binding packages by means of the PYTHONPATH variable."
)
if(NOT DUNE_PYTHON_USE_VENV AND DUNE_ENABLE_PYTHONBINDINGS)
message(STATUS "Found DUNE_PYTHON_USE_VENV=FALSE: Python bindings are configured but Python packages and dependencies are not installed")
endif()
# pre-populate DUNE_PYTHON_SYSTEM_IS_VIRTUALENV
set(DUNE_PYTHON_SYSTEM_IS_VIRTUALENV "" CACHE PATH
"Running in an external activated virtual environment"
)
# pre-populate DUNE_PYTHON_VIRTUALENV_PATH
set(DUNE_PYTHON_VIRTUALENV_PATH "" CACHE PATH
"Location of Python virtualenv created by the Dune build system"
)
# pre-populate DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR
set(DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR ON CACHE BOOL
"Place Python virtualenv in top-level directory \"dune-python-env\" when using an absolute build directory"
)
option(DUNE_RUNNING_IN_CI "This is turned on if running in dune gitlab ci" OFF)
if(DUNE_RUNNING_IN_CI)
set(DUNE_PIP_INDEX "--index-url=https://gitlab.dune-project.org/api/v4/projects/133/packages/pypi/simple")
else()
set(DUNE_PIP_INDEX "")
endif()
# Construct the wheel house installation option string
# First set the path to a Dune wheelhouse that is to be used during installation
# NB: Right now, the same logic is used to retrieve the location of the
# wheelhouse (which means that you have to use the same CMAKE_INSTALL_PREFIX
# when *using* installed modules, you used when *installing* them.
# TODO: Replace this with a better mechanism (like writing the location into
# dune-commons package config file)
set(DUNE_PYTHON_WHEELHOUSE "unset" CACHE PATH "The place where the wheels will be stored")
if(DUNE_PYTHON_WHEELHOUSE STREQUAL "unset")
set(DUNE_PYTHON_WHEELHOUSE "${CMAKE_INSTALL_PREFIX}/share/dune/wheelhouse")
endif()
# if use of venv is enabled
if(DUNE_PYTHON_USE_VENV)
# Determine whether the given interpreter is running inside a virtualenv
dune_execute_process(COMMAND "${Python3_EXECUTABLE}" "${scriptdir}/venvpath.py"
RESULT_VARIABLE DUNE_PYTHON_SYSTEM_IS_VIRTUALENV
OUTPUT_VARIABLE DUNE_PYTHON_VIRTUALENV_PATH
OUTPUT_STRIP_TRAILING_WHITESPACE
)
# If the user has not specified an absolute, we look through the dependency tree of this module
# for a build directory that already contains a virtual environment.
# if we haven't found it yet, check in the current build directory - this might be a reconfigure
if(NOT DUNE_PYTHON_VIRTUALENV_PATH)
foreach(mod ${DUNE_FOUND_DEPENDENCIES} CMAKE_BINARY)
if(IS_DIRECTORY ${${mod}_DIR}/dune-env)
set(DUNE_PYTHON_VIRTUALENV_PATH ${${mod}_DIR}/dune-env)
break()
endif()
endforeach()
endif()
# We didn't find anything, so figure out the correct location for building the virtualenv
if(NOT DUNE_PYTHON_VIRTUALENV_PATH)
if(DUNE_PYTHON_EXTERNAL_VIRTUALENV_FOR_ABSOLUTE_BUILDDIR AND DUNE_BUILD_DIRECTORY_ROOT_PATH)
# Use a dedicated directory not associated with any module
set(DUNE_PYTHON_VIRTUALENV_PATH "${DUNE_BUILD_DIRECTORY_ROOT_PATH}/dune-python-env")
else()
# Create the virtualenv inside our build directory
set(DUNE_PYTHON_VIRTUALENV_PATH ${CMAKE_BINARY_DIR}/dune-env)
endif()
endif()
# If it does not yet exist, set it up!
if(NOT IS_DIRECTORY "${DUNE_PYTHON_VIRTUALENV_PATH}")
# Check for presence of the virtualenv/venv package
dune_python_find_package(PACKAGE virtualenv)
dune_python_find_package(PACKAGE venv)
if(NOT(DUNE_PYTHON_virtualenv_FOUND OR DUNE_PYTHON_venv_FOUND))
message(WARNING "None of the Python packages virtualenv/venv was found. "
"Python bindings are therefore disabled. "
"If you are using Debian or Ubuntu, consider installing python3-venv "
"and/or python-virtualenv. "
"If you know what you are doing, you may also choose to use "
"Python bindings without a virtual environment by configuring "
"Dune with DUNE_PYTHON_USE_VENV=FALSE.")
# TODO: For now we stay with the solution to disable Python
# bindings entirely if venv was not actively disabled
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
# Set some options depending on which virtualenv package is used
if(DUNE_PYTHON_venv_FOUND)
set(VIRTUALENV_PACKAGE_NAME venv)
set(NOPIP_OPTION --without-pip)
set(INTERPRETER_OPTION "")
endif()
if(DUNE_PYTHON_virtualenv_FOUND)
set(VIRTUALENV_PACKAGE_NAME virtualenv)
set(NOPIP_OPTION --no-pip)
set(INTERPRETER_OPTION -p "${Python3_EXECUTABLE}")
endif()
if(DUNE_PYTHON_venv_FOUND AND DUNE_PYTHON_SYSTEM_IS_VIRTUALENV)
message(WARNING "You are using a system python interpreter which is a virtualenv and the venv "
"package. You might want to consider installing the virtualenv package if you "
"experience inconveniences.")
endif()
# Set up the env itself
message(STATUS "Building a virtualenv in ${DUNE_PYTHON_VIRTUALENV_PATH}")
# First, try to build it with pip installed, but only if the user has not set DUNE_PYTHON_ALLOW_GET_PIP
if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
dune_execute_process(COMMAND ${Python3_EXECUTABLE}
-m ${VIRTUALENV_PACKAGE_NAME}
${INTERPRETER_OPTION}
"${DUNE_PYTHON_VIRTUALENV_PATH}"
RESULT_VARIABLE venv_install_result
)
endif()
if(NOT "${venv_install_result}" STREQUAL "0")
if(NOT DUNE_PYTHON_ALLOW_GET_PIP)
# we attempted the default installation before, so issue a warning
message(WARNING "Failed to build a virtual env with pip installed, trying again without "
"pip. If you are using Debian or Ubuntu, consider installing python3-venv "
"and/or python-virtualenv")
endif()
# remove the remainder of a potential first attempt
file(REMOVE_RECURSE "${DUNE_PYTHON_VIRTUALENV_PATH}")
# try to build the env without pip
dune_execute_process(COMMAND ${Python3_EXECUTABLE}
-m ${VIRTUALENV_PACKAGE_NAME}
${INTERPRETER_OPTION}
${NOPIP_OPTION}
"${DUNE_PYTHON_VIRTUALENV_PATH}"
RESULT_VARIABLE venv_install_result2)
if(NOT "${venv_install_result2}" STREQUAL "0")
message(WARNING "Failed to build a virtual env without pip.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
endif()
else()
message(STATUS "Using existing virtualenv in ${DUNE_PYTHON_VIRTUALENV_PATH}")
endif()
# Also store the virtual env interpreter directly
set(DUNE_PYTHON_VIRTUALENV_EXECUTABLE ${DUNE_PYTHON_VIRTUALENV_PATH}/bin/python
CACHE INTERNAL "Python virtual environment executable used by Dune")
# Write a symlink for activation of the environment into all the
# build directories of the Dune stack
dune_execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${DUNE_PYTHON_VIRTUALENV_PATH}/bin/activate ${CMAKE_BINARY_DIR}/activate)
endif()
# Also write a small wrapper script 'run-in-dune-env' into the build directory
# This is necessary to execute installed python scripts (the bin path of a virtualenv
# is *not* in the sys path, so a simple `python scriptname` does not work.
if(UNIX)
find_package(UnixCommands QUIET)
dune_module_path(MODULE dune-common
RESULT scriptdir
SCRIPT_DIR)
configure_file(${scriptdir}/run-in-dune-env.sh.in
${CMAKE_BINARY_DIR}/run-in-dune-env
@ONLY)
else()
message(WARNING "Writing script 'run-in-dune-env' not implemented on your platform!")
endif()
if(DUNE_PYTHON_USE_VENV)
# The virtualenv might not contain pip due to the distribution bug described in
# https://bugs.launchpad.net/debian/+source/python3.4/+bug/1290847
# We need to install pip, so if pip is missing, we offer to download and run the get-pip
# script. We ask users for permission to do so, or we allow them to set it up themselves.
dune_python_find_package(PACKAGE pip
RESULT pippresent
INTERPRETER ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE}
)
if(NOT pippresent)
if(DUNE_PYTHON_ALLOW_GET_PIP)
# Fetch the get-pip.py script
message(STATUS "Installing pip using https://bootstrap.pypa.io/get-pip.py...")
file(DOWNLOAD https://bootstrap.pypa.io/get-pip.py ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py)
# Verify that the script was successfully fetched
file(READ ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py verify LIMIT 1)
if(NOT verify)
message(WARNING "Fetching get-pip.py failed. This often happens when CMake is built from "
"source without SSL/TLS support. Consider using a different cmake version or "
"fall back to manually installing pip into the virtualenv.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
# Execute the script
dune_execute_process(COMMAND ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/get-pip.py RESULT_VARIABLE pip_install_result)
if(NOT "${pip_install_result}" STREQUAL "0")
message(WARNING "Fatal error when installing pip into the virtualenv.")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
else()
message(WARNING "dune-common set up a virtualenv, but needs pip to be installed into it. "
"You can either install it yourself manually activating the virtualenv with "
"the activate script in your build directory ${CMAKE_BINARY_DIR} or you set "
"the CMake variable DUNE_PYTHON_ALLOW_GET_PIP to allow Dune to use get-pip.py "
"from https://bootstrap.pypa.io/get-pip.py")
set(DUNE_ENABLE_PYTHONBINDINGS OFF)
return()
endif()
endif()
# if pip was not found before then we can set it here since it was now found
set(DUNE_PYTHON_pip_FOUND ON CACHE INTERNAL "Whether dune has found python pip")
# install setuptools into the venv (needed to find dependencies later on)
dune_execute_process(COMMAND ${DUNE_PYTHON_VIRTUALENV_EXECUTABLE} -m pip install
"${DUNE_PIP_INDEX}"
setuptools>=41 ninja
WARNING_MESSAGE "python 'setuptools' package could not be installed - possibly connection to the python package index failed"
)
else()
# Write a small wrapper script 'run-in-dune-env' into the build directory
# This is necessary to execute installed python scripts (the bin path of a virtualenv
# is *not* in the sys path, so a simple `python scriptname` does not work.
if(UNIX)
# create list with python module paths
foreach( mod ${DUNE_FOUND_DEPENDENCIES} )
if( NOT "_${${mod}_PYTHON_WHEELHOUSE}_" STREQUAL "__")
set(MODULE_DEP_PYTHON_PATH "${MODULE_DEP_PYTHON_PATH}\n ${${mod}_PYTHON_WHEELHOUSE}")
endif()
endforeach()
# for dune-common this variable is not set but needed by the script
if( NOT dune-common_DIR )
set( dune-common_DIR ${CMAKE_BINARY_DIR} )
endif()
find_package(UnixCommands QUIET)
dune_module_path(MODULE dune-common
RESULT scriptdir
SCRIPT_DIR)
configure_file(${scriptdir}/set-dune-pythonpath.sh.in
${CMAKE_BINARY_DIR}/set-dune-pythonpath
@ONLY)
else()
message(WARNING "Writing script 'set-dune-pythonpath' not implemented on your platform!")
endif()
endif()
|