# SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root # SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception import importlib import logging import subprocess import os import sys import shlex import jinja2 import json import copy from pathlib import Path import dune from dune.packagemetadata import ( getDunePyDir, Description, getBuildMetaData, getCMakeFlags, defaultCMakeFlags, getExternalPythonModules ) from dune.common import comm from dune.common.locking import Lock, LOCK_EX, LOCK_SH from dune.common.utility import buffer_to_str, isString, reload_module from dune.generator.exceptions import CompileError from dune.generator.remove import removeGenerated logger = logging.getLogger(__name__) logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) cxxFlags = None noDepCheck = False moduleLogFile = None def setModuleLog( fileName=None, procs=4 ): global moduleLogFile # if this is the first call then `moduleLogFile` is still None and then # test the environment variable # DUNE_LOGMODULES: if greater zero then log module names using the main script name # if the environment variable is not set (or <=0) then use the fileName if provided if moduleLogFile is None: # check DUNE_LOGMODULES import __main__ as main procs_ = int( os.environ.get('DUNE_LOGMODULES', '-1') ) if procs_ > 1 and fileName is None: procs = procs_ try: fileName = Path(main.__file__).stem + ".modules" except AttributeError: # this can happen with notebooks fileName = None if not fileName is None: # if fileName is set rebuild all listed if file exists from dune.generator.make import makeGenerated moduleLogFile = fileName if Path(fileName).is_file(): makeGenerated([], fileName=fileName, threads=procs, force=False) open(fileName,"w").close() # clear file return moduleLogFile def deprecationMessage(dune_py_dir): print(f"Using a pre-existing old style dune-py with a newer version of dune-common. Remove dune-py folder `{dune_py_dir}` and re-run Python script!") print(f"It might also be required to execute 'cmake .' in the 'dune-common' build directory") print(f"It is possible to continue to use the old version by typing 'export DUNE_PY_USE_CMAKEBUILDER=1'") sys.exit(1) def getCMakeCommand(): try: return os.environ['DUNE_CMAKE'] except KeyError: return 'cmake' def getDefaultBuildArgs(): try: return shlex.split(os.environ['DUNE_BUILD_FLAGS']) except KeyError: return None class Builder: @staticmethod def sync_dir(dir_path): """ Execute fsync on a directory ensuring it is synced to disk :param str dir_path: The directory to sync :raise OSError: If fail opening the directory """ dir_fd = os.open(dir_path, os.O_DIRECTORY) try: os.fsync(dir_fd) except OSError as e: # On some filesystem doing a fsync on a directory # raises an EINVAL error. Ignoring it is usually safe. if e.errno != errno.EINVAL: raise finally: os.close(dir_fd) @staticmethod def generate_dunepy_from_template(dunepy_dir,force=False): # Extract the raw data dictionary metaData = getBuildMetaData() modules = metaData.combine_across_modules("MODULENAME") builddirs = metaData.zip_across_modules("DEPS", "DEPBUILDDIRS") # add dune modules which where available during the build of a # python module but don't provide their own python module for k,v in builddirs.items(): if k not in modules and not v.endswith("NOTFOUND"): modules += [k] modules.sort() logger.debug("Found the following Dune module dependencies for dune-py: " + str(modules)) # for an existing dune-py compare module dependencies to check if rebuild required if os.path.exists(os.path.join(dunepy_dir,"dune.module")): description = Description(os.path.join(dunepy_dir,"dune.module")) deps = description.depends force = not modules == [d[0] for d in deps] # ignore version number in dependency for now # todo also check if location of modules have changed to find # modules moved from build to install or vice versa - or tag that some other way # For now we simply always force a rebuild of the dune-py # module which might lead to more work than required force = True else: force = True if force: # Remove any remnants of an old dune-py installation # if os.path.exists(dunepy_dir): # shutil.rmtree(dunepy_dir) # only remove cache try: os.remove(os.path.join(dunepy_dir, 'CMakeCache.txt')) except FileNotFoundError: pass # Gather and reorganize meta data context that is used to write dune-py context = {} context["modules"] = modules context["builddirs"] = builddirs context["install_prefix"] = metaData.unique_value_across_modules("INSTALL_PREFIX") context["cmake_flags"] = getCMakeFlags().copy() context["dunepy_dir"] = dunepy_dir # to use the default launcher we move the CMAKE_CXX_FLAGS to DEFAULT_CXXFLAGS # to get the compile command without CXX_FLAGS we then remove them try: cxxflags = context["cmake_flags"].pop("CMAKE_CXX_FLAGS") context["DEFAULT_CXXFLAGS"] = cxxflags except KeyError: pass # we also need to make sure that no 'build type' (is set to 'release' in the packaging framework skbuild) # Note: this means the value can not be set by the user # perhaps we can find a better solution here but for now # if we keep it the launcher will not work with packages try: context["cmake_flags"].pop("CMAKE_BUILD_TYPE") except KeyError: pass # Find the correct template path path, _ = os.path.split(__file__) template_path = os.path.join(path, "template") # Run the template through Jinja2 env = jinja2.Environment( loader=jinja2.FileSystemLoader(template_path), keep_trailing_newline=True, ) for root, _, files in os.walk(template_path): # fix issues due to template taken from build dir if root.endswith("CMakeFiles") or root.endswith("src_dir"): continue for template_file in files: full_template_file = os.path.join(root, template_file) relative_template_file = os.path.relpath(full_template_file, start=template_path) gen_file = os.path.join(dunepy_dir, relative_template_file.split(".template")[0]) # we don't want to remove CMakeLists.txt in the generated folder if it already exists if root.endswith('generated') and os.path.exists(gen_file): continue if gen_file.endswith(".cmake") or gen_file.endswith("Makefile"): continue # issues due to template taken from build dir os.makedirs(os.path.split(gen_file)[0], exist_ok=True) # make sure the launcher can be executed def opener(path,flags): if "compiler_launcher" in path: return os.open(path, flags, 0o774) else: return os.open(path, flags, 0o664) with open(gen_file, "w", opener=opener) as outfile: outfile.write(env.get_template(relative_template_file).render(**context)) os.fsync(outfile) # make sure files are correctly synced before calling make or cmake logger.debug("Context used to configure dune-py: {}".format(context)) # return force variable because this might be needed in the overloaded versions return force @staticmethod def build_dunepy_from_template(dunepy_dir,force=False): force = Builder.generate_dunepy_from_template(dunepy_dir,force) if force: # configure dune-py Builder.callCMake(["cmake"]+defaultCMakeFlags()+["."], cwd=dunepy_dir, infoTxt="Configuring dune-py with CMake", active=True, # print details anyway ) return force def __init__(self, force=False, saveOutput=False): self.force = force self.skipTargetAll = False if saveOutput is True or saveOutput.lower() == "write": self.savedOutput = [open("generatorCompiler.out","w+"), open("generatorCompiler.err","w+")] elif saveOutput.lower() == "append": self.savedOutput = [open("generatorCompiler.out","a+"), open("generatorCompiler.err","a+")] elif saveOutput.lower() == "console" or \ saveOutput.lower() == "terminal" or \ saveOutput.lower() == "console_nocmake": self.savedOutput = [sys.stdout, sys.stderr] if saveOutput.lower() == "console_nocmake": self.skipTargetAll = True else: self.savedOutput = None self.dune_py_dir = getDunePyDir() self.build_args = getDefaultBuildArgs() self.generated_dir = os.path.join(self.dune_py_dir, 'python', 'dune', 'generated') self.initialized = False self.externalPythonModules = copy.deepcopy(getExternalPythonModules()) def cacheExternalModules(self): """Store external modules in dune-py""" externalModulesPath = os.path.join(self.dune_py_dir, ".externalmodules.json") with open(externalModulesPath, "w") as externalModulesFile: json.dump(self.externalPythonModules, externalModulesFile) def initialize(self): # check if we need to initialize dune-py either because # this is the first call to load or because an external module with metadata has been registered if self.initialized and self.externalPythonModules == getExternalPythonModules(): return False self.externalPythonModules = copy.deepcopy(getExternalPythonModules()) if comm.rank == 0: logger.debug("(Re-)Initializing JIT compilation module") os.makedirs(self.dune_py_dir, exist_ok=True) # need to lock here so that multiple processes don't try to # generate a new dune-py at the same time with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX): # check for existence of tag file - this is removed if a # new dune package is added tagfile = os.path.join(self.dune_py_dir, ".noconfigure") if not os.path.isfile(tagfile): logger.info('Generating dune-py module in ' + self.dune_py_dir) # create module cache for external modules that have been registered with dune-py self.cacheExternalModules() # create dune-py module self.build_dunepy_from_template(self.dune_py_dir) # create tag file so that dune-py is not rebuilt on the next build open(tagfile, 'a').close() else: logger.debug('Using existing dune-py module in ' + self.dune_py_dir) self.compile("Rebuilding dune-py module") # Auto-clean up dune-py: Remove all modules that have not been used in the last 30 days. removeGenerated(['30'], date=True) comm.barrier() try: dune.__path__._path.insert(0,os.path.join(self.dune_py_dir, 'python', 'dune')) except: dune.__path__.insert(0,os.path.join(self.dune_py_dir, 'python', 'dune')) self.initialized = True return True @staticmethod def callCMake(cmake_args, cwd, infoTxt, verbose=False, active=False, logLevel=logging.DEBUG, env=None): # print initial info, if we are verbose # or we know that we have to build something stdout, stderr = None, None active = active or verbose if active: if infoTxt: logger.log(logLevel,infoTxt) # call the cmake process if verbose: print(cmake_args) # make sure directory entries are properly written to avoid raceconditions on network storage. # Builder.sync_dir(self.generated_dir) with subprocess.Popen(cmake_args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as cmake: try: stdout, stderr = cmake.communicate(timeout=2) # no message if delay is < 2sec except subprocess.TimeoutExpired: if infoTxt and not active: logger.log(logLevel, infoTxt) # wait for cmd to finish stdout, stderr = cmake.communicate() # check return code if cmake.returncode > 0: # retrieve stderr output print(stderr,stdout) raise CompileError(buffer_to_str(stderr)) return stdout, stderr def _compile(self, infoTxt, target='all', verbose=False): # if called for other builders don't do anything here assert isinstance(self, Builder) cmake_command = getCMakeCommand() cmake_args = [cmake_command, "--build", self.dune_py_dir, "--target", target, "--parallel"] make_args = [] if self.build_args is not None: make_args += self.build_args if cxxFlags is not None: make_args += ['CXXFLAGS='+cxxFlags] if noDepCheck: make_args += ["-B"] if cmake_args != []: cmake_args += ["--"] + make_args stdout, stderr = Builder.callCMake(cmake_args, cwd=self.generated_dir, infoTxt=infoTxt, verbose=verbose, logLevel=logging.INFO ) writeOutput = self.savedOutput is not None and target != 'all' if self.skipTargetAll else self.savedOutput is not None if writeOutput: if stdout is not None: out = buffer_to_str(stdout) nlines = out.count('\n') if nlines > 1: self.savedOutput[0].write("###############################\n") self.savedOutput[0].write("###" + " ".join(cmake_args)+"\n") if nlines > 0: self.savedOutput[0].write(out) if nlines > 1: self.savedOutput[0].write("\n###############################\n") if stderr is not None: err = buffer_to_str(stderr) nlines = err.count('\n') if nlines > 1: self.savedOutput[1].write("###############################\n") self.savedOutput[1].write("###" + " ".join(cmake_args)+"\n") if nlines > 0: self.savedOutput[1].write(err) if nlines > 1: self.savedOutput[1].write("\n###############################\n") def compile(self, infoTxt, target='all', verbose=False): return self._compile(infoTxt, target=target, verbose=verbose) def _maybeConfigureWithCMake(self, moduleName, source, pythonName, extraCMake): sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc") line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)" # first check if this line is already present in the CMakeLists file # (possible if a previous script was stopped by user before module was compiled) with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out: found = line in out.read() if not os.path.isfile(sourceFileName) or not found: compilationInfoMessage = f"Compiling {pythonName} (new)" code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) assert os.path.isfile(sourceFileName), "Error in writing module .cc file" if not found: origPos = -1 with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out: # store original file size origPos = out.tell() out.write(line+"\n") if extraCMake is not None: for x in extraCMake: out.write(x.replace("TARGET",moduleName)+"\n") # update build system try: Builder.callCMake(["cmake","."], cwd=self.dune_py_dir, infoTxt="Configuring module {} ({}) with CMake".format( pythonName, moduleName ), ) except: # all exceptions will cause a problem here os.remove(os.path.join(sourceFileName)) # remove line from CMakeLists to avoid problems with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'a') as out: out.truncate(origPos) raise elif isString(source) and not source == open(os.path.join(sourceFileName), 'r').read(): compilationInfoMessage = f"Compiling {pythonName} (updated)" code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) else: # we can directly load the module - after checking if headers have changed compilationInfoMessage = f"Compiling {pythonName} (rebuilding)" # sanity check line = "dune_add_pybind11_module(NAME " + moduleName + " EXCLUDE_FROM_ALL)" # the CMakeLists file should already include this line with open(os.path.join(self.generated_dir, "CMakeLists.txt"), 'r') as out: found = line in out.read() assert found, "CMakeLists.txt file does not contain an entry to build"+moduleName return compilationInfoMessage def load(self, moduleName, source, pythonName, extraCMake=None): self.initialize() if pythonName is None: pythonName = moduleName # check whether module is already compiled and build it if necessary # (only try to build module on rank 0!) # TODO replace if rank with something better and remove barrier further down if comm.rank == 0: module = sys.modules.get("dune.generated." + moduleName) if setModuleLog(): with open( setModuleLog(), 'a' ) as file: file.write(moduleName + '\n') if module is None: self._buildModule( moduleName, source, pythonName, extraCMake ) ## TODO remove barrier here comm.barrier() logger.debug("Loading " + moduleName) module = importlib.import_module("dune.generated." + moduleName) if self.force: logger.info("Reloading " + pythonName) module = reload_module(module) return module def _buildModule(self, moduleName, source, pythonName, extraCMake): logger.debug("Module {} not loaded".format(moduleName)) # make sure nothing (compilation, generating and building) is taking place with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_EX): # module must be generated so lock the source file with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX): # the module might now be present, so check again # (see #295) module = sys.modules.get("dune.generated." + moduleName) if module is None: compilationMessage = self._maybeConfigureWithCMake( moduleName, source, pythonName, extraCMake ) else: compilationMessage = f"Compiling {pythonName} (rebuilding after concurrent build)" # end of exclusive dune-py lock # we always compile even if the module is always compiled since it can happen # that dune-py was updated in the mean time. # This step is quite fast but there is room for optimization. # for compilation a shared lock is enough # # A side effect is that during the parallel make calls # for whatever reason cmake might be invoked due to # changes to the module (i.e. a new target # added). Parallel cmake calls are not allowed and as # a consequence the complete build may fail. We take # care of such parallel cmake calls by additional # locking in the dune-py CMakeLists.txt with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_SH): # lock generated module with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX): self._compile(infoTxt=compilationMessage, target=moduleName) return module ############################################################################ ## ## Builder based on Makefiles ## ############################################################################ class MakefileBuilder(Builder): # static vars holding command for bash and make bashCmd = 'bash' # default values makeCmd = 'make' # set in dunepy_from_template @staticmethod def build_dunepy_from_template(dunepy_dir, force=False): # call base class dunepy_from_template force = Builder.generate_dunepy_from_template(dunepy_dir, force=force) if force: # generate path for the buildScript and other filed to be created generatedDir = os.path.join(dunepy_dir,'python','dune','generated') buildScriptName = os.path.join(dunepy_dir,'python','dune','generated','buildScript.sh') # check whether ninja is available with cmake or not useNinja = False # to be revised in later versions if useNinja: try: Builder.callCMake(["cmake", "-G", "Ninja"] + defaultCMakeFlags() + ["."], cwd=dunepy_dir, infoTxt="Configuring dune-py with CMake (ninja)", active=True, # print details anyway ) logger.debug("Using ninja as cmake generator!") except: logger.debug("Check for ninja failed!") useNinja = False if not useNinja: # call base class dunepy_from_template (re-initialize) force = Builder.generate_dunepy_from_template(dunepy_dir, force=True) # extract compiler launcher if provided by user launcher = getCMakeFlags().get('CMAKE_CXX_COMPILER_LAUNCHER', os.path.join(dunepy_dir,"compiler_launcher.sh")) # the default flags contain CXX_FLAGS but we need to keep # those empty to get the correct compiler command cmake_flags = defaultCMakeFlags(overwrite={'CMAKE_CXX_FLAGS':' '}) # the launcher might be given through a environment # variable which is taken care of here for flag in cmake_flags: if flag.find('CMAKE_CXX_COMPILER_LAUNCHER') > 0: launcher = flag.split("=")[1] # adding CMAKE_EXPORT_COMPILE_COMMANDS=ON because only # having that in the CMakeLists.txt file fails for some of # the CI runs # Builder.callCMake(["cmake"] + cmake_flags + ["."], Builder.callCMake(["cmake","-DCMAKE_EXPORT_COMPILE_COMMANDS=ON","."], cwd=dunepy_dir, infoTxt="Configuring dune-py with CMake (make)", active=True, # print details anyway ) # obtain bash and make command generated by cmake # when ninja is used makeCmd is the ninja command try: commandSourceName = os.path.join(dunepy_dir,'unix_commands.json') with open(commandSourceName) as commandFile: cmdList = json.load(commandFile)[0] MakefileBuilder.bashCmd = cmdList["bash"] # shell command MakefileBuilder.makeCmd = cmdList["make"] # make command MakefileBuilder.generatorCmd = cmdList["generator"] # generator (make or ninja) except FileNotFoundError: pass if not useNinja: ######################################################################### # Remark about ninja: # - the -B used below for the extractCompiler build needed for # the second approach does not work with ninja # - the link.txt used in approach 1/3 is not available with ninja # - the CXXFLAGS overwrite seems not to work with ninja - # the script is not generated so approach 2 will not work due to missing echo # We don't have to use ninja explicitly for dune-py but have # to make sure the user can't set up ninja for dune-py by # mistake ######################################################################### try: stdout, stderr = \ Builder.callCMake(["cmake"]+ ['--build','.','--target',"extractCompiler"]+ ['--','-B'], cwd=dunepy_dir, env={**os.environ, "CXXFLAGS":" ", }, infoTxt="extract compiler command", active=True, # print details anyway ) except CompileError: deprecationMessage(dunepy_dir) ######################################################################## # Write buildScript.sh ######################################################################## # we already have link.txt from the previous cmake run (extractCompiler) buildDirBase = os.path.join(generatedDir,'CMakeFiles','extractCompiler.dir') linkerSourceName = os.path.join(buildDirBase,'link.txt') buildSourceName = os.path.join(buildDirBase,'build.make') commandSourceName = os.path.join(dunepy_dir,'compile_commands.json') with open(buildScriptName, "w") as buildScript: # write bash line buildScript.write("#!" + MakefileBuilder.bashCmd + "\n") # write compiler commands with open(commandSourceName) as commandFile: compilerCmd = json.load(commandFile)[0]["command"] # replace target file compilerCmd = compilerCmd.replace('extractCompiler', '$1') # this line is extracted from build.make usedBuildMake = False #with open(buildSourceName, "r") as buildFile: # buildCmd = buildFile.readlines() # for idx,line in enumerate(buildCmd): # # compiler command is next line after "Building CXX object ..." # if "Building CXX object " in line and idx < len(buildCmd)-1: # cmd = buildCmd[idx+1].partition("(CXX_FLAGS)")[2] # cmd = cmd.partition(" -o ")[0].replace('extractCompiler', '$1') # compilerCmd += cmd # usedBuildMake = True # break # fall back in case something went wrong with parsing build.make if not usedBuildMake: compilerCmd = compilerCmd + " -MD -MT CMakeFiles/$1.dir/$1.cc.o -MF CMakeFiles/$1.dir/$1.cc.o.d" # forward errors so that compilation failure will be caught buildScript.write('set -e\n') # add launcher before compiler buildScript.write('if [ "$DUNE_CXX_COMPILER_LAUNCHER" == "" ]; then\n') buildScript.write(' DUNE_CXX_COMPILER_LAUNCHER='+launcher+'\n') buildScript.write('fi\n') buildScript.write("$DUNE_CXX_COMPILER_LAUNCHER " + compilerCmd+"\n") # write linker commands with open(linkerSourceName, "r") as linkerSource: linkerCmd = linkerSource.read() linkerCmd = linkerCmd.replace('extractCompiler','$1') buildScript.write(linkerCmd) os.fsync(buildScript) # make sure files are correctly synced before calling make or cmake else: # useNinja ninja = MakefileBuilder.generatorCmd assert ninja.find("ninja") # make sure directory entries are properly written to avoid raceconditions on network storage. Builder.sync_dir(self.generated_dir) with subprocess.Popen([ninja, "-t","commands","extractCompiler"], cwd=dunepy_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as ninjaProcess: stdout, stderr = ninjaProcess.communicate() exit_code = ninjaProcess.returncode assert exit_code == 0 ######################################################################## # Write buildScript.sh ######################################################################## out = buffer_to_str(stdout).strip().split("\n") assert len(out) == 2 with open(buildScriptName, "w") as buildScript: # write bash line buildScript.write("#!" + MakefileBuilder.bashCmd + "\n") buildScript.write("set -e\n") compilerCmd = out[0].replace('extractCompiler', '$1').\ replace(' python/dune/generated/',' ') # better to move the script to the root of dune-py then this can be kept compilerCmd = compilerCmd.split(' ',1) compilerCmd = compilerCmd[0] + " " + compilerCmd[1] # add launcher before compiler buildScript.write(launcher + " " + compilerCmd+"\n") # this needs fixing: Issue is that at the linker line begins with ': && ' linkerCmd = out[1].replace('extractCompiler','$1').\ replace(' python/dune/generated/',' ') # better to move the script to the root of dune-py then this can be kept linkerCmd = linkerCmd.split(' ',3) linkerCmd = linkerCmd[2] + " " + linkerCmd[3].replace("&& :", "") buildScript.write(linkerCmd+"\n") os.fsync(buildScript) # make sure files are correctly synced before calling make or cmake # constructor def __init__(self, force=False, saveOutput=False): # call __init__ of base class super().__init__(force=force, saveOutput=saveOutput) # just added to check for old versions of dune-py - can be removed once # this check is not needed anymore def initialize(self): if not super().initialize(): return # check that the compile script is available script = os.path.join(self.generated_dir,"buildScript.sh") if not os.path.exists(script): deprecationMessage(self.dune_py_dir) # nothing to be done in this class def compile(self, infoTxt, target='all', verbose=False): pass # open and safely close source file and return content def _equalToExistingFile(self, source, sourceFileName ): with open(os.path.join(sourceFileName), 'r') as sFile: content = sFile.read() if len(content) == len(source): return content == source # by default return False return False def _configureWithMake(self, moduleName, source, pythonName): sourceFileName = os.path.join(self.generated_dir, moduleName + ".cc") if not os.path.isfile(sourceFileName): compilationInfoMessage = f"Compiling {pythonName} (new)" code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) elif isString(source) and not self._equalToExistingFile(source, sourceFileName): # if source is not equal to existing source then replace it compilationInfoMessage = f"Compiling {pythonName} (updated)" code = str(source) with open(os.path.join(sourceFileName), 'w') as out: out.write(code) else: compilationInfoMessage = f"{pythonName} (loading)" os.makedirs(os.path.join(self.generated_dir,"CMakeFiles",moduleName+".dir"), exist_ok=True) return compilationInfoMessage def _buildModule(self, moduleName, source, pythonName, extraCMake ): logger.debug("Module {} not loaded".format(moduleName)) # make sure nothing (compilation, generating and building) is taking place # module must be generated so lock the source file with Lock(os.path.join(self.dune_py_dir, '..', 'lock-module.lock'), flags=LOCK_SH): with Lock(os.path.join(self.dune_py_dir, 'lock-'+moduleName+'.lock'), flags=LOCK_EX): # the module might now be present, so check again # (see #295) module = sys.modules.get("dune.generated." + moduleName) if module is None: compilationMessage = self._configureWithMake( moduleName, source, pythonName ) else: compilationMessage = f"Compiling {pythonName} (rebuilding after concurrent build)" # we always compile even if the module is already compiled since it can happen # that dune-py was updated in the mean time ????? # This step is quite fast but there is room for optimization. # we can always generate a new makefile - in case no # dependency file has already been generate leave # dependencies empty otherwise use existing depFile: depFileName = os.path.join(self.generated_dir,"CMakeFiles",moduleName+'.dir',moduleName+'.cc.o.d') makeFileName = self._makeFileName( moduleName ) with open(makeFileName, "w") as makeFile: makeFile.write('.SUFFIXES:\n') try: with open(depFileName, "r") as depFile: makeFile.write(depFile.read()) except FileNotFoundError: makeFile.write(os.path.join("CMakeFiles",moduleName+'.dir',moduleName+'.cc.o')+':\n') pass makeFile.write('\t'+MakefileBuilder.bashCmd+ ' buildScript.sh '+moduleName+"\n") makeFile.write(moduleName+'.so: '+os.path.join("CMakeFiles",moduleName+'.dir',moduleName+'.cc.o')+'\n') os.fsync(makeFile) # make sure files are correctly synced before calling make or cmake # make sure directory entries are properly written to avoid raceconditions on network storage. # Builder.sync_dir(self.generated_dir) # first just check if the makefile does not contain any error # An issue could be that a file in the dependency list has # moved (e.g. from installed to source build) with subprocess.Popen([MakefileBuilder.makeCmd, "-q", "-f", makeFileName, moduleName+'.so'], cwd=self.generated_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as make: stdout, stderr = make.communicate() exit_code = make.returncode # this means that there was a problem # with the makefile so we remove the dependencies and the # generated module so that it will be regenerated if exit_code == 2 or noDepCheck: with open(makeFileName, "w") as makeFile: makeFile.write('.SUFFIXES:\n') makeFile.write(os.path.join("CMakeFiles",moduleName+'.dir',moduleName+'.cc.o')+':\n') makeFile.write('\t'+MakefileBuilder.bashCmd+ ' buildScript.sh '+moduleName+"\n") makeFile.write(moduleName+'.so: '+os.path.join("CMakeFiles",moduleName+'.dir',moduleName+'.cc.o')+'\n') os.fsync(makeFile) # make sure files are correctly synced before calling make or cmake # make sure directory entries are properly written to avoid raceconditions on network storage. Builder.sync_dir(self.generated_dir) if exit_code > 0 or noDepCheck: # make sure directory entries are properly written to avoid raceconditions on network storage. # Builder.sync_dir(self.generated_dir) # call make to build shared library self.makeModule( moduleName, makeFileName, compilationMessage, force=noDepCheck ) def _makeFileName( self, moduleName ): return os.path.join(self.generated_dir,"CMakeFiles",moduleName+'.dir',moduleName+'.make') def makeModule(self, moduleName, makeFileName=None, compilationMessage=None, force=False ): if makeFileName is None: makeFileName = self._makeFileName( moduleName ) if compilationMessage is None: compilationMessage = f"{moduleName} rebuilding" makeArgs = [MakefileBuilder.makeCmd] if force: makeArgs += ["-B"] makeEnv = os.environ.copy() if cxxFlags is not None: makeEnv['CXXFLAGS'] = cxxFlags with subprocess.Popen(makeArgs + ["-f",makeFileName, moduleName+'.so'], cwd=self.generated_dir, env=makeEnv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as make: try: stdout, stderr = make.communicate(timeout=2) # no message if delay is <2sec except subprocess.TimeoutExpired: # compilation is taking place, replace loading with rebuilding compilationMessage = compilationMessage.replace("loading", "rebuilding") logger.log(logging.INFO,compilationMessage) # wait for cmd to finish stdout, stderr = make.communicate() exit_code = make.returncode if self.savedOutput is not None: self.savedOutput[0].write('make return:' + str(exit_code) + "\n") self.savedOutput[0].write(str(stdout.decode()) + "\n") self.savedOutput[1].write(str(stderr.decode()) + "\n") # check return code if exit_code > 0: # retrieve stderr output raise CompileError(buffer_to_str(stderr))