# -*- coding: utf-8 -*- #------------------------------------------------------------------------------- # This file is part of Code_Saturne, a general-purpose CFD tool. # # Copyright (C) 1998-2019 EDF S.A. # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 Franklin # Street, Fifth Floor, Boston, MA 02110-1301, USA. #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # Standard modules import #------------------------------------------------------------------------------- import os, sys import shutil, re import subprocess import threading import string import time import logging import fnmatch #------------------------------------------------------------------------------- # Application modules import #------------------------------------------------------------------------------- from cs_exec_environment import get_shell_type, enquote_arg from cs_compile import files_to_compile, compile_and_link import cs_create from cs_create import set_executable import cs_runcase from code_saturne.model import XMLengine from code_saturne.studymanager.cs_studymanager_pathes_model import PathesModel from studymanager.cs_studymanager_parser import Parser from studymanager.cs_studymanager_texmaker import Report1, Report2 try: from studymanager.cs_studymanager_drawing import Plotter except Exception: print("Warning: import studymanager Plotter failed. Plotting disabled.\n") pass from studymanager.cs_studymanager_run import run_studymanager_command #------------------------------------------------------------------------------- # log config. #------------------------------------------------------------------------------- logging.basicConfig() log = logging.getLogger(__file__) #log.setLevel(logging.DEBUG) log.setLevel(logging.NOTSET) #------------------------------------------------------------------------------- def nodot(item): return item[0] != '.' #------------------------------------------------------------------------------- def create_base_xml_file(filepath): """Create studymanager XML file. """ filename = os.path.basename(filepath) if os.path.isfile(filepath): print("Can not create XML file of parameter:\n" \ + filepath + " already exists.") sys.exit(1) # using xml engine from Code_Saturne GUI smgr = XMLengine.Case(None, studymanager=True) smgr['xmlfile'] = filename pm = PathesModel(smgr) # empty repo and dest pm.setRepositoryPath('') pm.setDestinationPath('') return smgr #------------------------------------------------------------------------------- def init_xml_file_with_study(smgr, studyp): """Initialize XML file with study content and save it. """ smgr_node = smgr.xmlGetNode('studymanager') studyd = os.path.basename(studyp) st_node = smgr_node.xmlInitChildNode('study', label = studyd) st_node['status'] = "on" cases = [] for elt in os.listdir(studyp): eltd = os.path.join(studyp, elt) if os.path.isdir(eltd): if isCase(eltd): cases.append(elt) cases.sort() for case in cases: c_node = st_node.xmlInitChildNode("case", label = case) c_node['status'] = "on" c_node['compute'] = "on" c_node['post'] = "on" smgr.xmlSaveDocument() #------------------------------------------------------------------------------- def isStudy(dirpath): """Try to determine if dirpath is a Code_Saturne study directory. """ meshd = os.path.join(dirpath, 'MESH') is_study = os.path.isdir(meshd) return is_study #------------------------------------------------------------------------------- def isCase(dirpath): """Try to determine if dirpath is a Code_Saturne case directory. """ datad = os.path.join(dirpath, 'DATA') scriptd = os.path.join(dirpath, 'SCRIPTS') is_case = os.path.isdir(datad) and os.path.isdir(scriptd) return is_case #=============================================================================== # Case class #=============================================================================== class Case(object): def __init__(self, pkg, rlog, diff, parser, study, data, repo, dest): """ @type data: C{Dictionary} @param data: contains all keyword and value read in the parameters file """ self.__log = rlog self.__diff = diff self.__parser = parser self.__study = study self.__data = data self.__repo = repo self.__dest = dest self.node = data['node'] self.label = data['label'] self.compute = data['compute'] self.plot = data['post'] self.run_id = data['run_id'] self.tags = data['tags'] self.compare = data['compare'] self.is_compiled= "not done" self.is_run = "not done" self.is_time = None self.is_plot = "not done" self.is_compare = "not done" self.threshold = "default" self.diff_value = [] # list of differences (in case of comparison) self.m_size_eq = True # mesh sizes equal (in case of comparison) self.subdomains = None self.run_dir = "" self.resu = 'RESU' # Specific case for coupling coupling = os.path.join(self.__repo, self.label, "coupling_parameters.py") if os.path.isfile(coupling): import cs_case_coupling try: exec(compile(open(coupling).read(), '', 'exec')) except Exception: execfile(coupling) run_ref = os.path.join(self.__repo, self.label, "runcase") self.exe, self.pkg = self.__get_exe(pkg, run_ref) self.subdomains = [] for d in locals()['domains']: if d['solver'] == self.pkg.code_name: self.subdomains.append(d['domain']) elif d['solver'].lower() == "syrthes": syrthes = True self.resu = 'RESU_COUPLING' # insert syrthes path if syrthes: syrthes_insert = cs_create.syrthes_path_line(pkg) if syrthes_insert: fd = open(coupling) fd_lines = fd.readlines() fd.close() fd = open(coupling, 'w') for line in fd_lines: if "sys.path.insert" in line: fd.write(syrthes_insert) else: fd.write(line) fd.close() else: run_ref = os.path.join(self.__repo, self.label, "SCRIPTS", "runcase") self.exe, self.pkg = self.__get_exe(pkg, run_ref) #--------------------------------------------------------------------------- def __get_exe(self, old_pkg, run_ref): """ Return the name of the exe of the case, in order to mix Code_Saturne and NEPTUNE_CFD test cases in the same study. """ # Read the runcase script from the Repository runcase = cs_runcase.runcase(run_ref) if runcase.cmd_name == "code_saturne": from cs_package import package elif runcase.cmd_name == "neptune_cfd": from nc_package import package pkg = package() return runcase.cmd_name, pkg #--------------------------------------------------------------------------- def __update_domain(self, subdir, xmlonly=False): """ Update path for the script in the Repository. """ # 1) Load the xml file of parameters in order to update it # with the __backwardCompatibility method. from model.XMLengine import Case if self.exe == "code_saturne": from model.XMLinitialize import XMLinit elif self.exe == "neptune_cfd": from model.XMLinitializeNeptune import XMLinit for fn in os.listdir(os.path.join(self.__repo, subdir, "DATA")): fp = os.path.join(self.__repo, subdir, "DATA", fn) if os.path.isfile(fp): fd = os.open(fp , os.O_RDONLY) f = os.fdopen(fd) l = f.readline() f.close() if l.startswith(''' 0: self.is_compiled = "OK" retcode += compile_and_link(self.pkg, s, dest_dir, stdout=log, stderr=log) if retcode > 0: self.is_compiled = "KO" return self.is_compiled #--------------------------------------------------------------------------- def __suggest_run_id(self): cmd = enquote_arg(os.path.join(self.pkg.get_dir('bindir'), self.exe)) + " run --suggest-id" if self.subdomains: cmd += " --coupling=coupling_parameters.py" p = subprocess.Popen(cmd, shell=True, executable=get_shell_type(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) i = p.communicate()[0] run_id = " ".join(i.split()) return run_id, os.path.join(self.__dest, self.label, self.resu, run_id) #--------------------------------------------------------------------------- def __updateRuncase(self, run_id): """ Update the command line in the launcher C{runcase}. """ from cs_exec_environment import separate_args, \ get_command_single_value # Prepare runcase path scripts_repo = os.path.join(self.__repo, self.label) scripts_dest = os.path.join(self.__dest, self.label) if not self.subdomains: scripts_repo = os.path.join(scripts_repo, "SCRIPTS") scripts_dest = os.path.join(scripts_dest, "SCRIPTS") run_ref = os.path.join(scripts_repo, "runcase") run_ref_win = os.path.join(scripts_repo, "runcase.bat") run_new = os.path.join(scripts_dest, "runcase") if sys.platform.startswith('win'): run_new = os.path.join(scripts_dest, "runcase.bat") # Read runcase from repo path = run_ref if not os.path.isfile(path): path = run_ref_win if not os.path.isfile(path): print("Error: could not find %s (or %s)\n" % run_ref, run_ref_win) sys.exit(1) runcase_repo = cs_runcase.runcase(path, create_if_missing=False) # Read runcase from dest path = run_new runcase_dest = cs_runcase.runcase(path, create_if_missing=False, ignore_batch=True) # Assign run command from repo in dest runcase_dest.set_run_args(runcase_repo.get_run_args()) # set run_id in dest runcase_dest.set_run_id(run_id=run_id) # Set number of processors if provided n_procs = self.__data['n_procs'] if n_procs: runcase_dest.set_nprocs(n_procs) # Write runcase runcase_dest.save() #--------------------------------------------------------------------------- def run(self): """ Check if a run with same result subdirectory name exists and launch run if not. """ home = os.getcwd() if self.subdomains: os.chdir(os.path.join(self.__dest, self.label)) else: os.chdir(os.path.join(self.__dest, self.label, 'SCRIPTS')) if self.run_id: run_id = self.run_id run_dir = os.path.join(self.__dest, self.label, self.resu, run_id) if os.path.isdir(run_dir): if os.path.isfile(os.path.join(run_dir, "error")): self.is_run = "KO" error = 1 else: self.is_run = "OK" error = 0 os.chdir(home) return error else: run_id, run_dir = self.__suggest_run_id() while os.path.isdir(run_dir): run_id, run_dir = self.__suggest_run_id() self.run_id = run_id self.run_dir = run_dir self.__updateRuncase(run_id) if sys.platform.startswith('win'): error, self.is_time = run_studymanager_command("runcase.bat", self.__log) else: error, self.is_time = run_studymanager_command("./runcase", self.__log) if not error: self.is_run = "OK" else: self.is_run = "KO" os.chdir(home) return error #--------------------------------------------------------------------------- def runCompare(self, studies, r, d, threshold, args, reference=None): home = os.getcwd() node = None if reference: result = os.path.join(reference, self.label, self.resu) else: result = os.path.join(self.__repo, self.label, self.resu) # check_dir called again here to get run_id (possibly date-hour) repo, msg = self.check_dir(node, result, r, "repo") if msg: studies.reporting(msg) repo = os.path.join(result, repo, 'checkpoint', 'main') result = os.path.join(self.__dest, self.label, self.resu) # check_dir called again here to get run_id (possibly date-hour) dest, msg = self.check_dir(node, result, d, "dest") if msg: studies.reporting(msg) dest = os.path.join(result, dest, 'checkpoint', 'main') cmd = self.__diff + ' ' + repo + ' ' + dest self.threshold = "default" if threshold != None: cmd += ' --threshold ' + threshold self.threshold = threshold if args != None: cmd += (" " + args) l = args.split() try: i = l.index('--threshold') self.threshold = l[i+1] except: pass l = subprocess.Popen(cmd, shell=True, executable=get_shell_type(), stdout=subprocess.PIPE, universal_newlines=True).stdout lines = l.readlines() # list of field differences tab = [] # meshes have same sizes m_size_eq = True # studymanager compare log only for field of real values for i in range(len(lines)): # only select line with "Type" (english and french) and ';' # since this should be only true for heads of section if lines[i].find("Type") != -1 and lines[i].find(";") != -1: line = [x.replace("\""," ").strip() for x in lines[i].split(";")] name = line[0] info = [x.split(":") for x in line[1:]] info = [[x[0].strip(),x[1].strip()] for x in info] # section with at least 2 informations (location, type) after # their name, and of type r (real) if len(info) >= 2 and info[1][1] in ['r4', 'r8']: # if next line contains size, this means sizes are different if lines[i+1].find("Taille") != -1 or lines[i+1].find("Size") != -1: m_size_eq = False break else: line = [x.strip() for x in lines[i+1].split(";")] vals = [x.split(":") for x in line] vals = [[x[0].strip(),x[1].strip()] for x in vals] tab.append([name.replace("_", "\_"), vals[1][1], vals[2][1], self.threshold]) os.chdir(home) return tab, m_size_eq #--------------------------------------------------------------------------- def run_ok(self, run_dir): """ Check if a result directory contains an error file or if it doesn't contain a summary file """ if not os.path.isdir(run_dir): print("Error: the result directory %s does not exist." % run_dir) sys.exit(1) msg = "" ok = True f_error = os.path.join(run_dir, 'error') if os.path.isfile(f_error): ok = False msg += "the result directory %s in case %s " \ "contains an error file." % (os.path.basename(run_dir), self.label) f_summary = os.path.join(run_dir, 'summary') if not os.path.isfile(f_summary): ok = False msg += "the result directory %s in case %s " \ "does not contain any summary file." \ % (os.path.basename(run_dir), self.label) return ok, msg #--------------------------------------------------------------------------- def check_dir(self, node, result, rep, attr): """ Check coherency between xml file of parameters and repository or destination. """ msg = "Warning: " if not os.path.isdir(result): msg += "the directory %s " \ "does not exist." % (result) return None, msg # 1. The result directory is given if rep != "": # check if it exists rep_f = os.path.join(result, rep) if not os.path.isdir(rep_f): msg += "the result directory %s " \ "does not exist." % (rep_f) return None, msg run_ok = self.run_ok(rep_f) if not run_ok[0]: return None, msg+run_ok[1] # 2. The result directory must be found/read automatically; elif rep == "": # check if there is at least one result directory. if len(list(filter(nodot, os.listdir(result)))) == 0: msg += "there is no result directory in %s." % (result) return None, msg # if no run_id is specified in the xml file # only one result directory allowed in RESU if len(list(filter(nodot, os.listdir(result)))) > 1 \ and self.run_id == "": msg += "there are several result directories in %s " \ "and no run id specified." % (result) return None, msg rep = self.run_id # if no run_id is specified in the xml file # the only result directory present in RESU is taken if rep == "": rep = list(filter(nodot, os.listdir(result)))[0] rep_f = os.path.join(result, rep) if not os.path.isdir(rep_f): msg += "the result directory %s " \ "does not exist." % (rep_f) return None, msg run_ok = self.run_ok(rep_f) if not run_ok[0]: return None, msg+run_ok[1] # 3. Update the file of parameters with the name of the result directory if node: self.__parser.setAttribute(node, attr, rep) return rep, None def check_dirs(self, node, repo, dest, reference=None): """ Check coherency between xml file of parameters and repository and destination. """ msg = None if repo != None: # build path to RESU directory with path to study and case label in the repo if reference: result = os.path.join(reference, self.label, self.resu) else: result = os.path.join(self.__repo, self.label, self.resu) rep, msg = self.check_dir(node, result, repo, "repo") if dest != None: # build path to RESU directory with path to study and case label in the dest result = os.path.join(self.__dest, self.label, self.resu) rep, msg = self.check_dir(node, result, dest, "dest") return msg #=============================================================================== # Study class #=============================================================================== class Study(object): """ Create, run and compare all cases for a given study. """ def __init__(self, pkg, parser, study, exe, dif, rlog, n_procs=None, force_rm=False, force_overwrite=False, with_tags=None, without_tags=None, debug=False): """ Constructor. 1. initialize attributes, 2. build the list of the cases, 3. build the list of the keywords from the runcase. @type parser: C{Parser} @param parser: instance of the parser @type study: C{String} @param study: label of the current study @type exe: C{String} @param exe: name of the solver executable: C{code_saturne} or C{neptune_cfd}. @type dif: C{String} @param dif: name of the diff executable: C{cs_io_dump -d}. @n_procs: C{int} @param n_procs: number of requested processors @type force_rm: C{True} or C{False} @param force_rm: remove always existing cases @type force_overwrite: C{True} or C{False} @param force_overwrite: overwrite files in dest by files in repo @type with_tags: C{List} @param with_tags: list of tags given at the command line @type without_tags: C{List} @param without_tags: list of tags given at the command line @type debug: C{True} or C{False} @param debug: if true, increase verbosity in stdout """ # Initialize attributes self.__package = pkg self.__parser = parser self.__main_exe = exe self.__diff = dif self.__log = rlog self.__force_rm = force_rm self.__force_ow = force_overwrite self.__debug = debug self.__repo = os.path.join(self.__parser.getRepository(), study) self.__dest = os.path.join(self.__parser.getDestination(), study) if not os.path.isdir(self.__repo): print("Error: the directory %s does not exist" % self.__repo) sys.exit(1) self.label = study self.cases = [] self.matplotlib_figures = [] self.input_figures = [] self.case_labels = [] # get list of cases in study on_cases = parser.getStatusOnCasesLabels(study) if not on_cases: print("\n\n\nWarning: no case defined in %s study\n\n\n" % study) else: for data in self.__parser.getStatusOnCasesKeywords(self.label): # n_procs given in smgr command line overwrites n_procs by case if n_procs: data['n_procs'] = str(n_procs) # check if every tag passed by option --with-tags belongs to # list of tags of the current case tagged = False if with_tags and data['tags']: tagged = all(tag in data['tags'] for tag in with_tags) elif not with_tags: tagged = True # check if none of tags passed by option --without-tags # belong to list of tags of the current case exclude = False if without_tags and data['tags']: exclude = any(tag in data['tags'] for tag in without_tags) # do not append case if tags do not match if tagged and not exclude: c = Case(pkg, self.__log, self.__diff, self.__parser, self.label, data, self.__repo, self.__dest) self.cases.append(c) self.case_labels.append(c.label) #--------------------------------------------------------------------------- def create_case(self, c, log_lines): """ Create a case in a study """ e = os.path.join(c.pkg.get_dir('bindir'), c.exe) if c.subdomains: os.mkdir(c.label) os.chdir(c.label) refdir = os.path.join(self.__repo, c.label) retval = 1 for node in os.listdir(refdir): ref = os.path.join(self.__repo, c.label, node) if node in c.subdomains: cmd = e + " create --case " + node \ + " --quiet --noref --copy-from " \ + ref node_retval, t = run_studymanager_command(cmd, self.__log) # negative retcode is kept retval = min(node_retval,retval) elif os.path.isdir(ref): shutil.copytree(ref, node, symlinks=True) else: shutil.copy2(ref, node) c.update_runcase_path(c.label, destdir=self.__dest) os.chdir(self.__dest) else: cmd = e + " create --case " + c.label \ + " --quiet --noref --copy-from " \ + os.path.join(self.__repo, c.label) retval, t = run_studymanager_command(cmd, self.__log) if retval == 0: log_lines += [' - create case: ' + c.label] else: log_lines += [' - create case: %s --> FAILED' % c.label] #--------------------------------------------------------------------------- def create_cases(self): """ Create a single study with all its cases. """ repbase = os.getcwd() # Create study if necessary if not os.path.isdir(self.__dest): # build instance of study class from cs_create cr_study = cs_create.Study(self.__package, self.label, [], # cases [], # syrthes cases None, # aster cases None, # cathare case None, # python case None, # copy False,# import_only False,# use ref 0) # quiet # TODO: copy-from for study. For now, an empty study # is created and cases are created one by one with # copy-from # create empty study cr_study.create() # Link meshes and copy other files ref = os.path.join(self.__repo, "MESH") if os.path.isdir(ref): l = os.listdir(ref) meshes = [] for cpr in ["", ".gz"]: for fmt in ["unv", "med", "ccm", "cgns", "neu", "msh", "des"]: meshes += fnmatch.filter(l, "*." + fmt + cpr) des = os.path.join(self.__dest, "MESH") for m in l: if m in meshes: if sys.platform.startswith('win'): shutil.copy2(os.path.join(ref, m), os.path.join(des, m)) else: os.symlink(os.path.join(ref, m), os.path.join(des, m)) elif m != ".svn": t = os.path.join(ref, m) if os.path.isdir(t): shutil.copytree(t, os.path.join(des, m)) elif os.path.isfile(t): shutil.copy2(t, des) # Copy external scripts for post-processing ref = os.path.join(self.__repo, "POST") if os.path.isdir(ref): des = os.path.join(self.__dest, "POST") shutil.rmtree(des) shutil.copytree(ref, des, symlinks=True) # Change directory to destination directory os.chdir(self.__dest) log_lines = [] for c in self.cases: if not os.path.isdir(c.label): self.create_case(c, log_lines); else: if self.__force_rm == True: if self.__debug: print("Warning: case %s exists in the destination " "and will be overwritten." % c.label) # Build short path to RESU dir. such as 'CASE1/RESU' _dest_resu_dir = os.path.join(c.label, 'RESU') if os.path.isdir(_dest_resu_dir): shutil.rmtree(_dest_resu_dir) os.makedirs(_dest_resu_dir) else: if self.__debug: print("Warning: case %s exists in the destination. " "It won't be overwritten." % c.label) # if overwrite option enabled, overwrite content of DATA, SRC, SCRIPTS if self.__force_ow: dirs_to_overwrite = ["DATA", "SRC", "SCRIPTS"] self.overwriteDirectories(dirs_to_overwrite, case_label=c.label) # update path in gui script data_subdir = os.path.join(c.label, "DATA") c.update_gui_script_path(data_subdir, self.__dest, xmlonly=False) # update path in runcase script scripts_subdir = os.path.join(c.label, "SCRIPTS") c.update_runcase_path(scripts_subdir, self.__dest, xmlonly=False) os.chdir(repbase) if self.__force_ow: dirs_to_overwrite = ["POST", "MESH"] self.overwriteDirectories(dirs_to_overwrite) return log_lines #--------------------------------------------------------------------------- def overwriteDirectories(self, dirs_to_overwrite, case_label=""): """ Overwrite given directories in the Study tree. Label of case is an empty string by default. """ for _dir in dirs_to_overwrite: ref = os.path.join(self.__repo, case_label, _dir) if os.path.isdir(ref): dest = os.path.join(self.__dest, case_label, _dir) for _ref_dir, _dirs, _files in os.walk(ref): _dest_dir = _ref_dir.replace(ref, dest, 1) for _file in _files: _ref_file = os.path.join(_ref_dir, _file) _dest_file = os.path.join(_dest_dir, _file) if os.path.isfile(_dest_file): os.remove(_dest_file) shutil.copy2(_ref_file, _dest_dir) #--------------------------------------------------------------------------- def getRunDirectories(self): list_cases = [] list_dir = [] for case in self.cases: if case.is_run != "KO": list_cases.append(case.label) list_dir.append(case.run_dir) return " ".join(list_cases), " ".join(list_dir) #--------------------------------------------------------------------------- def disable_case(self, case): try: self.cases.remove(case) self.case_labels.remove(case.label) except Exception: pass msg = " - Case %s" % (case.label) if case.run_id != "": msg += ", run id %s" % (case.run_id) msg += " --> DISABLED" return msg #--------------------------------------------------------------------------- def needs_report_detailed(self, postpro): """ check if study needs a section in the detailed report (for figures, comparison or input) """ # study has figures or input figures needs = self.matplotlib_figures or self.input_figures for case in self.cases: if case.is_compare == "done": needs = True break # handle the input nodes that are inside case nodes if case.plot == "on" and case.is_run != "KO": nodes = self.__parser.getChildren(case.node, "input") if nodes: needs = True break # handle the input nodes that are inside postpro nodes if postpro: script, label, nodes, args = self.__parser.getPostPro(self.label) for i in range(len(label)): if script[i]: input_nodes = self.__parser.getChildren(nodes[i], "input") if input_nodes: needs = True break return needs #=============================================================================== # Studies class #=============================================================================== class Studies(object): """ Manage all Studies and all Cases described in the files of parameters. """ def __init__(self, pkg, options, exe, dif): """ Constructor. 1. create if necessary the destination directory, 2. initialize the parser and the plotter, 3. build the list of the studies, 4. start the report. @type options: C{Structure} @param options: structure the parameters options. @type exe: C{String} @param exe: name of the solver executable: C{code_saturne} or C{neptune_cfd}. @type dif: C{String} @param dif: name of the diff executable: C{cs_io_dump -d}. """ # try to determine if current directory is a study one cwd = os.getcwd() is_study = isStudy(cwd) studyd = None studyp = None if is_study: # default study directory is current one studyp = cwd # Create file of parameters filename = options.filename if options.create_xml and is_study: if filename == None: studyd = os.path.basename(studyp) filename = "smgr_" + studyd + ".xml" filepath = os.path.join(studyp, filename) smgr = create_base_xml_file(filepath) init_xml_file_with_study(smgr, studyp) elif options.create_xml and not is_study: msg = "Can not create XML file of parameter:\n" \ + "current directory is apparently not a study (no MESH directory).\n" sys.exit(msg) if filename == None: msg = "A file of parameters must be specified or created " \ + "for studymanager to run.\n" \ + "See help message and use '--file' or '--create-xml' option.\n" sys.exit(msg) # create a first xml parser only for # the repository verification and # the destination creation if os.path.isfile(filename): self.__parser = Parser(filename) else: msg = "Specified XML parameter file for studymanager does not exist.\n" sys.exit(msg) self.__xmlupdate = options.update_xml # set repository if len(options.repo_path) > 0: self.__parser.setRepository(options.repo_path) self.__repo = self.__parser.getRepository() if self.__repo: if not os.path.isdir(self.__repo): msg="Studies.__init__() >> self.__repo = {0}\n".format(self.__repo) sys.exit(msg+"Error: repository path is not valid.\n") else: # default value # if current directory is a study # set repository as directory containing the study if is_study: studyd = os.path.basename(studyp) self.__parser.setRepository(os.path.join(studyp,"..")) self.__repo = self.__parser.getRepository() else: msg = "Can not set a default repository directory:\n" \ + "current directory is apparently not a study (no MESH directory).\n" \ + "Add a repository path to the parameter file or use the command " \ + "line option (--repo=..).\n" sys.exit(msg) # set destination if self.__xmlupdate: self.__dest = self.__repo else: if len(options.dest_path) > 0: self.__parser.setDestination(options.dest_path) self.__dest = self.__parser.getDestination() if not self.__dest: # default value # if current directory is a study # set destination as a directory "../RUN_(study_name) if is_study and studyd != None: self.__parser.setDestination(os.path.join(studyp, "../RUN_"+studyd)) self.__dest = self.__parser.getDestination() else: msg = "Can not set a default destination directory:\n" \ + "current directory is apparently not a study (no MESH directory).\n" \ + "Add a destination path to the parameter file or use the command " \ + "line option (--dest=..).\n" sys.exit(msg) # create if necessary the destination directory if not os.path.isdir(self.__dest): os.makedirs(self.__dest) # copy the xml file of parameters for update and restart file = os.path.join(self.__dest, os.path.basename(filename)) try: shutil.copyfile(filename, file) except: pass # create a new parser, which is definitive and the plotter self.__parser = Parser(file) self.__parser.setDestination(self.__dest) self.__parser.setRepository(self.__repo) if options.debug: print(" Studies >> Repository >> ", self.__repo) print(" Studies >> Destination >> ", self.__dest) try: self.__plotter = Plotter(self.__parser) except Exception: self.__plotter = None # create list of restricting and excluding tags self.__with_tags = None if options.with_tags: with_tags = re.split(',', options.with_tags) self.__with_tags = [tag.strip() for tag in with_tags] self.__without_tags = None if options.without_tags: without_tags = re.split(',', options.without_tags) self.__without_tags = [tag.strip() for tag in without_tags] # build the list of the studies doc = os.path.join(self.__dest, options.log_file) self.__log = open(doc, "w") self.labels = self.__parser.getStudiesLabel() self.studies = [] for l in self.labels: self.studies.append( [l, Study(pkg, self.__parser, l, \ exe, dif, self.__log, \ options.n_procs, \ options.remove_existing, \ options.force_overwrite, \ self.__with_tags, \ self.__without_tags, \ options.debug)] ) if options.debug: print(" >> Append study ", l) # start the report self.report = os.path.join(self.__dest, "report.txt") self.reportFile = open(self.report, mode='w') self.reportFile.write('\n') # attributes self.__debug = options.debug self.__quiet = options.quiet self.__running = options.runcase self.__n_iter = options.n_iterations self.__compare = options.compare self.__ref = options.reference self.__postpro = options.post self.__default_fmt = options.default_fmt # do not use tex in matplotlib (built-in mathtext is used instead) self.__dis_tex = options.disable_tex # tex reports compilation with pdflatex self.__pdflatex = not options.disable_pdflatex # in case of restart iok = 0 for l, s in self.studies: for case in s.cases: if case.compute == 'on': iok+=1 if not iok: self.__running = False if self.__xmlupdate: os.remove(file) os.remove(doc) #--------------------------------------------------------------------------- def getDestination(self): """ @rtype: C{String} @return: destination directory of all studies. """ if self.__dest == None: msg=" cs_studymanager_study.py >> Studies.getDestination()" msg+=" >> self.__dest is not set" sys.exit(msg) return self.__dest #--------------------------------------------------------------------------- def getRepository(self): """ @rtype: C{String} @return: repository directory of all studies. """ if self.__repo == None: msg=" cs_studymanager_study.py >> Studies.getRepository()" msg+=" >> self.__repo is not set" sys.exit(msg) return self.__repo #--------------------------------------------------------------------------- def reporting(self, msg, stdout=True, report=True, status=False): """ Write message on standard output and/or in report. @type l: C{String} @param l: the sentence to be written. """ s = "" if not status: s = chr(10) if stdout and not self.__quiet: sys.stdout.write (msg + chr(13) + s) sys.stdout.flush() if report: self.reportFile.write(msg + '\n') self.reportFile.flush() #--------------------------------------------------------------------------- def report_action_location(self, header_msg, destination=True): """ Add a message to report/stdout at head of sections specifying if action is performed in dest or repo. """ if destination: header_msg = header_msg + " (in destination)" else: header_msg = header_msg + " (in repository)" self.reporting(header_msg) #--------------------------------------------------------------------------- def updateRepository(self, xml_only=False): """ Update all studies and all cases. """ for l, s in self.studies: self.reporting(' o Update repository: ' + l) for case in s.cases: self.reporting(' - update %s' % case.label) case.update(xml_only) self.reporting('') #--------------------------------------------------------------------------- def create_studies(self): """ Create all studies and all cases. """ for l, s in self.studies: create_msg = " o Create study " + l dest = True self.report_action_location(create_msg, dest) log_lines = s.create_cases() for line in log_lines: self.reporting(line) self.reporting('') #--------------------------------------------------------------------------- def test_compilation(self): """ Compile sources of all runs with compute attribute at on. """ iko = 0 for l, s in self.studies: # build study dir. (in repo.) study_path = os.path.join(self.__repo, l) self.reporting(' o Compile study: ' + l) for case in s.cases: if case.compute == 'on': # test compilation (logs are redirected to smgr log file) is_compiled = case.test_compilation(study_path, self.__log) # report if is_compiled == "OK": self.reporting(' - compile %s --> OK' % case.label) elif is_compiled == "KO": self.reporting(' - compile %s --> FAILED' % case.label) iko+=1 self.reporting('') if iko: self.reporting('Error: compilation failed for %s case(s).\n' % iko) sys.exit(1) #--------------------------------------------------------------------------- def prepro(self, l, s, case): """ Launch external additional scripts with arguments. """ pre, label, nodes, args = self.__parser.getPrepro(case.node) if self.__debug: print(" >> prepro ", pre) print(" >> label ", label) print(" >> nodes ", nodes) print(" >> args ", args) for i in range(len(label)): if pre[i]: # search if the script is in the MESH directory # if not, the script is searched in the directories # of the current case cmd = os.path.join(self.__dest, l, "MESH", label[i]) if self.__debug: print("Path to prepro script ", cmd) if not os.path.isfile(cmd): filePath = "" for root, dirs, fs in os.walk(os.path.join(self.__dest, l, case.label)): if label[i] in fs: filePath = root break cmd = os.path.join(filePath, label[i]) if os.path.isfile(cmd): sc_name = os.path.basename(cmd) # ensure script is executable set_executable(cmd) cmd += " " + args[i] cmd += " -c " + os.path.join(self.__dest, l, case.label) repbase = os.getcwd() os.chdir(os.path.join(self.__dest, l, "MESH")) # Prepro external script often need install python directory # and package python directory: code_saturne or neptune_cfd p_dir = case.pkg.get_dir('pythondir') pkg_dir = case.pkg.get_dir('pkgpythondir') p_dirs = p_dir + ":" + pkg_dir # if package is neptune_cfd, prepro script often needs # code_saturne package python directory cs_pkg_dir = None if case.pkg.name == 'neptune_cfd': cs_pkg_dir = os.path.join(pkg_dir, '../code_saturne') cs_pkg_dir = os.path.normpath(cs_pkg_dir) p_dirs = p_dirs + ":" + cs_pkg_dir retcode, t = run_studymanager_command(cmd, self.__log, pythondir = p_dirs) stat = "FAILED" if retcode != 0 else "OK" os.chdir(repbase) self.reporting(' - script %s --> %s (%s s)' % (stat, sc_name, t), stdout=True, report=False) self.reporting(' - script %s --> %s (%s s)' % (stat, cmd, t), stdout=False, report=True) else: self.reporting(' - script %s not found' % cmd) #--------------------------------------------------------------------------- def run(self): """ Update and run all cases. Warning, if the markup of the case is repeated in the xml file of parameters, the run of the case is also repeated. """ for l, s in self.studies: self.reporting(" o Prepro scripts and runs for study: " + l) for case in s.cases: self.prepro(l, s, case) if self.__running: if case.compute == 'on' and case.is_compiled != "KO": if self.__n_iter is not None: if case.subdomains: case_dir = os.path.join(self.__dest, s.label, case.label, case.subdomains[0], "DATA") else: case_dir = os.path.join(self.__dest, s.label, case.label, "DATA") os.chdir(case_dir) # Create a control_file in each case DATA if not os.path.exists('control_file'): control_file = open('control_file','w') control_file.write("time_step_limit " + str(self.__n_iter) + "\n") # Flush to ensure that control_file content is seen # when control_file is copied to the run directory on all systems control_file.flush() control_file.close self.reporting(' - running %s ...' % case.label, stdout=True, report=False, status=True) error = case.run() if case.is_time: is_time = "%s s" % case.is_time else: is_time = "existed already" if not error: if not case.run_id: self.reporting(" - run %s --> Warning suffix" " is not read" % case.label) self.reporting(' - run %s --> OK (%s) in %s' \ % (case.label, \ is_time, \ case.run_id)) self.__parser.setAttribute(case.node, "compute", "off") # update dest="" attribute n1 = self.__parser.getChildren(case.node, "compare") n2 = self.__parser.getChildren(case.node, "script") n3 = self.__parser.getChildren(case.node, "data") n4 = self.__parser.getChildren(case.node, "probe") n5 = self.__parser.getChildren(case.node, "resu") n6 = self.__parser.getChildren(case.node, "input") for n in n1 + n2 + n3 + n4 + n5 + n6: if self.__parser.getAttribute(n, "dest") == "": self.__parser.setAttribute(n, "dest", case.run_id) else: if not case.run_id: self.reporting(' - run %s --> FAILED (%s)' \ % (case.label, is_time)) else: self.reporting(' - run %s --> FAILED (%s) in %s' \ % (case.label, \ is_time, \ case.run_id)) self.__log.flush() self.reporting('') #--------------------------------------------------------------------------- def check_compare(self, destination=True): """ Check coherency between xml file of parameters and repository. Stop if you try to make a comparison with a file which does not exist. """ for l, s in self.studies: check_msg = " o Check compare of study: " + l self.report_action_location(check_msg, destination) # reference directory passed in studymanager command line overwrites # destination in all cases (even if compare is defined by a compare # markup with a non empty destination) ref = None if self.__ref: ref = os.path.join(self.__ref, s.label) cases_to_disable = [] for case in s.cases: if case.compare == 'on' and case.is_run != "KO": compare, nodes, repo, dest, threshold, args = self.__parser.getCompare(case.node) if compare: is_checked = False for i in range(len(nodes)): if compare[i]: is_checked = True if destination == False: dest[i]= None msg = case.check_dirs(nodes[i], repo[i], dest[i], reference=ref) if msg: self.reporting(msg) cases_to_disable.append(case) if not compare or not is_checked: node = None repo = "" dest = "" if destination == False: dest = None msg = case.check_dirs(node, repo, dest, reference=ref) if msg: self.reporting(msg) cases_to_disable.append(case) for case in cases_to_disable: msg = s.disable_case(case) self.reporting(msg) self.reporting('') #--------------------------------------------------------------------------- def compare_case_and_report(self, case, repo, dest, threshold, args, reference=None): """ Compare the results for one computation and report """ case.is_compare = "done" diff_value, m_size_eq = case.runCompare(self, repo, dest, threshold, args, reference=reference) case.diff_value += diff_value case.m_size_eq = case.m_size_eq and m_size_eq if args: s_args = 'with args: %s' % args else: s_args = 'default mode' if not m_size_eq: self.reporting(' - compare %s (%s) --> DIFFERENT MESH SIZES FOUND' % (case.label, s_args)) elif diff_value: self.reporting(' - compare %s (%s) --> DIFFERENCES FOUND' % (case.label, s_args)) else: self.reporting(' - compare %s (%s) --> NO DIFFERENCES FOUND' % (case.label, s_args)) #--------------------------------------------------------------------------- def compare(self): """ Compare the results of the new computations with those from the Repository. """ if self.__compare: for l, s in self.studies: self.reporting(' o Compare study: ' + l) # reference directory passed in studymanager command line overwrites # destination in all cases (even if compare is defined by a # compare markup with a non empty destination) ref = None if self.__ref: ref = os.path.join(self.__ref, s.label) for case in s.cases: if case.compare == 'on' and case.is_run != "KO": is_compare, nodes, repo, dest, t, args = self.__parser.getCompare(case.node) if is_compare: for i in range(len(nodes)): if is_compare[i]: self.compare_case_and_report(case, repo[i], dest[i], t[i], args[i], reference=ref) if not is_compare or case.is_compare != "done": repo = "" dest = "" t = None args = None self.compare_case_and_report(case, repo, dest, t, args, reference=ref) self.reporting('') #--------------------------------------------------------------------------- def check_script(self, destination=True): """ Check coherency between xml file of parameters and repository. Stop if you try to run a script with a file which does not exist. """ scripts_checked = False for l, s in self.studies: # search for scripts to check before check_scripts = False for case in s.cases: script, label, nodes, args, repo, dest = \ self.__parser.getScript(case.node) if nodes: check_scripts = True break if not check_scripts: continue # if scripts have to be checked check_msg = " o Check scripts of study: " + l self.report_action_location(check_msg, destination) scripts_checked = True cases_to_disable = [] for case in s.cases: script, label, nodes, args, repo, dest = \ self.__parser.getScript(case.node) for i in range(len(nodes)): if script[i] and case.is_run != "KO": if destination == False: dest[i] = None msg = case.check_dirs(nodes[i], repo[i], dest[i]) if msg: self.reporting(msg) cases_to_disable.append(case) for case in cases_to_disable: msg = s.disable_case(case) self.reporting(msg) if scripts_checked: self.reporting('') return scripts_checked #--------------------------------------------------------------------------- def scripts(self): """ Launch external additional scripts with arguments. """ for l, s in self.studies: self.reporting(" o Run scripts of study: " + l) for case in s.cases: script, label, nodes, args, repo, dest = self.__parser.getScript(case.node) for i in range(len(label)): if script[i] and case.is_run != "KO": cmd = os.path.join(self.__dest, l, "POST", label[i]) if os.path.isfile(cmd): sc_name = os.path.basename(cmd) # ensure script is executable set_executable(cmd) cmd += " " + args[i] if repo[i]: r = os.path.join(self.__repo, l, case.label, "RESU", repo[i]) cmd += " -r " + r if dest[i]: d = os.path.join(self.__dest, l, case.label, "RESU", dest[i]) cmd += " -d " + d retcode, t = run_studymanager_command(cmd, self.__log) stat = "FAILED" if retcode != 0 else "OK" self.reporting(' - script %s --> %s (%s s)' % (stat, sc_name, t), stdout=True, report=False) self.reporting(' - script %s --> %s (%s s)' % (stat, cmd, t), stdout=True, report=False) else: self.reporting(' - script %s not found' % cmd) self.reporting('') #--------------------------------------------------------------------------- def postpro(self): """ Launch external additional scripts with arguments. """ for l, s in self.studies: # fill results directories and ids for the cases of the current study # that were not run by the current studymanager command for case in s.cases: if case.is_run != "KO": if case.run_dir == "": resu = os.path.join(self.__dest, l, case.label, case.resu) rep, msg = case.check_dir(None, resu, "", "dest") if msg: self.reporting(msg) s.disable_case(case) else: case.run_id = rep case.run_dir = os.path.join(resu, rep) script, label, nodes, args = self.__parser.getPostPro(l) if not label: continue self.reporting(' o Postprocessing cases of study: ' + l) for i in range(len(label)): if script[i]: cmd = os.path.join(self.__dest, l, "POST", label[i]) if os.path.isfile(cmd): sc_name = os.path.basename(cmd) # ensure script is executable set_executable(cmd) list_cases, list_dir = s.getRunDirectories() cmd += ' ' + args[i] + ' -c "' + list_cases + '" -d "' \ + list_dir + '" -s ' + l self.reporting(' - running postpro %s' % sc_name, stdout=True, report=False, status=True) retcode, t = run_studymanager_command(cmd, self.__log) stat = "FAILED" if retcode != 0 else "OK" self.reporting(' - postpro %s --> %s (%s s)' \ % (stat, sc_name, t), stdout=True, report=False) self.reporting(' - postpro %s --> %s (%s s)' \ % (stat, cmd, t), stdout=False, report=True) else: self.reporting(' - postpro %s not found' % cmd) self.reporting('') #--------------------------------------------------------------------------- def check_data(self, case, destination=True): """ Check coherency between xml file of parameters and repository for data markups of a run. """ for node in self.__parser.getChildren(case.node, "data"): plots, file, dest, repo = self.__parser.getResult(node) if destination == False: dest = None msg = case.check_dirs(node, repo, dest) if msg: self.reporting(msg) return False return True #--------------------------------------------------------------------------- def check_probes(self, case, destination=True): """ Check coherency between xml file of parameters and repository for probes markups of a run. """ for node in self.__parser.getChildren(case.node, "probes"): file, dest, fig = self.__parser.getProbes(node) if destination == False: dest = None repo = None msg = case.check_dirs(node, repo, dest) if msg: self.reporting(msg) return False return True #--------------------------------------------------------------------------- def check_input(self, case, destination=True): """ Check coherency between xml file of parameters and repository for probes markups of a run. """ for node in self.__parser.getChildren(case.node, "input"): file, dest, repo, tex = self.__parser.getInput(node) if destination == False: dest = None msg = case.check_dirs(node, repo, dest) if msg: self.reporting(msg) return False return True #--------------------------------------------------------------------------- def check_plots_and_input(self, destination=True): """ Check coherency between xml file of parameters and repository. Stop if you try to make a plot of a file which does not exist. """ for l, s in self.studies: check_msg = " o Check plots and input of study: " + l self.report_action_location(check_msg, destination) cases_to_disable = [] for case in s.cases: if case.plot == "on" and case.is_run != "KO": if not self.check_data(case, destination): cases_to_disable.append(case) continue if not self.check_probes(case, destination): cases_to_disable.append(case) continue if not self.check_input(case, destination): cases_to_disable.append(case) continue for case in cases_to_disable: msg = s.disable_case(case) self.reporting(msg) self.reporting('') #--------------------------------------------------------------------------- def plot(self): """ Plot data. """ if self.__plotter: for l, s in self.studies: self.reporting(' o Plot study: ' + l) self.__plotter.plot_study(l, s, self.__dis_tex, self.__default_fmt) self.reporting('') #--------------------------------------------------------------------------- def report_input(self, doc2, i_nodes, s_label, c_label=None): """ Add input to report detailed. """ for i_node in i_nodes: f, dest, repo, tex = self.__parser.getInput(i_node) doc2.appendLine("\\subsubsection{%s}" % f) if dest: d = dest dd = self.__dest elif repo: d = repo dd = self.__repo else: d = "" dd = "" if c_label: ff = os.path.join(dd, s_label, c_label, 'RESU', d, f) else: ff = os.path.join(dd, s_label, 'POST', d, f) if not os.path.isfile(ff): print("\n\nWarning: this file does not exist: %s\n\n" % ff) elif ff[-4:] in ('.png', '.jpg', '.pdf') or ff[-5:] == '.jpeg': doc2.addFigure(ff) elif tex == 'on': doc2.addTexInput(ff) else: doc2.addInput(ff) #--------------------------------------------------------------------------- def build_reports(self, report1, report2): """ @type report1: C{String} @param report1: name of the global report. @type report2: C{String} @param report2: name of the detailed report. @rtype: C{List} of C{String} @return: list of file to be attached to the report. """ attached_files = [] # First global report doc1 = Report1(self.__dest, report1, self.__log, self.report, self.__parser.write(), self.__pdflatex) for l, s in self.studies: for case in s.cases: if case.diff_value or not case.m_size_eq: is_nodiff = "KO" else: is_nodiff = "OK" doc1.add_row(l, case.label, case.is_compiled, case.is_run, case.is_time, case.is_compare, is_nodiff) attached_files.append(doc1.close()) # Second detailed report if self.__compare or self.__postpro: doc2 = Report2(self.__dest, report2, self.__log, self.__pdflatex) for l, s in self.studies: if not s.needs_report_detailed(self.__postpro): continue doc2.appendLine("\\section{%s}" % l) if s.matplotlib_figures or s.input_figures: doc2.appendLine("\\subsection{Graphical results}") for g in s.matplotlib_figures: doc2.addFigure(g) for g in s.input_figures: doc2.addFigure(g) for case in s.cases: if case.is_compare == "done": run_id = None if case.run_id != "": run_id = case.run_id doc2.appendLine("\\subsection{Comparison for case " "%s (run_id: %s)}" % (case.label, run_id)) if not case.m_size_eq: doc2.appendLine("Repository and destination " "have apparently not been run " "with the same mesh (sizes do " "not match).") elif case.diff_value: doc2.add_row(case.diff_value, l, case.label) elif self.__compare: doc2.appendLine("No difference between the " "repository and the " "destination.") # handle the input nodes that are inside case nodes if case.plot == "on" and case.is_run != "KO": nodes = self.__parser.getChildren(case.node, "input") if nodes: doc2.appendLine("\\subsection{Results for " "case %s}" % case.label) self.report_input(doc2, nodes, l, case.label) # handle the input nodes that are inside postpro nodes if self.__postpro: script, label, nodes, args = self.__parser.getPostPro(l) needs_pp_input = False for i in range(len(label)): if script[i]: input_nodes = \ self.__parser.getChildren(nodes[i], "input") if input_nodes: needs_pp_input = True break if needs_pp_input: doc2.appendLine("\\subsection{Results for " "post-processing cases}") for i in range(len(label)): if script[i]: input_nodes = \ self.__parser.getChildren(nodes[i], "input") if input_nodes: self.report_input(doc2, input_nodes, l) attached_files.append(doc2.close()) return attached_files #--------------------------------------------------------------------------- def getlabel(self): return self.labels #--------------------------------------------------------------------------- def logs(self): try: self.reportFile.close() except: pass self.reportFile = open(self.report, mode='r') return self.reportFile.read() #--------------------------------------------------------------------------- def __del__(self): try: self.__log.close() except: pass try: self.reportFile.close() except: pass #-------------------------------------------------------------------------------