#/*########################################################################## # Copyright (C) 2001-2013 European Synchrotron Radiation Facility # # PyHST2 # European Synchrotron Radiation Facility, Grenoble,France # # PyHST2 is developed at # the ESRF by the Scientific Software staff. # Principal author for PyHST2: Alessandro Mirone. # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation; either version 2 of the License, or (at your option) # any later version. # # PyHST2 is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # PyHST2; if not, write to the Free Software Foundation, Inc., 59 Temple Place, # Suite 330, Boston, MA 02111-1307, USA. # # PyHST2 follows the dual licensing model of Trolltech's Qt and Riverbank's PyQt # and cannot be used as a free plugin for a non-free program. # # Please contact the ESRF industrial unit (industry@esrf.fr) if this license # is a problem for you. #############################################################################*/ """ CODE INVOCATION: There are two distinct cases which are dealt differently * You are in a OAR_ environment. (OAR is a versatile resource and task manager) * You dont have a resource allocation system, but you know which hosts you can use The details for both cases are given below. An important information to retain is that the resource granularity is one cpu with its eventually associated GPU. * Things are extremely easy within an OAR, or SLURM environment : * PyHST2_XX input.par where input.par is the input file, XX is the version number * Resources allocations * OAR * provided that your OAR request demanded an integer number of CPUs like this for an interactive job : oarsub -p"gpu='YES' " -l"nodes=1,walltime=2:20:00" -I or using more evoluted, eventually non interactive, requests as thouroughly explained here : http://wikiserv.esrf.fr/software/index.php/Main_Page In any case alway request an integer number of CPUs ( a CPU has several cores ) * SLURM * INTERACTIVE : witha the actual configuration at ESRF this request takes a whole node with gpu, because all nodes have 2 GPUS we ask 2 process per node. Infact PyHST2 is organised in such a way that each (multithreaded ) process uses a GPU :: salloc -p gpu --exclusive --nodes=1 --tasks-per-node=2 --time=200 --gres=gpu:2 srun --pty bash Then once you are in the command shell give the simple command PyHST2_XX input.par . Two process will be runned using the allocated gpus. If you dont take the whole node, remember to ask a bunch of cpus. Each process needs some cpus to be efficient. You can add the SLURM directive :: --cpus-per-task=N with N greater than one. A task is a process, if you use GPUS each allocated GPU will match a process * BATCH : to obtain the same result as in the interactive example you can *sbatch myscript* where script is the following script :: #!/bin/bash #SBATCH --nodes=1 #SBATCH --exclusive #SBATCH --tasks-per-node=2 #SBATCH --time=200 #SBATCH --gres=gpu:2 #SBATCH -p gpu PyHST2_XXXX input.par * Things are no more difficult when you have to manually specify the resources you want. If on your machine neither OAR nor SLURM exist, this will be the way to go. Alternatively, if SLURM or OAR are installedm you can manually disable the OAR/SLURM features of PyHST2 by setting :: export PYHSTNOSCHEDULER=YES Then you can run PyHST : * PyHST2_XX input.par gpuA,1,0,gpuB,1,0 in this example gpuA and gpuB are the two hosts where you are running PyHST. Each host has two CPUs, the first of which will be associated with GPU number 1 and the second with GPU number 2 * PyHST2_XX input.par hostname_with_CPUonly,-1 In this example the -1 flag indicates that no GPUs have to be used * You must have created beforehand a file *machinefile*. The content of which will be, for the first example:: gpuA gpuA gpuB gpuB in other words machinefile file contains one instance of the hostname per each cpu. A process will be spawned per each cpu * NOTICE for parallelism : on some cluster things may get complicated with docker being present as an example which creates virtual network interfaces that must be avoided. To avoid an interface name docker0 use this variable in the input file (IF as InterFace):: IF_EXCLUDE = docker0 if instead you know the name of realiable interface, for example eth1, use this:: IF_INCLUDE = eth1 """ from __future__ import absolute_import from __future__ import division from __future__ import print_function import string import sys if(sys.argv[0][-12:]!="sphinx-build"): from . import string_six from . import __init__ from .__init__ import version as myversion import os import subprocess as sub import sys import glob # import mpi4py.MPI as MPI ### strange but I need to have mpi4py already loaded before fabio or on infiniband it crashes ## import fabio import tempfile ####################### # only needed in ff_matcher # import scipy # import scipy.ndimage import numpy as np ShellTrue=False def get_SCHEDULER_TYPE(): SCHEDULER_TYPE = None if( os.getenv("PYHSTNOSCHEDULER") is not None): print(" PYHSTNOSCHEDULER has been set ") return SCHEDULER_TYPE try: args=["oarprint", "host", "-P", "host,cpu", "-F", "'% %'" ] if (sys.version_info >= (3, 7)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue) elif (sys.version_info > (3, 0)): p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue) else: p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue) resources, errors = p.communicate() print(" RESOURCES ", resources ) if len(errors)==0 and len(resources): SCHEDULER_TYPE = "OAR" except: pass if SCHEDULER_TYPE is None: args=["set" ] if (sys.version_info >= (3, 7)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=True) elif (sys.version_info > (3, 0)): p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=True) else: p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=True) resources, errors = p.communicate() if "SLURM_JOB_ID" in resources: SCHEDULER_TYPE = "SLURM" return SCHEDULER_TYPE def get_NPROCS( stype ): if stype is None: return get_NPROCS_nosched() elif stype == "OAR": return get_NPROCS_oar() elif stype == "SLURM": return get_NPROCS_slurm() else: raise Exception(" unrecognised SCHEDULER_TYPE ") def get_NPROCS_nosched(): machinefile_name="machinefile" if os.path.exists(machinefile_name): nprocs=0 f=open("machinefile","r") for line in f: if line.strip()!="": nprocs+=1 f=None else: nprocs=1 open("machinefile", "w").write("localhost slots=1\n"*nprocs) rshagent="ssh" return nprocs, machinefile_name, rshagent def get_NPROCS_oar(): args=["oarprint", "host", "-P", "host,cpu", "-F", "'% %'" ] if (sys.version_info >= (3, 7)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue) elif (sys.version_info > (3, 0)): p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue) else: p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue) resources, errors = p.communicate() resources_L = resources.split( "\n" ) print( "RISORSE ") print( "/", resources,"/", len(resources)) jobid=os.getenv("OAR_JOB_ID") machinefile_name = "machinefile_OAR" + str(jobid) f=open(machinefile_name, "w") nprocs=0 for l in resources_L: if len(l)==0: continue if l[0]==l[-1] and l[0] in ["'", '"']: l=l[1:-1] l=l.strip() node, cpus = l.split( " ") ncpus=len(cpus.split(",")) if "htc" in node: ncpus=1 f.write(("%s slots=1\n"%node)*ncpus) nprocs=nprocs+ncpus f.close() rshagent="oarsh" return nprocs, machinefile_name, rshagent def get_NPROCS_slurm(): machinefile_name = None rshagent = None nprocs = int( os.getenv("SLURM_NPROCS") ) return nprocs, machinefile_name, rshagent def get_GPUS_STRING( stype ): if stype is None: return "" elif stype=="OAR" : return get_GPUS_STRING_oar() elif stype=="SLURM": return get_GPUS_STRING_slurm() else: raise Exception(" unrecognised SCHEDULER_TYPE ") def get_GPUS_STRING_oar(): args=["oarprint", "host", "-P", "host,gpu_num", "-F", "'% %'" ] if (sys.version_info >= (3, 7)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue) elif (sys.version_info > (3, 0)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue) else: p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue) resources, errors = p.communicate() print( " resources gpu ", resources) if(len(resources)): resources_L = resources.split( "\n" ) gpus_string=" " for l in resources_L: if len(l)==0: continue if l[0]==l[-1] and l[0] in ["'", '"']: l=l[1:-1] l=l.strip() try: node, gpus = l.split( " ") except: node = l gpus = "-1" gpus_string = gpus_string +node+","+gpus+"," gpus_string=gpus_string[:-1] else: # la linea gpu-string sara presa dalla linea di comando ( viene aggiunto sys.argv[2] ) gpus_string="" return gpus_string def get_GPUS_STRING_slurm(): # SLURM_NODELIST='gpu2-[0101-0102]' # SLURM_JOB_GPUS='0,1' gpus = os.getenv( "GPU_DEVICE_ORDINAL" ) if gpus is None: return "" nlstring = os.getenv( "SLURM_NODELIST" ) args=["scontrol","show","hostname", nlstring] print(" COMANDI ", args) if (sys.version_info >= (3, 7)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue) elif (sys.version_info > (3, 0)): p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue) else: p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue) resources, errors = p.communicate() resources=resources.split("\n") print(" resources ", resources ) result = "" for hn in resources: if hn=="": continue if result != "": result = result +"," result = result +hn+","+gpus return result def ff_matcher(ff_name, imf_name, mff_name , span ): ff = fabio.open(ff_name ,"r").data.astype("d") ima = fabio.open(imf_name,"r").data.astype("d") ff = scipy.ndimage.filters.gaussian_filter(ff, 2 , mode='nearest') ima = scipy.ndimage.filters.gaussian_filter(ima, 2 , mode='nearest') maxerr = None Gshiftx = 0 for shiftx in np.arange( -span,span,span/20.01 ): ff_shifted = scipy.ndimage.interpolation.shift(ff, [0,shiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True) res = np.gradient(ima/ff_shifted) error = np.linalg.norm(res) if maxerr is None: maxerr = error Gshiftx = shiftx elif error < maxerr: maxerr = error Gshiftx = shiftx print( " Gshiftx = " , Gshiftx) Gshifty = 0 for shifty in np.arange( -span,span,span/20.01 ): ff_shifted = scipy.ndimage.interpolation.shift(ff, [shifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True) res = np.gradient(ima/ff_shifted) error = np.linalg.norm(res) if error < maxerr: maxerr = error Gshifty = shifty print( " Gshifty = " , Gshifty) if 1: span = span/10.0 for shiftx in np.arange( Gshiftx -span,Gshiftx + span,span/20.01 ): ff_shifted = scipy.ndimage.interpolation.shift(ff, [Gshifty,shiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True) res = np.gradient(ima/ff_shifted) error = np.linalg.norm(res) if error < maxerr: maxerr = error Gshiftx = shiftx print( " Gshiftx = " , Gshiftx) for shifty in np.arange(Gshifty -span, Gshifty + span,span/20.01 ): ff_shifted = scipy.ndimage.interpolation.shift(ff, [shifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True) res = np.gradient(ima/ff_shifted) error = np.linalg.norm(res) if error < maxerr: maxerr = error Gshifty = shifty print( " Gshifty = " , Gshifty) ff = fabio.open(ff_name ,"r").data print( " OPTIMAL SHIFT ", Gshifty, Gshifty) new_ff = scipy.ndimage.interpolation.shift(ff, [Gshifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True) newf = fabio.edfimage.EdfImage() newf.data = new_ff newf.write(mff_name) dirname=os.path.dirname(os.path.abspath(__file__)) SCHEDULER_TYPE = None # ["SLURM", "OAR" ] SCHEDULER_TYPE = get_SCHEDULER_TYPE() print(" SCHEDULER TYPE is ", SCHEDULER_TYPE ) ########################################################################################### ### HOW MANY PROCESSES ? print("HOW MANY PROCESSES " ) nprocs, machinefile_name, rshagent = get_NPROCS( SCHEDULER_TYPE) print("HOW MANY PROCESSES ", nprocs ) gpus_string = get_GPUS_STRING( SCHEDULER_TYPE ) print(" GPU STRING IS ", gpus_string ) def get_setted_multipag_option(optname,files_to_cancel, tmpdir ): setted_res= False if optname in P.MULTI_PAGANIN_PARS: val_res = P.MULTI_PAGANIN_PARS[optname] setted_res = True else: tmp = tempfile.NamedTemporaryFile(dir=tmpdir,suffix=".vol", delete=False) val_res = tmp.name tmp.close() files_to_cancel.append(val_res) return setted_res, val_res def callpyhst(inputfile, mpiextra): if P.DO_V3D_UNSHARP : args= [ "unsharp3D_2_"+myversion ] + [inputfile]+sys.argv[2:] seguito = " %s "*len(sys.argv[1:]) comando = ("%s "+seguito) % tuple(args) print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: if SCHEDULER_TYPE != "SLURM": args= [mpi_extra,rshagent, machinefile_name, nprocs, "pyhst_2_"+myversion ] + [inputfile]+sys.argv[2:] seguito = " %s "*len(sys.argv[1:]) comando = ("mpirun %s "%mpiextra+" %s --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) print( comando +" " +gpus_string) os.system( "for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) if P.DOUBLERUN4DFF: print(" SECOND RUN FOR DOUBLE FF ") os.system( "for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: args= [nprocs, "pyhst_2_"+myversion ] + [inputfile]+sys.argv[2:] seguito = " %s "*(len(args)-2) # comando = ("mpirun %s "%mpiextra+" -n %d %s "+seguito) % tuple(args) comando = ("mpirun %s "%mpiextra+" %s "+seguito) % tuple(list(args)[1:]) os.system( comando+" " +gpus_string) if P.DOUBLERUN4DFF: print(" SECOND RUN FOR DOUBLE FF ") os.system( comando+" " +gpus_string) from . import mpistatus mpistatus.status=0 from . import Parameters_module if(sys.argv[0][-12:]!="sphinx-build"): from .Parameters_module import Parameters as P # mpi_extra=" --map-by slot " mpi_extra=" --byslot " comando = 'mpirun -V' if (sys.version_info >= (3, 7)): p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE, text=True) elif (sys.version_info > (3, 0)): p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True) else: p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE) msg,errors= p1.communicate() if " 2." in msg or " 3." in msg or " 4." in msg: mpi_extra=" --bind-to none --map-by slot " if P.IF_EXCLUDE is not None: mpi_extra+=" -mca btl_tcp_if_exclude %s " % P.IF_EXCLUDE if P.IF_INCLUDE is not None: mpi_extra+=" -mca btl_tcp_if_include %s " % P.IF_INCLUDE inputfile = sys.argv[1] if P.JP2EDF_DIR is not None: if SCHEDULER_TYPE != "SLURM": args= [mpi_extra, rshagent,machinefile_name, nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["jp2edf"]+[P.ncpus]+sys.argv[2:] seguito = " %s "*(len(args)-5) comando = ("mpirun %s --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: npp = 4 if SCHEDULER_TYPE == "SLURM": npp = 1 args= [ nprocs*npp,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["jp2edf"]+[P.ncpus]+sys.argv[2:] seguito = " %s "*(len(args)-2) comando = ("mpirun -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system( comando+" " +gpus_string) jpfilename = os.path.join (os.path.dirname(inputfile),"jp2edf_"+os.path.basename(inputfile)) +".par" f=open(jpfilename,"w") f.write(open(inputfile,"r").read()) f.write("\nFILE_PREFIX = %s \n" % P.FILE_PREFIX ) f.close() inputfile = jpfilename if P.FF_MATCHER_SPAN : # ff_filename = "ff_matched_"+inputfile ff_filename = os.path.join (os.path.dirname(inputfile),"ff_matched_"+os.path.basename(inputfile)) f=open(ff_filename,"w") f.write(open(inputfile,"r").read()) ffprefix = "matched_"+os.path.basename(P.FF_PREFIX ) f.write("\nFF_PREFIX =%s \n" % ffprefix) f.close() ffs = glob.glob( P.FF_PREFIX+"*" ) for ff in ffs: mff = "matched_"+os.path.basename(ff) imf = glob.glob( P.FILE_PREFIX +"*" +P.FILE_POSTFIX )[0] ff_matcher(ff, imf, mff, P.FF_MATCHER_SPAN ) inputfile = ff_filename if P.MULTI_PAGANIN_PARS is None: callpyhst( inputfile, mpi_extra ) else: steps = [1,1,1,1,1,1,1] files_to_cancel = [] tmpdir="/tmp/" if "TMPDIR" in P.MULTI_PAGANIN_PARS: tmpdir = P.MULTI_PAGANIN_PARS["TMPDIR"]+"/" if "STEPS" in P.MULTI_PAGANIN_PARS: steps = P.MULTI_PAGANIN_PARS["STEPS"] pagbone_file_manualset , pagbone_outputfile = get_setted_multipag_option("PAGBONE_FILE", files_to_cancel , tmpdir) maskbone_file_manualset, maskbone_outputfile= get_setted_multipag_option("MASKBONE_FILE", files_to_cancel, tmpdir ) absbone_file_manualset , absbone_outputfile = get_setted_multipag_option("ABSBONE_FILE", files_to_cancel , tmpdir) corrbone_file_manualset , corrbone_outputfile= get_setted_multipag_option("CORRBONE_FILE", files_to_cancel , tmpdir) corrprefix_manualset , corrprefix= get_setted_multipag_option("CORRPROJSPATTERN", files_to_cancel , tmpdir) if not os.path.exists(os.path.dirname(corrprefix) ): os.makedirs(os.path.dirname(corrprefix) ) if corrprefix_manualset==0 and corrprefix[:5]=="/tmp/": if(os.path.exists(corrprefix)): os.remove(corrprefix) files_to_cancel[-1] = corrprefix+"/proj_*" os.mkdir(corrprefix) corrpattern = corrprefix+"/proj_*" corrprefix = corrprefix+"/proj_%05d.edf" correctedvol_manualset , correctedvol_outputfile= get_setted_multipag_option("CORRECTEDVOL_FILE", files_to_cancel , tmpdir) if steps[0]: mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_0.par" f=open(mp_filename,"w") f.write(open(inputfile,"r").read()) f.write("\nPAGANIN_Lmicron = %e \n" % P.MULTI_PAGANIN_PARS["BONE_Lmicron"] ) f.write("OUTPUT_FILE = %s \n" % pagbone_outputfile ) f.close() callpyhst( mp_filename, mpi_extra ) if steps[1]: if not steps[0]: if not pagbone_file_manualset: msg="ERROR : To start from a previous PAGBONE file you must provide the PAGBONE_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) # threshold, remplacement = P.MULTI_PAGANIN_PARS["THRESHOLD"] threshold = P.MULTI_PAGANIN_PARS["THRESHOLD"] if "DILATE" in P.MULTI_PAGANIN_PARS: dilatation = P.MULTI_PAGANIN_PARS["DILATE"] else: dilatation = 0 medianR = P.MULTI_PAGANIN_PARS["MEDIANR"] if SCHEDULER_TYPE != "SLURM": args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + ["segmenta", pagbone_outputfile, threshold, medianR ,maskbone_outputfile, dilatation ]+sys.argv[2:] seguito = " %s "*(len(args)-5) comando = ("mpirun %s --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: args= [ nprocs, "pyhst_segment2_"+myversion ] + ["segmenta", pagbone_outputfile, threshold, medianR ,maskbone_outputfile, dilatation ]+sys.argv[2:] seguito = " %s "*(len(args)-2) comando = (("mpirun %s"%mpi_extra) + " -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system( comando+" " +gpus_string) if steps[2]: mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_2.par" # mp_filename = "multipaganin_"+inputfile+"_2.par" f=open(mp_filename,"w") f.write(open(inputfile,"r").read()) f.write("\nDO_PAGANIN = 0 \n" ) f.write("OUTPUT_FILE = %s \n" % absbone_outputfile ) f.close() callpyhst( mp_filename, mpi_extra ) if steps[3]: if not steps[1]: if not maskbone_file_manualset: msg="ERROR : To start from a previous MASKBONE file you must provide the MASKBONE_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) if not steps[2]: if not absbone_file_manualset: msg="ERROR : To start from a previous ABSBONE file you must provide the ABSBONE_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) # threshold, remplacement = P.MULTI_PAGANIN_PARS["THRESHOLD"] threshold = P.MULTI_PAGANIN_PARS["THRESHOLD"] if P.VERBOSITY>2 : print( " QUI DEVO spillare ", absbone_outputfile,maskbone_outputfile, corrbone_outputfile) remplacement = 0.0 if "REPLACEMENT" in P.MULTI_PAGANIN_PARS: remplacement =P.MULTI_PAGANIN_PARS["REPLACEMENT"] if SCHEDULER_TYPE != "SLURM": args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + ["spilla",pagbone_outputfile, absbone_outputfile,maskbone_outputfile, remplacement, corrbone_outputfile]+sys.argv[2:] seguito = " %s "*(len(args)-5) comando = ("mpirun %s --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>2 : print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: args= [ nprocs, "pyhst_segment2_"+myversion ] + ["spilla",pagbone_outputfile, absbone_outputfile,maskbone_outputfile, remplacement, corrbone_outputfile]+sys.argv[2:] seguito = " %s "*(len(args)-2) comando = ("mpirun -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>2 : print( comando +" " +gpus_string) os.system(comando+" " +gpus_string) if not absbone_file_manualset and os.path.exists(absbone_outputfile): os.remove(absbone_outputfile) if steps[4]: if not steps[3]: if not corrbone_file_manualset: msg="ERROR : To start from a previous CORRBONE file you must provide the CORRBONE_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) # mp_filename = "multipaganin_"+inputfile+"_4.par" mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_4.par" f=open(mp_filename,"w") f.write(open(inputfile,"r").read()) f.write("\nSTEAM_INVERTER = 1 \n" ) f.write("PROJ_OUTPUT_FILE = %s \n" % corrprefix ) f.write("PROJ_OUTPUT_RESTRAINED = 1 \n" ) f.write("OUTPUT_FILE = %s \n" % corrbone_outputfile ) f.close() callpyhst( mp_filename , mpi_extra) if not corrbone_file_manualset and os.path.exists(corrbone_outputfile): os.remove(corrbone_outputfile) if steps[5]: if not steps[4]: if not corrprefix_manualset: msg="ERROR : To start from a previous CORRPROJSPATTERN file you must provide the CORRPROJSPATTERN in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) # mp_filename = "multipaganin_"+inputfile+"_5.par" mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_5.par" f=open(mp_filename,"w") f.write(open(inputfile,"r").read()) # f.write("PAGANIN_Lmicron = %e \n" % P.MULTI_PAGANIN_PARS["BONE_Lmicron"] ) f.write("\nCORR_PROJ_FILE = %s \n" % corrprefix ) f.write("OUTPUT_FILE = %s \n" % correctedvol_outputfile ) f.close() callpyhst( mp_filename , mpi_extra) if not corrprefix_manualset : if corrprefix_manualset==0 and corrprefix[:5]=="/tmp/": files = glob.glob(corrpattern) for file in files: os.remove(file) if steps[6]: if not steps[5]: if not correctedvol_manualset: msg="ERROR : To start from a previous CORRECTEDVOL file you must provide the CORRECTEDVOL_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) if not steps[0]: if not pagbone_file_manualset: msg="ERROR : To start from a previous PAGBONE file you must provide the PAGBONE_FILE filename in parameters MULTI_PAGANIN_PARS" print( msg) raise Exception( msg) if P.VERBOSITY>2 : print( " QUI DEVO UNIRE ", pagbone_outputfile,maskbone_outputfile, correctedvol_outputfile, P.OUTPUT_FILE) if SCHEDULER_TYPE != "SLURM": args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + [ "incolla", pagbone_outputfile,maskbone_outputfile, correctedvol_outputfile, P.OUTPUT_FILE]+sys.argv[2:] seguito = " %s "*(len(args)-5) comando = ("mpirun %s --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>2 :print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: args= [ nprocs, "pyhst_segment2_"+myversion ] + [ "incolla", pagbone_outputfile,maskbone_outputfile, correctedvol_outputfile, P.OUTPUT_FILE]+sys.argv[2:] seguito = " %s "*(len(args)-2) comando = ("mpirun -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>2 :print( comando +" " +gpus_string) os.system( comando+" " +gpus_string) ##mp_filename = "multipaganin_"+inputfile+"_1.par" for pat in files_to_cancel: if P.VERBOSITY>2 :print( " REMOVING ", pat) files = glob.glob(pat) for file in files: os.remove(file) if P.JP2EDF_DIR and P.JP2EDF_REMOVE: if SCHEDULER_TYPE != "SLURM": args= [mpi_extra, rshagent,machinefile_name, nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["removejp2"]+sys.argv[2:] seguito = " %s "*(len(args)-5) comando = ("mpirun %s -mca btl_tcp_if_exclude docker0 --mca mpi_warn_on_fork 0 --mca orte_rsh_agent %s -machinefile %s -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string) else: args= [ nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["removejp2"]+sys.argv[2:] seguito = " %s "*(len(args)-2) comando = ("mpirun -n %d %s "+seguito) % tuple(args) if P.VERBOSITY>1 : print( comando +" " +gpus_string) os.system(comando+" " +gpus_string)