#/*##########################################################################
# Copyright (C) 2001-2013 European Synchrotron Radiation Facility
#
#              PyHST2  
#  European Synchrotron Radiation Facility, Grenoble,France
#
# PyHST2 is  developed at
# the ESRF by the Scientific Software  staff.
# Principal author for PyHST2: Alessandro Mirone.
#
# This program is free software; you can redistribute it and/or modify it 
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option) 
# any later version.
#
# PyHST2 is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# PyHST2; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA 02111-1307, USA.
#
# PyHST2 follows the dual licensing model of Trolltech's Qt and Riverbank's PyQt
# and cannot be used as a free plugin for a non-free program. 
#
# Please contact the ESRF industrial unit (industry@esrf.fr) if this license 
# is a problem for you.
#############################################################################*/
"""  CODE INVOCATION:
  There are two distinct cases which are dealt differently

      * You are in a OAR_ environment. (OAR is a versatile resource and task manager)
      * You dont have a resource allocation system, but you know which hosts you can use

  The details for both cases are given below. An important information to retain is that 
  the resource granularity is one cpu with its eventually associated GPU.

      * Things are extremely easy within an OAR, or SLURM  environment :
            * PyHST2_XX input.par
                where input.par is the input file, XX is the version number

            * Resources allocations

               * OAR 

                    * provided that your OAR request demanded an integer number of CPUs
                      like this for an interactive job :
                        oarsub  -p"gpu='YES' " -l"nodes=1,walltime=2:20:00"  -I
                      or using more evoluted, eventually  non interactive, requests as thouroughly explained here :  http://wikiserv.esrf.fr/software/index.php/Main_Page 
                      In any case alway request an integer number of CPUs ( a CPU has several cores )
               * SLURM 

                   * INTERACTIVE : witha the actual configuration at ESRF this request takes a whole node with gpu,
                     because all nodes have 2 GPUS we ask 2 process per node. Infact PyHST2 is  organised in such
                     a way that each (multithreaded ) process uses a GPU  ::

                           salloc  -p gpu --exclusive --nodes=1 --tasks-per-node=2 --time=200 --gres=gpu:2  srun --pty bash
                    
                     Then once you are in the command shell give the simple command PyHST2_XX input.par . Two process will be runned using the 
                     allocated gpus. If you dont take the whole node,  remember to ask a bunch of cpus. Each process needs some cpus to be efficient.
                     You can add the SLURM directive ::

                               --cpus-per-task=N
                        
                     with N greater than one. A task is a process, if you use GPUS each allocated GPU will match a process
 

                   * BATCH : to obtain the same result as in the interactive example you can *sbatch myscript* where script is the following script ::

                        #!/bin/bash
                        #SBATCH --nodes=1
                        #SBATCH  --exclusive 
                        #SBATCH --tasks-per-node=2
                        #SBATCH --time=200
                        #SBATCH --gres=gpu:2  
                        #SBATCH -p gpu
                        PyHST2_XXXX input.par


      * Things are no more difficult  when you have to manually specify the resources you want. 
        If on your machine neither OAR nor SLURM exist, this will be the way to go.
        Alternatively, if SLURM or OAR are installedm  you can manually disable the OAR/SLURM features of PyHST2 
        by setting ::
         
             export PYHSTNOSCHEDULER=YES

        Then you can run PyHST :
        * PyHST2_XX input.par gpuA,1,0,gpuB,1,0
            in this example gpuA and gpuB are the two hosts where you are running PyHST.
            Each host has two CPUs, the first of which will be associated with GPU number 1 and the second
            with GPU number 2
        *  PyHST2_XX input.par hostname_with_CPUonly,-1
            In this example the -1 flag indicates that no GPUs have to be used

        * You must have created beforehand a file *machinefile*. The content of which will be, for the first example::

                gpuA 
                gpuA 
                gpuB 
                gpuB 

          in other words machinefile file contains one instance of the hostname per each cpu. A process will be spawned
          per each cpu

          * NOTICE for parallelism :  on some cluster things may get complicated with docker being present  as an example
            which creates virtual network interfaces that must be avoided.
            To avoid an interface name docker0 use this variable in the input file (IF as InterFace)::

                  IF_EXCLUDE = docker0

            if instead you know the name of realiable interface, for example eth1, use this::

                  IF_INCLUDE = eth1 

"""

 
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function


import string
import sys

if(sys.argv[0][-12:]!="sphinx-build"):

    from .  import string_six
    from . import __init__
    from .__init__ import version as myversion 



import os
import subprocess as sub
import sys
import glob

# import mpi4py.MPI as MPI   ### strange but I need to have mpi4py already loaded before fabio or on infiniband it crashes

## import fabio
import tempfile



#######################
# only needed in ff_matcher
# import scipy
# import scipy.ndimage

import numpy as np

ShellTrue=False

def get_SCHEDULER_TYPE():

    SCHEDULER_TYPE = None
    if( os.getenv("PYHSTNOSCHEDULER") is not None):
        print(" PYHSTNOSCHEDULER has been set ")
        return SCHEDULER_TYPE

    try:
        args=["oarprint",  "host", "-P",  "host,cpu",  "-F",  "'% %'" ]
        if (sys.version_info >= (3, 7)):
            p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue)
        elif (sys.version_info > (3, 0)):
            p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue)
        else:
            p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue)
        resources, errors = p.communicate()     
        print(" RESOURCES ", resources )

        if len(errors)==0 and len(resources):
            SCHEDULER_TYPE = "OAR"
    except:
        pass

    if SCHEDULER_TYPE is None:
        
        args=["set" ]
        
        if (sys.version_info >= (3, 7)):
            p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=True)
        elif (sys.version_info > (3, 0)):
            p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=True)
        else:
            p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=True)

        resources, errors = p.communicate()

        if "SLURM_JOB_ID" in resources:
            SCHEDULER_TYPE = "SLURM"
            
    return SCHEDULER_TYPE




def get_NPROCS( stype ):
    if stype is None:
        return get_NPROCS_nosched()
    elif stype  == "OAR":
        return get_NPROCS_oar()
    elif stype  == "SLURM":
        return get_NPROCS_slurm()
    else:
        raise Exception(" unrecognised SCHEDULER_TYPE ")

    
def get_NPROCS_nosched():
    machinefile_name="machinefile"
    if os.path.exists(machinefile_name):
        nprocs=0
        f=open("machinefile","r")
        for line in  f:
            if line.strip()!="":
                nprocs+=1
        f=None
    else:
        nprocs=1
        open("machinefile", "w").write("localhost   slots=1\n"*nprocs)
    rshagent="ssh"
    
    return nprocs, machinefile_name, rshagent

def get_NPROCS_oar():
    args=["oarprint",  "host", "-P",  "host,cpu",  "-F",  "'% %'" ]
    if (sys.version_info >= (3, 7)):
        p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue)
    elif (sys.version_info > (3, 0)):
        p = sub.Popen(args=args, stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue)
    else:
        p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue)
    resources, errors = p.communicate()
     
    resources_L = resources.split( "\n"  )

    print( "RISORSE ")
    print( "/", resources,"/", len(resources))

    jobid=os.getenv("OAR_JOB_ID")

    machinefile_name = "machinefile_OAR" + str(jobid)

    f=open(machinefile_name, "w")

    nprocs=0
    for l in resources_L:
        if len(l)==0:
            continue
        if l[0]==l[-1] and l[0] in ["'", '"']:
            l=l[1:-1]
        l=l.strip()

        node, cpus  =  l.split( " ")

        ncpus=len(cpus.split(","))
        if "htc"  in node:
            ncpus=1
        
        f.write(("%s  slots=1\n"%node)*ncpus)
        nprocs=nprocs+ncpus

    f.close()

    rshagent="oarsh" 
    
    return nprocs, machinefile_name, rshagent


def get_NPROCS_slurm():
    machinefile_name = None
    rshagent = None
    nprocs   = int( os.getenv("SLURM_NPROCS")  )
    return nprocs, machinefile_name, rshagent
        



def get_GPUS_STRING( stype  ):
    if stype is None:
        return ""
    elif stype=="OAR" :
        return get_GPUS_STRING_oar()
    elif stype=="SLURM":
        return get_GPUS_STRING_slurm()
    else:
        raise Exception(" unrecognised SCHEDULER_TYPE ")

    
def get_GPUS_STRING_oar():
 args=["oarprint",  "host", "-P",  "host,gpu_num",  "-F",  "'% %'" ]
 if (sys.version_info >= (3, 7)):    
     p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue)
 elif (sys.version_info > (3, 0)):
     p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue)
 else:
     p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue)
 resources, errors = p.communicate()
 print( " resources gpu ", resources)
 if(len(resources)):
    resources_L = resources.split( "\n"  )
    gpus_string=" "
    for l in resources_L:
        if len(l)==0:
            continue
        if l[0]==l[-1] and l[0] in ["'", '"']:
            l=l[1:-1]
        l=l.strip()
        try:
            node, gpus  =  l.split( " ")
        except:
            node = l
            gpus = "-1"
        gpus_string = gpus_string +node+","+gpus+"," 
    gpus_string=gpus_string[:-1]
 else:
    # la linea gpu-string sara presa dalla linea di comando  ( viene aggiunto sys.argv[2] )
    gpus_string=""
 return gpus_string


  
def get_GPUS_STRING_slurm():
    # SLURM_NODELIST='gpu2-[0101-0102]'
    # SLURM_JOB_GPUS='0,1'
    gpus = os.getenv( "GPU_DEVICE_ORDINAL" )
    if gpus is None:
        return ""
    nlstring = os.getenv( "SLURM_NODELIST" )
    args=["scontrol","show","hostname",  nlstring]
    print(" COMANDI ",  args)
    
    if (sys.version_info >= (3, 7)):    
        p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, text=True, shell=ShellTrue)
    elif (sys.version_info > (3, 0)):
        p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True, shell=ShellTrue)
    else:
        p = sub.Popen(args=args ,stdout=sub.PIPE,stderr=sub.PIPE, shell=ShellTrue)
    resources, errors = p.communicate()
    resources=resources.split("\n")
    print(" resources ", resources )
    
    result = ""
    for hn in resources:
        if hn=="":
            continue
        if result != "":
            result = result +","
        result = result +hn+","+gpus

  
    return result
    




def ff_matcher(ff_name, imf_name, mff_name , span ):
    ff = fabio.open(ff_name ,"r").data.astype("d")
    ima = fabio.open(imf_name,"r").data.astype("d")
    
    ff = scipy.ndimage.filters.gaussian_filter(ff, 2 ,  mode='nearest')
    ima = scipy.ndimage.filters.gaussian_filter(ima, 2 ,  mode='nearest')

    maxerr = None

    Gshiftx = 0
    for shiftx in np.arange( -span,span,span/20.01 ):
        ff_shifted = scipy.ndimage.interpolation.shift(ff, [0,shiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True)
        res = np.gradient(ima/ff_shifted)
        error = np.linalg.norm(res)
        if maxerr is None:
            maxerr = error
            Gshiftx = shiftx
        elif error < maxerr:
            maxerr =  error
            Gshiftx = shiftx

        print( " Gshiftx = " , Gshiftx)
            

    Gshifty = 0
    for shifty in np.arange( -span,span,span/20.01 ):
        ff_shifted = scipy.ndimage.interpolation.shift(ff, [shifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True)
        res = np.gradient(ima/ff_shifted)
        error = np.linalg.norm(res)
        if error < maxerr:
            maxerr =  error
            Gshifty = shifty
        print( " Gshifty = " , Gshifty)


    if 1:    
     span = span/10.0

     for shiftx in np.arange( Gshiftx -span,Gshiftx + span,span/20.01 ):

         ff_shifted = scipy.ndimage.interpolation.shift(ff, [Gshifty,shiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True)
         res = np.gradient(ima/ff_shifted)
         error = np.linalg.norm(res)
         if error < maxerr:
             maxerr =  error
             Gshiftx = shiftx
         print( " Gshiftx = " , Gshiftx)



     for shifty in np.arange(Gshifty -span, Gshifty +  span,span/20.01 ):
         ff_shifted = scipy.ndimage.interpolation.shift(ff, [shifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True)
         res = np.gradient(ima/ff_shifted)
         error = np.linalg.norm(res)
         if error < maxerr:
             maxerr =  error
             Gshifty = shifty
         print( " Gshifty = " , Gshifty)

            
    ff = fabio.open(ff_name ,"r").data
    print( " OPTIMAL SHIFT ", Gshifty, Gshifty)
    new_ff = scipy.ndimage.interpolation.shift(ff, [Gshifty,Gshiftx], output=None, order=1, mode='nearest', cval=0.0, prefilter=True)
    newf = fabio.edfimage.EdfImage()
    newf.data = new_ff
    newf.write(mff_name)




dirname=os.path.dirname(os.path.abspath(__file__))

SCHEDULER_TYPE = None # ["SLURM", "OAR" ]
SCHEDULER_TYPE = get_SCHEDULER_TYPE()

print(" SCHEDULER TYPE is ",  SCHEDULER_TYPE  ) 

###########################################################################################
### HOW MANY PROCESSES ?

print("HOW MANY PROCESSES " ) 


nprocs,  machinefile_name, rshagent =   get_NPROCS( SCHEDULER_TYPE)

print("HOW MANY PROCESSES ", nprocs ) 

gpus_string  = get_GPUS_STRING( SCHEDULER_TYPE  )

print(" GPU STRING IS ", gpus_string )


def get_setted_multipag_option(optname,files_to_cancel, tmpdir ):
    setted_res=   False
    if optname  in P.MULTI_PAGANIN_PARS:
        val_res = P.MULTI_PAGANIN_PARS[optname]
        setted_res = True
    else:
        tmp = tempfile.NamedTemporaryFile(dir=tmpdir,suffix=".vol", delete=False)
        val_res = tmp.name
        tmp.close()
        files_to_cancel.append(val_res)
    return setted_res, val_res




def callpyhst(inputfile, mpiextra):
    
    if P.DO_V3D_UNSHARP :
        args= [ "unsharp3D_2_"+myversion ] + [inputfile]+sys.argv[2:]
        seguito = " %s "*len(sys.argv[1:])
        comando = ("%s "+seguito) % tuple(args)
        print( comando +" " +gpus_string)
        os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
    
    else:
        if SCHEDULER_TYPE != "SLURM":
            args= [mpi_extra,rshagent, machinefile_name, nprocs, "pyhst_2_"+myversion ] + [inputfile]+sys.argv[2:]
            seguito = " %s "*len(sys.argv[1:])
            comando = ("mpirun %s "%mpiextra+"  %s     --mca mpi_warn_on_fork 0  --mca orte_rsh_agent %s  -machinefile %s  -n %d     %s "+seguito) % tuple(args)            
            print( comando +" " +gpus_string)
            os.system(  "for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " +   comando+" " +gpus_string)
            if P.DOUBLERUN4DFF:
                print(" SECOND RUN FOR DOUBLE FF ")
                os.system(  "for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " +   comando+" " +gpus_string)

        else:
            args= [nprocs, "pyhst_2_"+myversion ] + [inputfile]+sys.argv[2:]
            seguito = " %s "*(len(args)-2)
            # comando = ("mpirun  %s "%mpiextra+"  -n %d     %s "+seguito) % tuple(args)
            
            comando = ("mpirun   %s "%mpiextra+" %s  "+seguito) % tuple(list(args)[1:])
            os.system(    comando+" " +gpus_string)
            if P.DOUBLERUN4DFF:
                print(" SECOND RUN FOR DOUBLE FF ")
                os.system(    comando+" " +gpus_string)
            

from . import mpistatus
mpistatus.status=0
from . import Parameters_module
            
if(sys.argv[0][-12:]!="sphinx-build"):
    from  .Parameters_module import Parameters as P
    
    # mpi_extra="  --map-by slot "
    mpi_extra="  --byslot "
    comando = 'mpirun -V'

    if (sys.version_info >= (3, 7)):
        
        p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE, text=True)
        
    elif (sys.version_info > (3, 0)):
      
        p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE, universal_newlines=True)
     
    else:
        p1 = sub.Popen(args= comando.split( " ") ,stdin=sub.PIPE,stdout=sub.PIPE,stderr=sub.PIPE)

        
    msg,errors=  p1.communicate()
    if " 2." in msg or " 3." in msg or " 4." in msg:
        mpi_extra=" --bind-to none  --map-by slot "
    
                   
    if P.IF_EXCLUDE is not None:
        mpi_extra+="  -mca btl_tcp_if_exclude  %s " % P.IF_EXCLUDE
    if P.IF_INCLUDE is not None:
        mpi_extra+="  -mca btl_tcp_if_include  %s " % P.IF_INCLUDE
    
    
    inputfile =  sys.argv[1]
    
    if P.JP2EDF_DIR is not None:

        if SCHEDULER_TYPE != "SLURM":
            args= [mpi_extra, rshagent,machinefile_name, nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["jp2edf"]+[P.ncpus]+sys.argv[2:] 
            seguito = " %s "*(len(args)-5)
            comando = ("mpirun   %s   --mca mpi_warn_on_fork 0  --mca orte_rsh_agent %s  -machinefile %s  -n %d      %s "+seguito) % tuple(args)
            if P.VERBOSITY>1 : print( comando +" " +gpus_string)
            os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
        else:
            npp = 4
            if SCHEDULER_TYPE == "SLURM":
                    npp = 1
            args= [ nprocs*npp,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["jp2edf"]+[P.ncpus]+sys.argv[2:]
            seguito = " %s "*(len(args)-2)
            comando = ("mpirun   -n %d     %s "+seguito) % tuple(args)
            if P.VERBOSITY>1 : print( comando +" " +gpus_string)
            os.system(  comando+" " +gpus_string)
            
        jpfilename = os.path.join (os.path.dirname(inputfile),"jp2edf_"+os.path.basename(inputfile)) +".par"

        f=open(jpfilename,"w")
        f.write(open(inputfile,"r").read())
        f.write("\nFILE_PREFIX  = %s \n" % P.FILE_PREFIX  )
        f.close()

        inputfile = jpfilename
        
    if P.FF_MATCHER_SPAN :
        # ff_filename = "ff_matched_"+inputfile

        ff_filename = os.path.join (os.path.dirname(inputfile),"ff_matched_"+os.path.basename(inputfile)) 

        
        f=open(ff_filename,"w")
        f.write(open(inputfile,"r").read())
        ffprefix = "matched_"+os.path.basename(P.FF_PREFIX )
        f.write("\nFF_PREFIX  =%s \n" %  ffprefix)
        f.close()
        
        ffs = glob.glob( P.FF_PREFIX+"*" )
        for ff in ffs:
         mff = "matched_"+os.path.basename(ff)
         imf = glob.glob( P.FILE_PREFIX +"*" +P.FILE_POSTFIX )[0]

         ff_matcher(ff, imf, mff, P.FF_MATCHER_SPAN  )

         
        inputfile = ff_filename
        
     

    if P.MULTI_PAGANIN_PARS is None:
        callpyhst( inputfile, mpi_extra )
    else:
        steps = [1,1,1,1,1,1,1]

        files_to_cancel = []

        tmpdir="/tmp/"
        if "TMPDIR" in P.MULTI_PAGANIN_PARS:
            tmpdir = P.MULTI_PAGANIN_PARS["TMPDIR"]+"/"

        if "STEPS" in P.MULTI_PAGANIN_PARS:
            steps = P.MULTI_PAGANIN_PARS["STEPS"]

        pagbone_file_manualset ,  pagbone_outputfile = get_setted_multipag_option("PAGBONE_FILE", files_to_cancel , tmpdir) 
        maskbone_file_manualset,  maskbone_outputfile= get_setted_multipag_option("MASKBONE_FILE", files_to_cancel, tmpdir ) 
        absbone_file_manualset ,  absbone_outputfile = get_setted_multipag_option("ABSBONE_FILE", files_to_cancel , tmpdir) 
        corrbone_file_manualset , corrbone_outputfile= get_setted_multipag_option("CORRBONE_FILE", files_to_cancel , tmpdir) 
        corrprefix_manualset , corrprefix= get_setted_multipag_option("CORRPROJSPATTERN", files_to_cancel , tmpdir)
        
        if not os.path.exists(os.path.dirname(corrprefix)  ):
            os.makedirs(os.path.dirname(corrprefix) )
            
        if corrprefix_manualset==0 and corrprefix[:5]=="/tmp/":
            if(os.path.exists(corrprefix)):
                os.remove(corrprefix)
            files_to_cancel[-1] = corrprefix+"/proj_*"
            os.mkdir(corrprefix)
            corrpattern =  corrprefix+"/proj_*"
            corrprefix = corrprefix+"/proj_%05d.edf"

        correctedvol_manualset , correctedvol_outputfile= get_setted_multipag_option("CORRECTEDVOL_FILE", files_to_cancel , tmpdir) 
 

        if steps[0]:
            mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_0.par"

  
            f=open(mp_filename,"w")
            f.write(open(inputfile,"r").read())
            f.write("\nPAGANIN_Lmicron  = %e \n" % P.MULTI_PAGANIN_PARS["BONE_Lmicron"]  )
            f.write("OUTPUT_FILE  = %s \n" %  pagbone_outputfile  )
            f.close()
            callpyhst( mp_filename, mpi_extra )
            
        if steps[1]:
            if not steps[0]:
                if not pagbone_file_manualset:
                    msg="ERROR : To start from a previous PAGBONE file you must provide the  PAGBONE_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
              
            # threshold, remplacement =  P.MULTI_PAGANIN_PARS["THRESHOLD"]
            threshold =  P.MULTI_PAGANIN_PARS["THRESHOLD"]
            if "DILATE" in P.MULTI_PAGANIN_PARS:
                dilatation = P.MULTI_PAGANIN_PARS["DILATE"]
            else:
                dilatation = 0  
            medianR = P.MULTI_PAGANIN_PARS["MEDIANR"]

            if SCHEDULER_TYPE != "SLURM":
                args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + ["segmenta", pagbone_outputfile, threshold, medianR ,maskbone_outputfile, dilatation ]+sys.argv[2:]
                seguito = " %s "*(len(args)-5)
                comando = ("mpirun  %s   --mca mpi_warn_on_fork 0   --mca orte_rsh_agent %s  -machinefile %s  -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>1 : print( comando +" " +gpus_string)
                os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
            else:
                args= [ nprocs, "pyhst_segment2_"+myversion ] + ["segmenta", pagbone_outputfile, threshold, medianR ,maskbone_outputfile, dilatation ]+sys.argv[2:]
                seguito = " %s "*(len(args)-2)
                comando = (("mpirun %s"%mpi_extra) + "  -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>1 : print( comando +" " +gpus_string)
                os.system( comando+" " +gpus_string)
                    
        if steps[2]:
            mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_2.par"
            # mp_filename = "multipaganin_"+inputfile+"_2.par"
            f=open(mp_filename,"w")
            f.write(open(inputfile,"r").read())
            f.write("\nDO_PAGANIN  = 0  \n"  )
            f.write("OUTPUT_FILE  = %s \n" % absbone_outputfile   )
            f.close()
            callpyhst( mp_filename, mpi_extra )

        if steps[3]:
            if not steps[1]:
                if not maskbone_file_manualset:
                    msg="ERROR : To start from a previous MASKBONE file you must provide the  MASKBONE_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
            if not steps[2]:
                if not absbone_file_manualset:
                    msg="ERROR : To start from a previous ABSBONE file you must provide the  ABSBONE_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
              
            # threshold, remplacement =  P.MULTI_PAGANIN_PARS["THRESHOLD"]
            threshold =  P.MULTI_PAGANIN_PARS["THRESHOLD"]
            if P.VERBOSITY>2 : print( " QUI DEVO spillare  ",  absbone_outputfile,maskbone_outputfile,  corrbone_outputfile)
            remplacement = 0.0
            if "REPLACEMENT" in P.MULTI_PAGANIN_PARS:
                remplacement =P.MULTI_PAGANIN_PARS["REPLACEMENT"]

            if SCHEDULER_TYPE != "SLURM":
                args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + ["spilla",pagbone_outputfile, absbone_outputfile,maskbone_outputfile, remplacement, corrbone_outputfile]+sys.argv[2:]
                seguito = " %s "*(len(args)-5)
                comando = ("mpirun  %s  --mca mpi_warn_on_fork 0   --mca orte_rsh_agent %s  -machinefile %s  -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>2 : print( comando +" " +gpus_string)
                os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
            else:
                args= [ nprocs, "pyhst_segment2_"+myversion ] + ["spilla",pagbone_outputfile, absbone_outputfile,maskbone_outputfile, remplacement, corrbone_outputfile]+sys.argv[2:]
                seguito = " %s "*(len(args)-2)
                comando = ("mpirun   -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>2 : print( comando +" " +gpus_string)
                os.system(comando+" " +gpus_string)
                
            
        if not absbone_file_manualset  and os.path.exists(absbone_outputfile):
            os.remove(absbone_outputfile)
            

        if steps[4]:
            if not steps[3]:
                if not corrbone_file_manualset:
                    msg="ERROR : To start from a previous CORRBONE file you must provide the  CORRBONE_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
            # mp_filename = "multipaganin_"+inputfile+"_4.par"
            mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_4.par"
            
            f=open(mp_filename,"w")
            f.write(open(inputfile,"r").read())
            f.write("\nSTEAM_INVERTER = 1  \n"  )
            f.write("PROJ_OUTPUT_FILE = %s \n" %  corrprefix  )
            f.write("PROJ_OUTPUT_RESTRAINED = 1 \n" )

            f.write("OUTPUT_FILE  = %s \n" % corrbone_outputfile   )
            f.close()
            callpyhst( mp_filename , mpi_extra)

        if not corrbone_file_manualset  and os.path.exists(corrbone_outputfile):
            os.remove(corrbone_outputfile)

        if steps[5]:
            if not steps[4]:
                if not corrprefix_manualset:
                    msg="ERROR : To start from a previous CORRPROJSPATTERN file you must provide the  CORRPROJSPATTERN  in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
            # mp_filename = "multipaganin_"+inputfile+"_5.par"
            mp_filename = os.path.join (os.path.dirname(inputfile),"multipaganin_"+os.path.basename(inputfile)) +"_5.par"
            
            f=open(mp_filename,"w")
            f.write(open(inputfile,"r").read())
            #  f.write("PAGANIN_Lmicron  = %e \n" % P.MULTI_PAGANIN_PARS["BONE_Lmicron"]  )
            f.write("\nCORR_PROJ_FILE = %s \n" %  corrprefix  )
            f.write("OUTPUT_FILE  = %s \n" %  correctedvol_outputfile  )
            f.close()
            callpyhst( mp_filename , mpi_extra)


            if not corrprefix_manualset  :
                if corrprefix_manualset==0 and corrprefix[:5]=="/tmp/":
                    files = glob.glob(corrpattern)
                    for file in files:
                        os.remove(file)


        if steps[6]:
            if not steps[5]:
                if not correctedvol_manualset:
                    msg="ERROR : To start from a previous CORRECTEDVOL file you must provide the  CORRECTEDVOL_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)

            if not steps[0]:
                if not pagbone_file_manualset:
                    msg="ERROR : To start from a previous PAGBONE file you must provide the  PAGBONE_FILE filename in parameters MULTI_PAGANIN_PARS"
                    print( msg)
                    raise Exception( msg)
              
            if P.VERBOSITY>2 : print( " QUI DEVO UNIRE   ",  pagbone_outputfile,maskbone_outputfile,  correctedvol_outputfile, P.OUTPUT_FILE)

            if SCHEDULER_TYPE != "SLURM":
                args= [mpi_extra,rshagent,machinefile_name, nprocs, "pyhst_segment2_"+myversion ] + [ "incolla", pagbone_outputfile,maskbone_outputfile,  correctedvol_outputfile, P.OUTPUT_FILE]+sys.argv[2:]
                seguito = " %s "*(len(args)-5)
                comando = ("mpirun  %s   --mca mpi_warn_on_fork 0  --mca orte_rsh_agent %s  -machinefile %s  -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>2 :print( comando +" " +gpus_string)
                os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
            else:                
                args= [ nprocs, "pyhst_segment2_"+myversion ] + [ "incolla", pagbone_outputfile,maskbone_outputfile,  correctedvol_outputfile, P.OUTPUT_FILE]+sys.argv[2:]
                seguito = " %s "*(len(args)-2)
                comando = ("mpirun  -n %d     %s "+seguito) % tuple(args)
                if P.VERBOSITY>2 :print( comando +" " +gpus_string)
                os.system( comando+" " +gpus_string)
            


            ##mp_filename = "multipaganin_"+inputfile+"_1.par"


        for pat in files_to_cancel:
            if P.VERBOSITY>2 :print( " REMOVING ", pat)
            files = glob.glob(pat)
            for file in files:
                os.remove(file)


    if P.JP2EDF_DIR and P.JP2EDF_REMOVE:

        if SCHEDULER_TYPE != "SLURM":
            args= [mpi_extra, rshagent,machinefile_name, nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["removejp2"]+sys.argv[2:]
            seguito = " %s "*(len(args)-5)
            comando = ("mpirun  %s   -mca btl_tcp_if_exclude docker0   --mca mpi_warn_on_fork 0  --mca orte_rsh_agent %s  -machinefile %s  -n %d     %s "+seguito) % tuple(args)
            if P.VERBOSITY>1 : print( comando +" " +gpus_string)
            os.system("for nvar in `awk 'BEGIN{for(v in ENVIRON) print v}' | grep MPI` ; do unset $nvar ; done ; " + comando+" " +gpus_string)
        else:
            args= [ nprocs,"pyhst_jp2edf2_"+myversion ]+[inputfile]+["removejp2"]+sys.argv[2:]
            seguito = " %s "*(len(args)-2)
            comando = ("mpirun  -n %d     %s "+seguito) % tuple(args)
            if P.VERBOSITY>1 : print( comando +" " +gpus_string)
            os.system(comando+" " +gpus_string)