#! /bin/sh
if [ "$MPIR_HOME" = "" ] ; then 
    MPIR_HOME=#MPIR_HOME#
fi
if [ "$MPIR_HOME" = "#""MPIR_HOME""#" ] ; then
    MPIR_HOME=`pwd`/..
fi
if [ "#MPIRUN_BIN#" = "#""MPIRUN_BIN""#" ] ; then 
    MPIRUN_HOME=$MPIR_HOME/bin
else
    MPIRUN_HOME=$MPIR_HOME/#MPIRUN_BIN#
fi
if [ "$argsset" = "" ] ; then
   . $MPIRUN_HOME/mpirun.args
   argsset=1
fi
# This if for the ANL SP1/2, using the ANL "spsubmit" program.
# Users of "loadleveler" will probably want something like this
#
# One special feature is that this script can use a pre-existing
# partition if one is present.  It uses the environment variable
# MPI_PARTITION to indicate a file that contains a list of nodes to
# use, one per line.  If this file is found, then the partition is
# assumed to already be available.
#
# This is the copy program
spxcp=$MPIR_HOME/lib/rs6000/ch_mpl/spxcp
#
# Check to see if we need to move the file
firstchar=`expr "$localpgm" : '\(.\)'`
if [ $mvhome = 0 -a "$firschar" != "/" ] ; then
    # May need to set mvhome.  This looks for /bonnie or /clyde in the
    # current directory
    curdir=`pwd | sed -e 's%/tmp_mnt%%g' -e 's%/Net%%g'`
    curname6=`expr "$curdir" : '/\(......\)'`
    curname5=`expr "$curdir" : '/\(.....\)'`
    if [ "$curname6" != "bonnie" -a "$curname5" != "clyde" ] ; then
	mvhome=1
	echo "Setting -mvhome for you (executable appears to not be"
	echo "in /sphome file system)."
    fi
fi
#
# Create a new file.  Also, make sure that 
# path is absolute; users can be surprised by things in their path.
#
if [ $mvhome = 1 ] ; then
    localpgm=`basename $progname`
    # if mvhome set, and the executable does NOT exist, then
    # remove it when we exit
    localpgm=/sphome/$LOGNAME/$localpgm	
    if [ -x $localpgm ] ; then
	rmhome=0
    else
	rmhome=1
    fi
    if [ $just_testing = 1 ] ; then
	echo cp $progname $localpgm
    else
	# If you don't remove the file first, sometimes the 
	# WRONG FILE (the OLD ONE) is found by the system.
	/bin/rm -f $localpgm
	if [ -s $localpgm ] ; then
	    echo "Could not move $progname to /sphome/$LOGNAME/$localpgm"
	    if [ -d $localpgm ] ; then
		echo "Destination is a directory!"
	    fi
	    exit 1
	fi
        cp $progname $localpgm
    fi
    PWD_LOCAL=/sphome/$LOGNAME
else
    rmhome=0
    localpgm=$progname
    PWD_LOCAL=$PWD_TRIAL
fi
#
# See if we can write in $PWD_LOCAL.  If not, try $HOME.
SCRIPT_DIR=$PWD_LOCAL
/bin/rm -f $PWD_LOCAL/PItest$$
echo "Test" > $PWD_LOCAL/PItest$$
if [ ! -s $PWD_LOCAL/PItest$$ ] ; then
    SCRIPT_DIR=$HOME
fi
/bin/rm -f $PWD_LOCAL/PItest$$
/bin/rm -f $SCRIPT_DIR/PIrun.$$
/bin/rm -f /sphome/$LOGNAME/job.output.$$
if [ -z "$stdinfile" ] ; then
   stdinfile=/dev/null
fi
#
# The actual script depends on how we're running
#
# If a partition is available, use it.  Otherwise, get time from the scheduler
#
if [ -n "$MPI_PARTITION" -a -s "$MPI_PARTITION" ] ; then
    cat >> $SCRIPT_DIR/PIrun.$$ <<EOF
#! /bin/sh
cd $PWD_LOCAL
MP_EUILIB=us
MP_RMPOOL=0
MP_HOSTFILE=$MPI_PARTITION
MP_PROCS=$np
MP_INFOLEVEL=0
MP_HOLD_STDIN=YES
MP_PULSE=0
export MP_EUILIB
export MP_RMPOOL
export MP_HOSTFILE
export MP_PROCS
export MP_INFOLEVEL
export MP_HOLD_STDIN
export MP_PULSE
# For MPI_INIT
MPIRUN_DEVICE=ch_mpl
export MPIRUN_DEVICE
#
if [ $polling_mode = 0 ] ; then
   MP_CSS_INTERRUPT=YES
   export MP_CSS_INTERRUPT
fi
/bin/rm -f /sphome/$LOGNAME/job.output.$$
# Wait for no-zero-sized SPnodes file
while test \! -s \$MP_HOSTFILE ; do sleep 1 ; done
#
# If the spxcp program is available, use it and change localpgm
if [ $np -gt 2 -a -x "$spxcp" ] ; then
    newlocalpgm=`basename $localpgm`
    cp $localpgm /tmp/\$newlocalpgm
    localpgm="/tmp/\$newlocalpgm"
    eval $spxcp \$localpgm
fi
#
if [ $mpirun_verbose = 1 ] ; then
    echo "About to run poe ... "
fi
poe $localpgm $cmdLineArgs < $stdinfile 2>&1
if [ $mpirun_verbose = 1 ] ; then
    echo "Poe exited ..."
fi
# Attempt to fix file cache problems
#SYNCLOC#
#sleep 2
exit 0
EOF
   # End of script for running progrma
   chmod a+x $SCRIPT_DIR/PIrun.$$
   # Get the home node
   homenode=`head -1 $MPI_PARTITION`
   if [ $just_testing = 1 ] ; then
       echo rsh $homenode -n $SCRIPT_DIR/PIrun.$$
   else
       rsh $homenode -n $SCRIPT_DIR/PIrun.$$
   fi
   if [ $rmhome = 1 ] ; then
	/bin/rm $localpgm
   fi
   /bin/rm -f $SCRIPT_DIR/PIrun.$$ 
   if [ "$mvback" != "" ] ; then
	(cd $PWD_LOCAL ; for file in $mvback ; do \
	if [ -f $file ] ; then cp $file $PWD_TRIAL ; /bin/rm $file ; \
	fi ; done )
   fi
   exit 0
fi
#
# If we reach here, there was no partition available.  This script will be
# run in a partition that will be allocated and removed
#
cat >$SCRIPT_DIR/PIrun.$$ <<.
#! /bin/sh
JID=\`/usr/local/bin/getjid\`
trap "sprelease \$JID"
cd $PWD_LOCAL
MP_EUILIB=us
MP_RMPOOL=0
MP_HOSTFILE=/sphome/$LOGNAME/SPnodes.\$JID    
MP_PROCS=$np
MP_INFOLEVEL=0
MP_HOLD_STDIN=YES
MP_PULSE=0
export MP_EUILIB
export MP_RMPOOL
export MP_HOSTFILE
export MP_PROCS
export MP_INFOLEVEL
export MP_HOLD_STDIN
export MP_PULSE
# For MPI_INIT
MPIRUN_DEVICE=ch_mpl
export MPIRUN_DEVICE
#
if [ $polling_mode = 0 ] ; then
   MP_CSS_INTERRUPT=YES
   export MP_CSS_INTERRUPT
fi
/bin/rm -f /sphome/$LOGNAME/job.output.$$
# Wait for no-zero-sized SPnodes file
while test \! -s \$MP_HOSTFILE ; do sleep 1 ; done
#
# If the spxcp program is available, use it and change localpgm
if [ $np -gt 2 -a -x "$spxcp" ] ; then
    newlocalpgm=`basename $localpgm`
    cp $localpgm /tmp/\$newlocalpgm
    localpgm="/tmp/\$newlocalpgm"
    eval $spxcp \$localpgm
fi
#
if [ $mpirun_verbose = 1 ] ; then
    echo "About to run poe ... " >> /sphome/$LOGNAME/job.output.$$
fi
poe $localpgm $cmdLineArgs < $stdinfile >> /sphome/$LOGNAME/job.output.$$ 2>&1
if [ $mpirun_verbose = 1 ] ; then
    echo "Poe exited ..." >> /sphome/$LOGNAME/job.output.$$
fi
# Attempt to fix file cache problems
#SYNCLOC#
sleep 2
sprelease \$JID
exit 0
.
#
# End of script for running progrma
chmod a+x $SCRIPT_DIR/PIrun.$$
#
# Determine CAC
if [ "$CAC" = "" ] ; then
   CAC=`whatcac | head -1 | cut -d' ' -f 9 | sed -e s/\"//g -e s/://`
fi
if [ $just_testing = 1 ] ; then
    echo "spsubmit -nomail <<."
    echo $CAC
    echo $max_time
    echo $np
    echo B
    echo M
    echo n
    echo $SCRIPT_DIR/PIrun.$$
    echo 
    echo C
    echo .
    echo JID=\`/usr/local/bin/getjid\`
    echo spwait \$JID
    if [ $rmhome = 1 ] ; then
	echo "/bin/rm $localpgm"
    fi
    if [ "$mvback" != "" ] ; then
	echo "(cd $PWD_LOCAL ; cp $mvback $PWD_TRIAL ; /bin/rm $mvback )"
    fi
else
    # Send output to /dev/null in case we're trying to 
    # use this to match up output
    #JID=`/usr/local/bin/getjid`
    #if [ "$JID" != "" ] ; then 
    #	echo "Can not start SP job while one is in the queue"
    #	echo "Job $JID was found"
    #	exit 1
    #fi
    # 
    # I've had lots of problems with files not being present that I'm going
    # to chalk up to NFS problems.  To work around this, I'm going to 
    # add some sleeps and syncs
    #SYNCLOC# > /dev/null 2>&1
    sleep 2
    JID=`sptest -cac $CAC -min $max_time -nodes $np -batch -mpl -us \
    	-path $SCRIPT_DIR/PIrun.$$ -noemail` 
#    spsubmit -nomail <<. > /dev/null 2>&1
#$CAC
#$max_time
#$np
#B
#M
#n
#$SCRIPT_DIR/PIrun.$$
#
#C
#.
#    JID=`/usr/local/bin/getjid`
    trap "sprelease ${JID}" 2 3
    # There may be some filesystem race condition in spwait
    # THIS IS FIXED IN THE MOST RECENT ANL CODE; IF YOU HAVE TROUBLE,
    # GET A NEW VERSION OF spq and spwait
    #SYNCLOC# > /dev/null 2>&1
    spwait $JID
    trap 2 3
    # 
    # I've had lots of problems with files not being present (or not being
    # written!) that I'm going to chalk up to NFS problems.  To work around 
    # this, I'm going to add some sleeps and syncs 
    # (sleeps removed now that spwait works!)
    #SYNCLOC# > /dev/null 2>&1
    if [ $rmhome = 1 ] ; then
	/bin/rm $localpgm
    fi
    /bin/rm -f /sphome/$LOGNAME/SPnodes.$JID $SCRIPT_DIR/PIrun.$$ 
    if [ "$mvback" != "" ] ; then
	(cd $PWD_LOCAL ; for file in $mvback ; do \
	if [ -f $file ] ; then cp $file $PWD_TRIAL ; /bin/rm $file ; \
	fi ; done )
    fi
    if [ -s /sphome/$LOGNAME/job.output.$$ ] ; then 
	# Try a touch to synchronize file caches... Sigh... 
	touch /sphome/$LOGNAME/job.output.$$
	cat /sphome/$LOGNAME/job.output.$$
	/bin/rm -f /sphome/$LOGNAME/job.output.$$
    # Note that there is no good way to identify problems with the run
    # now...
#    else
#	echo "mpirun: No output file!  Job did not run! "
    fi
fi