1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
|
#!/bin/ksh
#set -x
######################################################################
#
host=`uname -n`
user=`whoami`
#
## - Define paths
srcdir=`pwd`
datadir=$srcdir/data_oasis3
casename=`basename $srcdir`
#
## - Name of the executables
exe1=model1
exe2=model2
#
############### User's section #######################################
#
## - Define architecture and coupler
arch=tioman_pgi_mpich #tioman_pgi_mpich/corail/curie(CEA)/jade/training_computer/ubuntu
#crayXE6
#
# - Define number of processes to run each executable
nproc_exe1=1
nproc_exe2=1
#
## - Define rundir
if [ ${arch} == tioman_pgi_mpich ] || [ ${arch} == training_computer ]; then
MPIRUN=/usr/local/pgi/linux86-64/2011/mpi/mpich/bin/mpirun
rundir=${HOME}/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == tioman_intel_openmpi ]; then
MPIRUN=/usr/local_intel12/openmpi143/bin/mpirun
rundir=${HOME}/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == corail ]; then
rundir=/lustre/globc/${user}/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == curie ]; then
rundir=/ccc/work/cont005/pa0490/coquartl/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == jade ]; then
rundir=/data/11coqu/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == neptune_gfortran ]; then
rundir=/scratch/globc/coquart/oasis3-mct_buildbot/examples/${casename}/work_${casename}
elif [ ${arch} == ubuntu ]; then
MPIRUN=mpirun
rundir=${HOME}/Devel/oasis3-mct/examples/${casename}/work_${casename}
elif [ ${arch} == crayXE6 ]; then
rundir=/zhome/academic/HLRS/imk/imkbreil/Programme/oasis/oasis3-mct/examples/tutorial/work_${casename}
elif [ ${arch} == beaufix ]; then
# The toy must be directly in the TMPDIRs the jobs can only run in the TMPDIR
rundir=/scratch/utmp/cglo355/oasis3-mct/examples/tutorial/work_${casename}
elif [ ${arch} == ada ]; then
MPIRUN=/opt/intel/impi/4.1.0.024/intel64/bin/mpirun
rundir=/workgpfs/rech/ces/rces980/modBRETAGNE/oasis3-mct/examples/tutorial/work_${casename}
fi
#
############### End of user's section ################################
#
##- Define the name of the source grid and the target grid
config=$datadir/name_grids.dat
x=$(cut -d ' ' -f 1 < $config)
echo 'x:' $x
SRC_GRID=`echo $x | cut -b 42-45`
TGT_GRID=`echo $x | cut -b 94-97`
echo 'Source grid :' $SRC_GRID
echo 'Target grid :' $TGT_GRID
#
echo ''
echo '*****************************************************************'
echo '*** '$casename' : '$run
echo ''
echo 'Rundir :' $rundir
echo 'Architecture :' $arch
echo 'Host : '$host
echo 'User : '$user
echo ''
echo $exe1' runs on '$nproc_exe1 'processes'
echo $exe2' runs on '$nproc_exe2 'processes'
echo ''
echo ''
######################################################################
###
### 1. Copy source example directory containing everything needed
### into rundir
\rm -fr $rundir
mkdir -p $rundir
cp -f $datadir/*nc $rundir/.
cp -f $datadir/*.jnl $rundir/.
cp -f $datadir/*.dat $rundir/.
cp -f $srcdir/$exe1 $rundir/.
cp -f $srcdir/$exe2 $rundir/.
cp -f $datadir/namcouple $rundir/.
#
cd $rundir
#
######################################################################
###
### 3. Creation of configuration scripts
###---------------------------------------------------------------------
### Linux
###---------------------------------------------------------------------
if [ $arch == tioman_pgi_mpich ] || [ $arch == training_computer ]; then
if [ $nproc_exe1 == 1 ]; then
cat <<EOF >> $rundir/appl-linux.conf
$host 0 $rundir/$exe1
EOF
else
cat <<EOF >> $rundir/appl-linux.conf
$host 0 $rundir/$exe1
EOF
count=1
while [[ $count -lt $nproc_exe1 ]];do
cat <<EOF >> $rundir/appl-linux.conf
$host 1 $rundir/$exe1
EOF
(( count += 1 ))
done
fi
count=0
while [[ $count -lt $nproc_exe2 ]];do
cat <<EOF >> $rundir/appl-linux.conf
$host 1 $rundir/$exe2
EOF
(( count += 1 ))
done
###---------------------------------------------------------------------
### CORAIL
###---------------------------------------------------------------------
elif [ $arch == corail ] ; then
cat <<EOF > $rundir/run_$casename.$arch
# Nom du job
#PBS -N ${casename}
# Temps limite du job
#PBS -l walltime=00:10:00
# Nombre de processus
#PBS -l select=1:mpiprocs=24:ncpus=24
#PBS -l place=scatter:excl
# adresse email a utiliser
##PBS -M your_email
# Mail envoye a la fin du job
#PBS -m ae
cd $rundir
source /usr/local/bin/intelmpi.sh
ulimit -s unlimited
#
#
time mpirun -np $nproc_exe1 ./$exe1 : -np $nproc_exe2 ./$exe2
#
EOF
###---------------------------------------------------------------------
### BEAUFIX
###---------------------------------------------------------------------
elif [ $arch == beaufix ] ; then
(( nproc = $nproc_exe1 + $nproc_exe2 ))
cat <<EOF > $rundir/run_$casename.$arch
#!/bin/bash
#SBATCH --time=01:00:00
#SBATCH -p normal32 # partition/queue
#SBATCH --job-name=toys # job name
#SBATCH -N 1 # number of nodes
#SBATCH -n $nproc # number of procs
#SBATCH -o job.out%j
#SBATCH -o job.err%j
#SBATCH --exclusive
ulimit -s unlimited
# rundir must be in the TMPDIR
cd $rundir
module load intel/13.1.4.183
module load intelmpi/4.1.1.036
module load netcdf/4.3.0
#
#
time mpirun -np $nproc_exe1 ./$exe1 : -np $nproc_exe2 ./$exe2
#
EOF
###---------------------------------------------------------------------
### NEPTUNE
###---------------------------------------------------------------------
elif [ $arch == neptune_gfortran ] ; then
cat <<EOF > $rundir/run_$casename.$arch
# Nom du job
#PBS -N tests
# Temps limite du job
#PBS -l walltime=00:10:00
# Nombre de processus
#PBS -l select=1:mpiprocs=16:ncpus=16
#PBS -l place=scatter:excl
cd $rundir
export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH
ulimit -s unlimited
#
#
time mpirun -np $nproc_exe1 ./$exe1 : -np $nproc_exe2 ./$exe2
#
EOF
###---------------------------------------------------------------------
### CURIE (normal nodes)
###---------------------------------------------------------------------
elif [ $arch == curie ] ; then
cat <<EOF > $rundir/appl-curie.conf
$nproc_exe1 $rundir/$exe1
$nproc_exe2 $rundir/$exe2
EOF
(( nproc = $nproc_exe1 + $nproc_exe2 ))
cat <<EOF > $rundir/run_$casename.$arch
#!/bin/bash
#MSUB -r test_interp # Request name
#MSUB -n $nproc # Number of tasks to use
#MSUB -T 3600 # Elapsed time limit in seconds
#MSUB -o $casename_%I.o # Standard output. %I is the job id
#MSUB -e $casename_%I.e # Error output. %I is the job id
#MSUB -A gen6028 # Project ID
set -x
cd $rundir
ccc_mprun -f appl-curie.conf
EOF
###-----------------------------------------------------------------
### JADE - CINES
###-----------------------------------------------------------------
elif [ $arch == jade ] ; then
(( NNODE = ( $nproc_exe1 + $nproc_exe2 ) / 8 ))
(( RESTE = ( $nproc_exe1 + $nproc_exe2 ) - ( $NNODE * 8 ) ))
if [[ $RESTE -gt 0 ]]; then
(( NNODE = $NNODE + 1 ))
fi
echo $NNODE
cat << EOF > $rundir/run_$casename.$arch
#PBS -S /bin/bash
#PBS -N OASIS
#PBS -l walltime=00:10:00
#PBS -l select=$NNODE:ncpus=8:mpiprocs=8
#PBS -o $rundir/out
#PBS -e $rundir/err
#PBS -j oe
#
set -evx
module load netcdf
cd $rundir
cat $PBS_NODEFILE
## Lancement executable
which mpiexec
export MPI_GROUP_MAX=100
time mpiexec -n $nproc_exe1 ./$exe1 : -n $nproc_exe2 ./$exe2
EOF
###---------------------------------------------------------------------
### CRAYXE6
###---------------------------------------------------------------------
elif [ $arch == crayXE6 ] ; then
(( nproc = $nproc_exe1 + $nproc_exe2 ))
d_aus=lm_aus.log # Ausgabe- u. Fehlerdateien
d_err=lm_fehler.log
cat <<EOF > $rundir/run_$casename.$arch
#!/bin/ksh
# Nom du job
#PBS -N ${casename}
# Temps limite du job
#PBS -l walltime=00:10:00
# Nombre de processus
#PBS -l mppwidth=$nproc
#PBS -l mppnppn=1
#PBS -l mppdepth=1
#PBS -o $d_aus
#PBS -e $d_err
#
cd $rundir
#
ulimit -c unlimited
export LIBDWD_FORCE_CONTROLWORDS=1
export LIBDWD_BITMAP_TYPE=ASCII
#
#export MPICH_GNI_DYNAMIC_CONN=disabled
export MPICH_ENV_DISPLAY=1
export MPICH_GNI_MAX_EAGER_MSG_SIZE=64000
export MPICH_ABORT_ON_ERROR=1
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=-1
#
aprun -n $nproc_exe1 ./$exe1 : -n $nproc_exe2 ./$exe2
#
EOF
###---------------------------------------------------------------------
### ada
###---------------------------------------------------------------------
elif [ $arch == ada ] ; then
(( nproc = $nproc_exe1 + $nproc_exe2 ))
cat <<EOF > $rundir/run_$casename.$arch
#!/bin/ksh
# ######################
# ## ADA IDRIS ##
# ######################
# Nom de la requete
# @ job_name = ${casename}
# Type de travail
# @ job_type = parallel
# Fichier de sortie standard
# @ output = Script_Output_${casename}.000001
# Fichier de sortie erreur (le meme)
# @ error = Script_Output_${casename}.000001
# Nombre de processus demandes
# @ total_tasks = ${nproc}
# @ environment = "BATCH_NUM_PROC_TOT=32"
# Temps CPU max. par processus MPI hh:mm:ss
# @ wall_clock_limit = 0:30:00
# Fin de l entete
# @ queue
#
# pour avoir l'echo des commandes
set -x
# on se place dans le repertoire rundir
cd ${rundir}
module load netcdf
module load hdf5
#
export KMP_STACKSIZE=64m
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/smplocal/pub/NetCDF/4.1.3/lib
poe -pgmmodel MPMD -cmdfile run_file
#
EOF
fi
######################################################################
###
### 4. Execute the model
if [ $arch == tioman_pgi_mpich ] || [ $arch == training_computer ]; then
echo 'Executing the model using '$MPIRUN
$MPIRUN -p4pg appl-linux.conf ./$exe1 > runjob.err
elif [ $arch == ubuntu ] || [ $arch == tioman_intel_openmpi ] ; then
echo 'Executing the model using '$MPIRUN
$MPIRUN -np $nproc_exe1 ./$exe1 : -np $nproc_exe2 ./$exe2 > runjob.err
elif [ $arch == corail ]; then
echo 'Submitting the job to queue using qsub'
qsub -q submit $rundir/run_$casename.$arch
qstat | grep $user
elif [ $arch == beaufix ]; then
echo 'Submitting the job to queue using sbatch'
sbatch $rundir/run_$casename.$arch
squeue -u $user
elif [ $arch == neptune_gfortran ]; then
echo 'Submitting the job to queue using qsub'
qsub $rundir/run_$casename.$arch
qstat | grep $user
elif [ $arch == curie ]; then
ccc_msub $rundir/run_$casename.$arch
ccc_mpp | grep $user
elif [ $arch == jade ] ; then
qsub $rundir/run_$casename.$arch
qstat -awu $user
elif [ $arch == crayXE6 ]; then
echo 'Submitting the job to queue using qsub'
chmod u+x $rundir/run_$casename.$arch
qsub $rundir/run_$casename.$arch
qstat
elif [ $arch == ada ]; then
echo 'Submitting the job to queue using llsubmit'
llsubmit $rundir/run_$casename.$arch
fi
echo $casename 'is executed or submitted to queue.'
echo 'Results are found in rundir : '$rundir
######################################################################
|