1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
#!/bin/bash
# Bash script to submit many files to Cori/Edison/Queue
EXIT_SUCCESS=0
EXIT_HOST=1
EXIT_PARAM=2
# MAX_PARAMS=1
# # ^^^ This should be fixed, as it should just loop through everything
# if [[ $# -eq 0 ]]; then
# echo "Must have at least one parameter; exiting"
# exit $EXIT_PARAM
# fi
# if [[ $# -gt $MAX_PARAMS ]]; then
# echo "Too many parameters; exiting"
# exit $EXIT_PARAM
# fi
# INPUT_FILE=$1
# # ^^^ Get the input ile
CUR_DIR=`pwd`
FILE_DIR=$CUR_DIR/EXAMPLE
INPUT_DIR=/project/projectdirs/sparse/liuyangz/my_research/matrix
FILE_NAME=pddrive
FILE=$FILE_DIR/$FILE_NAME
TMP_BATCH_FILE=tmp_batch_file.slurm
# ^^^ Should check that this is not taken,
# but I would never use this, so ...
> $TMP_BATCH_FILE
if [[ $NERSC_HOST == edison ]]; then
CORES_PER_NODE=24
THREADS_PER_NODE=48
elif [[ $NERSC_HOST == cori ]]; then
CORES_PER_NODE=32
THREADS_PER_NODE=64
# This does not take hyperthreading into account
else
# Host unknown; exiting
exit $EXIT_HOST
fi
#nprows=(6 12 24)
#npcols=(6 12 24)
#nprows=(2048 1 32)
#npcols=(1 2048 64)
# nprows=(32 )
# npcols=(64 )
#nprows=(24 48 1 1 576 2304)
#npcols=(24 48 576 2304 1 1)
#nprows=(48 1 2304)
#npcols=(48 2304 1)
#nprows=(6 12 24 48 )
#npcols=(6 12 24 48 )
#nprows=(6 12 24 48 1 1 1 1 36 144 576 2304)
#npcols=(6 12 24 48 36 144 576 2304 1 1 1 1)
#nprows=(32 128 512 1 1 1 4 8 16)
#npcols=(1 1 1 32 128 512 8 16 32)
#nprows=(2048 1 32)
#npcols=(1 2048 64)
#nprows=(12 1 144)
#npcols=(12 144 1)
nprows=(4)
npcols=(4)
for ((i = 0; i < ${#npcols[@]}; i++)); do
NROW=${nprows[i]}
NCOL=${npcols[i]}
# NROW=36
CORE_VAL=`expr $NCOL \* $NROW`
NODE_VAL=`expr $CORE_VAL / $CORES_PER_NODE`
MOD_VAL=`expr $CORE_VAL % $CORES_PER_NODE`
if [[ $MOD_VAL -ne 0 ]]
then
NODE_VAL=`expr $NODE_VAL + 1`
fi
#PARTITION=debug
PARTITION=regular
LICENSE=SCRATCH
TIME=00:20:00
if [[ $NERSC_HOST == edison ]]
then
CONSTRAINT=0
fi
if [[ $NERSC_HOST == cori ]]
then
CONSTRAINT=haswell
fi
for NTH in 1 2 4
do
OMP_NUM_THREADS=$NTH
TH_PER_RANK=`expr $NTH \* 2`
#for NSUP in 128 64 32 16 8
#do
# for MAT in atmosmodl.rb nlpkkt80.mtx torso3.mtx Ga19As19H42.mtx A22.mtx cage13.rb
# for MAT in torso3.mtx
# for MAT in matrix121.dat matrix211.dat tdr190k.dat tdr455k.dat nlpkkt80.mtx torso3.mtx helm2d03.mtx
# for MAT in tdr190k.dat Ga19As19H42.mtx
# for MAT in torso3.mtx hvdc2.mtx matrix121.dat nlpkkt80.mtx helm2d03.mtx
for MAT in DG_GrapheneDisorder_8192.bin Li4244.bin torso3.bin
# for MAT in Ga19As19H42.mtx
do
# Start of looping stuff
> $TMP_BATCH_FILE
echo "#!/bin/bash -l" >> $TMP_BATCH_FILE
echo " " >> $TMP_BATCH_FILE
echo "#SBATCH -p $PARTITION" >> $TMP_BATCH_FILE
echo "#SBATCH -N $NODE_VAL" >> $TMP_BATCH_FILE
echo "#SBATCH -t $TIME" >> $TMP_BATCH_FILE
echo "#SBATCH -L $LICENSE" >> $TMP_BATCH_FILE
echo "#SBATCH -J SLU_$MAT" >> $TMP_BATCH_FILE
#echo "#SBATCH -o ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_async_simple_over_icollec_flat_mrhs" >> $TMP_BATCH_FILE
#echo "#SBATCH -e ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_async_simple_over_icollec_flat_mrhs" >> $TMP_BATCH_FILE
# echo "#SBATCH --mail-type=BEGIN" >> $TMP_BATCH_FILE
# echo "#SBATCH --mail-type=END" >> $TMP_BATCH_FILE
echo "#SBATCH --mail-user=liuyangzhuan@lbl.gov" >> $TMP_BATCH_FILE
if [[ $NERSC_HOST == cori ]]
then
echo "#SBATCH -C $CONSTRAINT" >> $TMP_BATCH_FILE
fi
mkdir -p $MAT
echo "export OMP_NUM_THREADS=$OMP_NUM_THREADS" >> $TMP_BATCH_FILE
echo "export KMP_NUM_THREADS=$OMP_NUM_THREADS" >> $TMP_BATCH_FILE
echo "export MKL_NUM_THREADS=$OMP_NUM_THREADS" >> $TMP_BATCH_FILE
echo "export NSUP=128" >> $TMP_BATCH_FILE
echo "export NREL=20" >> $TMP_BATCH_FILE
echo "export OMP_PLACES=threads" >> $TMP_BATCH_FILE
echo "export OMP_PROC_BIND=spread" >> $TMP_BATCH_FILE
echo "export MPICH_MAX_THREAD_SAFETY=multiple" >> $TMP_BATCH_FILE
echo " " >> $TMP_BATCH_FILE
echo "FILE=$FILE" >> $TMP_BATCH_FILE
echo "FILEMAT=$INPUT_DIR/$MAT" >> $TMP_BATCH_FILE
echo " " >> $TMP_BATCH_FILE
echo "CORE_VAL=$CORE_VAL" >> $TMP_BATCH_FILE
echo "NCOL=$NCOL" >> $TMP_BATCH_FILE
echo "NROW=$NROW" >> $TMP_BATCH_FILE
# This should be computed individually for each script...
export OMP_NUM_THREADS=$OMP_NUM_THREADS
export OMP_PLACES=threads
export OMP_PROC_BIND=spread
export MPICH_MAX_THREAD_SAFETY=multiple
srun -n $CORE_VAL -N 64 -c $TH_PER_RANK --cpu_bind=cores $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_hybridOMPMPI_${OMP_NUM_THREADS}_cori
# Add final line (srun line) to temporary slurm script
#cat $TMP_BATCH_FILE
#echo " "
# sbatch $TMP_BATCH_FILE
done
#one
done
done
exit $EXIT_SUCCESS
|
