File: debian_sge

package info (click to toggle)
fsl 5.0.8-6
  • links: PTS, VCS
  • area: non-free
  • in suites: buster
  • size: 23,660 kB
  • sloc: cpp: 166,922; tcl: 25,439; sh: 14,612; ansic: 9,707; makefile: 1,336; python: 151; xml: 106; csh: 17
file content (306 lines) | stat: -rw-r--r-- 12,840 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
Description: out-of-the-box compatibility with Debian's SGE
 Moreover, the patch established the env variable FSLPARALLEL as a uniform
 trigger for cluster-based processing in FSL. If set, jobs are submitted to
 SGE, if not, not.
--- a/src/fdt/bedpostx
+++ b/src/fdt/bedpostx
@@ -262,16 +262,21 @@
 mkdir -p ${subjdir}.bedpostX/logs/monitor
 mkdir -p ${subjdir}.bedpostX/xfms
 
-mailto=`whoami`@fmrib.ox.ac.uk
+# This setting is already present in fsl_sub and allows for a more flexible,
+# centralized configuration
+#mailto=`whoami`@fmrib.ox.ac.uk
 echo Queuing preprocessing stages
-preprocid=`${FSLDIR}/bin/fsl_sub -T 60 -m as -N bpx_preproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}`
+# don't setup email options, rely on fsl_sub's setting
+#preprocid=`${FSLDIR}/bin/fsl_sub -T 60 -m as -N bpx_preproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}`
+preprocid=`${FSLDIR}/bin/fsl_sub -T 60 -N bpx_preproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_preproc.sh ${subjdir} ${gflag}`
 
 echo Queuing parallel processing stage
 nslices=`${FSLDIR}/bin/fslval ${subjdir}/data dim3`
 [ -f ${subjdir}.bedpostX/commands.txt ] && rm ${subjdir}.bedpostX/commands.txt
 
 monitor
-if [ "x$SGE_ROOT" = "x" ]; then
+# Don't be SGE-specific -- rely on FSLPARALLEL
+if [ "x$FSLPARALLEL" = "x" ]; then
     ${subjdir}.bedpostX/monitor&
 fi
 
@@ -293,12 +298,15 @@
     fi
     slice=$(($slice + 1))
 done
-bedpostid=`${FSLDIR}/bin/fsl_sub -j $preprocid -l ${subjdir}.bedpostX/logs -M $mailto -N bedpostx -t ${subjdir}.bedpostX/commands.txt`
+# don't specify a mailto address, but rely on fsl_sub doing it
+bedpostid=`${FSLDIR}/bin/fsl_sub -j $preprocid -l ${subjdir}.bedpostX/logs -N bedpostx -t ${subjdir}.bedpostX/commands.txt`
 
 echo Queuing post processing stage
-mergeid=`${FSLDIR}/bin/fsl_sub -j $bedpostid -T 60 -m as -N bpx_postproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_postproc.sh $subjdir`
+# don't setup email options, rely on fsl_sub's setting
+mergeid=`${FSLDIR}/bin/fsl_sub -j $bedpostid -T 60 -N bpx_postproc -l ${subjdir}.bedpostX/logs ${FSLDIR}/bin/bedpostx_postproc.sh $subjdir`
 
-if [ "x$SGE_ROOT" != "x" ]; then
+# Don't be SGE-specific -- rely on FSLPARALLEL
+if [ "x$FSLPARALLEL" != "x" ]; then
     echo
     echo Type ${subjdir}.bedpostX/monitor to show progress.
     echo Type ${subjdir}.bedpostX/cancel to terminate all the queued tasks.
--- a/src/fdt/fdt.tcl
+++ b/src/fdt/fdt.tcl
@@ -858,8 +858,11 @@
 	probtrackx {
 	    global probtrack env
 	    set errorStr ""
-            set FSLPARALLEL 0
-            if { [ info exists env(SGE_ROOT) ] && $env(SGE_ROOT) != "" } { set FSLPARALLEL 1 }
+	    # there seems to be no need to reset FSLPARALLEL since the desired
+	    # configuration is already determined in fslstart.tcl (sourced at the
+	    #top of this file) in a way that is not SGE-specific.
+        #set FSLPARALLEL 0
+        #if { [ info exists env(SGE_ROOT) ] && $env(SGE_ROOT) != "" } { set FSLPARALLEL 1 }
 	    if { $probtrack(bedpost_dir) == ""  } { set errorStr "You must specify the bedpostX directory!" }
 	    if { $probtrack(mode) == "simple" && $probtrack(usereference_yn) && $probtrack(reference) == "" } { set errorStr "$errorStr You must specify a reference image" } 
 	    if { $probtrack(mode) == "seedmask" && $probtrack(reference) == "" } { set errorStr "$errorStr You must specify a mask image" } 
@@ -1025,7 +1028,11 @@
                     puts $script "mv $logfile $copylog"
                     puts $script "rm ${filebase}_script.sh"
 		    close $script
-		    exec $FSLDIR/bin/fsl_sub -q long.q ${filebase}_script.sh
+			# Don't specify specific queue name (to leave the logic within
+			# fsl_sub, but specify a long time (20 hours) to achieve the same
+			# goal)
+		    #exec $FSLDIR/bin/fsl_sub -q long.q ${filebase}_script.sh
+		    exec $FSLDIR/bin/fsl_sub -T 1200 ${filebase}_script.sh
 		} else {
 
 		    fdt_monitor_short $w "$FSLDIR/bin/probtrackx2 $flags"
--- a/src/misc_tcl/fslstart.tcl
+++ b/src/misc_tcl/fslstart.tcl
@@ -55,7 +55,12 @@
 
 #Choose load_varian specific OR FSLPARALLEL
 set FSLPARALLEL 0
-if { [ info exists env(SGE_ROOT) ] && $env(SGE_ROOT) != "" } { set FSLPARALLEL 1 }
+# Instead of relying on SGE_ROOT let's use FSLPARALLEL itself to determine
+# whether using a cluster is desired. This allows for other cluster engines
+# to be used.
+#if { [ info exists env(SGE_ROOT) ] && $env(SGE_ROOT) != "" } { set FSLPARALLEL 1 }
+if { [ info exists env(FSLPARALLEL) ] && $env(FSLPARALLEL) != "" } { set FSLPARALLEL 1 }
+
 
 # what type of OS?
 set OSFLAVOUR unix
--- a/src/possum/possumX
+++ b/src/possum/possumX
@@ -251,7 +251,9 @@
 mkdir -p ${subjdir}/diff_proc
 mkdir -p ${subjdir}/logs
 mkdir -p ${subjdir}/logs/pid_${$}
-mailto=`whoami`@fmrib.ox.ac.uk
+# This setting is already present in fsl_sub and allows for a more flexible,
+# centralized configuration
+#mailto=`whoami`@fmrib.ox.ac.uk
 
 echo Preprocessing stage
 echo "${POSSUMDIR}/bin/possum_matrix -p ${subjdir}/pulse -m ${subjdir}/motion --seg=$segs -o ${subjdir}/matrix"
@@ -274,9 +276,10 @@
 #tejas-add
 if [ ! -z $holdjb ]; then hold="-j $holdjb "; fi
 
-echo "${FSLDIR}/bin/fsl_sub -T $ntime -l ${subjdir}/logs $arch -M $mailto -N possum -t ${subjdir}/possum.com"
+# don't specify a mailto address, but rely on fsl_sub doing it
+echo "${FSLDIR}/bin/fsl_sub -T $ntime -l ${subjdir}/logs $arch -N possum -t ${subjdir}/possum.com"
 unset LD_LIBRARY_PATH DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH
-possumid=`${FSLDIR}/bin/fsl_sub -T $ntime ${hold}-l ${subjdir}/logs $arch -M $mailto -N possum -t ${subjdir}/possum.com` 
+possumid=`${FSLDIR}/bin/fsl_sub -T $ntime ${hold}-l ${subjdir}/logs $arch -N possum -t ${subjdir}/possum.com` 
 #tejas-end
 
 
--- a/src/sgeutils/fsl_sub
+++ b/src/sgeutils/fsl_sub
@@ -71,6 +71,13 @@
 # cluster.
 ###########################################################################
 
+# Domain mail should be sent to if necessary. The complete address will be
+# `whoami`@${fslcluster_mailto_domain}
+fslcluster_mailto_domain=localhost
+# When should the cluster engine send status email (see qsub -m ... docs)
+# Don't send email by default; can be overidden by FSLCLUSTER_MAILOPTS
+# environment variable
+fslcluster_mailopts="${FSLCLUSTER_MAILOPTS:-n}"
 
 ###########################################################################
 # The following section determines what to do when fsl_sub is called
@@ -92,6 +99,24 @@
 	fi
 fi
 
+# define the default 'qsub' implementation to be used for cluster submission
+qsub_cmd="qsub"
+
+# Allow to override the above automatic detection result with FSLPARALLEL
+if [ -n "$FSLPARALLEL" ] ; then
+    # Whenever FSLPARALLEL is set enfore using SGE even if no SGE_ROOT is set
+    # which, for example, is the case on Debian systems running SGE
+    METHOD=SGE
+    # TODO: move cluster engine detection here to be able to support more than
+    #       just SGE
+    if [ "$FSLPARALLEL" = "condor" ] ; then
+        # if condor shall be used, simply switch to Condor's qsub emulation
+        qsub_cmd="condor_qsub"
+    fi
+else
+    METHOD=NONE
+fi
+
 # stop submitted scripts from submitting jobs themselves
 if [ "X$FSLSUBALREADYRUN" = "Xtrue" ] ; then
     METHOD=NONE
@@ -116,16 +141,22 @@
 
 map_qname ()
 {
-    if [ $1 -le 20 ] ; then
-	queue=veryshort.q
-    elif [ $1 -le 120 ] ; then
-	queue=short.q
-    elif [ $1 -le 1440 ] ; then
-	queue=long.q
-    else
-	queue=verylong.q
-    fi
-    queueCmd=" -q $queue " 
+	# for Debian we can't do the stuff below, because it would be hard
+	# to determine how particular queues are meant to be used on any given
+	# system. Instead of translating into a queue name we specify proper
+	# resource limits, and let SGE decide what queue matches
+	# (qsub wants the time limit in seconds)
+	queueCmd="$queueCmd -l h_rt=$(echo "$1 * 60" | bc)"
+    #if [ $1 -le 20 ] ; then
+	#queue=veryshort.q
+    #elif [ $1 -le 120 ] ; then
+	#queue=short.q
+    #elif [ $1 -le 1440 ] ; then
+	#queue=long.q
+    #else
+	#queue=verylong.q
+    #fi
+    #queueCmd=" -q $queue "
 
     #echo "Estimated time was $1 mins: queue name is $queue"
 }
@@ -151,13 +182,13 @@
 $command -q short.q gzip *.img *.hdr
 $command -a darwin regscript rawdata outputdir ...
 
-  -T <minutes>          Estimated job length in minutes, used to auto-set queue name
-  -q <queuename>        Possible values for <queuename> are "verylong.q", "long.q" 
-                        and "short.q". See below for details
-                        Default is "long.q".
+  -T <minutes>          Estimated job length in minutes, used to let SGE select
+                        an appropriate queue
+  -q <queuename>        Possible values for <queuename> can be queried with
+                        'qconf -sql'
   -a <arch-name>        Architecture [e.g., darwin or lx24-amd64]
   -p <job-priority>     Lower priority [0:-1024] default = 0                 
-  -M <email-address>    Who to email, default = `whoami`@fmrib.ox.ac.uk 
+  -M <email-address>    Who to email, default = `whoami`@localhost
   -j <jid>              Place a hold on this task until job jid has completed
   -t <filename>         Specify a task file of commands to execute in parallel
   -N <jobname>          Specify jobname as it will appear on queue
@@ -170,24 +201,26 @@
                         <threads> specifies the number of threads to run
   -v                    Verbose mode.
 
-Queues:
-
-There are several batch queues configured on the cluster, each with defined CPU
-time limits. All queues, except bigmem.q, have a 8GB memory limit.
-
-veryshort.q:This queue is for jobs which last under 30mins.
-short.q:    This queue is for jobs which last up to 4h. 
-long.q:     This queue is for jobs which last less than 24h. Jobs run with a
-            nice value of 10.
-verylong.q: This queue is for jobs which will take longer than 24h CPU time.
-            There is one slot per node, and jobs on this queue have a nice value
-            of 15.
-bigmem.q:   This queue is like the verylong.q but has no memory limits.
-
 EOF
 
   exit 1
 }
+# This part of the help is not applicable anymore
+#
+#Queues:
+#
+#There are several batch queues configured on the cluster, each with defined CPU
+#time limits. All queues, except bigmem.q, have a 8GB memory limit.
+#
+#veryshort.q:This queue is for jobs which last under 30mins.
+#short.q:    This queue is for jobs which last up to 4h. 
+#long.q:     This queue is for jobs which last less than 24h. Jobs run with a
+#            nice value of 10.
+#verylong.q: This queue is for jobs which will take longer than 24h CPU time.
+#            There is one slot per node, and jobs on this queue have a nice value
+#            of 15.
+#bigmem.q:   This queue is like the verylong.q but has no memory limits.
+
 
 nargs=$#
 if [ $nargs -eq 0 ] ; then
@@ -222,10 +255,13 @@
 # change. It also sets up the basic emailing control.
 ###########################################################################
 
-queue=long.q
-queueCmd=" -q long.q "
-mailto=`whoami`@fmrib.ox.ac.uk
-MailOpts="a"
+# SGE should already have a default queue, but allow for overwrite
+queueCmd=""
+if [ "x$FSLCLUSTER_DEFAULT_QUEUE" != "x" ] ; then
+    queueCmd=" -q $FSLCLUSTER_DEFAULT_QUEUE "
+fi
+mailto=`whoami`@${fslcluster_mailto_domain}
+MailOpts=${fslcluster_mailopts}
 
 
 ###########################################################################
@@ -423,6 +459,11 @@
 		exit -1
 	    fi
 
+		if [ "x$queue" = "x" ] ; then
+		echo $@
+		echo "No queue specified."
+		exit -1
+	    fi
             # Get a list of queues configured for this PE and confirm that the queue
             # we have submitted to has that PE set up.
 	    qstat -g c -pe $peName >/dev/null 2>&1
@@ -447,9 +488,9 @@
 
 	if [ "x$tasks" = "x" ] ; then
 	    if [ $scriptmode -ne 1 ] ; then
-		sge_command="qsub -V -cwd -shell n -b y -r y $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold"
+		sge_command="$qsub_cmd -V -cwd -shell n -b y -r y $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold"
 	    else
-		sge_command="qsub $LogOpts $sge_arch $sge_hold"
+		sge_command="$qsub_cmd $LogOpts $sge_arch $sge_hold"
 	    fi
 	    if [ $verbose -eq 1 ] ; then 
 		echo sge_command: $sge_command >&2
@@ -457,7 +498,7 @@
 	    fi
 	    exec $sge_command $@ | awk '{print $3}'
 	else
-	    sge_command="qsub -V -cwd $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold $sge_tasks"
+	    sge_command="$qsub_cmd -V -cwd $queueCmd $pe_options -M $mailto -N $JobName -m $MailOpts $LogOpts $sge_arch $sge_hold $sge_tasks"
 	    if [ $verbose -eq 1 ] ; then 
 		echo sge_command: $sge_command >&2
 		echo control file: $taskfile >&2