1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
##################################################################################################################################
########################## ########################################
########################## Trinity PBS job submission with multi part dependencies ########################################
########################## ########################################
##################################################################################################################################
### Author: Josh Bowden, Alexie Papanicolaou, CSIRO
### Version 1.0
### QuantifyGraph and Butterfly p4a Script
##################################################################################################################################
# we will not use array in order to ensure only jobs that have not finished are Submitting.
# this does cause a problem when wanting to kill them manually but best to use kill script
## JOBPREFIX is passed via HASHBANG
JOBSTRING4=""$HASHBANG"
"$NODESCPUS"
JOB_CHRYSALIS="$JOBPREFIX"_p4b
JOB_BUTTERFLY="$JOBPREFIX"_p5b
cd "$OUTPUTDIR"
rm -f \$JOB_CHRYSALIS.jobnames \$JOB_BUTTERFLY.jobnames
FILENAME=""$OUTPUTDIR"/chrysalis/quantifyGraph_commands"
FILENAMEBFLY=""$OUTPUTDIR"/chrysalis/butterfly_commands"
if [ ! -e \$FILENAME.pbs ];then
sed -e 's/.*/if [[ -e SEDPLACEHOLDER ]]; then &;fi/' \$FILENAME|sed -r 's/(-e\s)SEDPLACEHOLDER(\s.+-i\s)(\S+).tmp/\1\3.tmp\2\3.tmp/' > \$FILENAME.pbs
split -d -a 3 -l "1000" \$FILENAME.pbs \$FILENAME.pbs.
sleep 5
fi
if [ ! -e \$FILENAMEBFLY.pbs ];then
sed -e 's/.*/if [ -e SEDPLACEHOLDER ]; then &;fi/' \$FILENAMEBFLY|sed -r 's/(-e\s)SEDPLACEHOLDER(\s.+-C\s)(\S+)/\1\3.out\2\3/' > \$FILENAMEBFLY.pbs
split -d -a 3 -l "1000" \$FILENAMEBFLY.pbs \$FILENAMEBFLY.pbs.
sleep 5
fi
FILENAME=\$FILENAME.pbs
FILENAMEBFLY=\$FILENAMEBFLY.pbs
NUMCMDS=\`ls -l \$FILENAME.??? | wc -l\`
NUMCMDSBFLY=\`ls -l \$FILENAMEBFLY.??? | wc -l\`
let NUMCMDS=\$NUMCMDS-1
let NUMCMDSBFLY=\$NUMCMDSBFLY-1
let SUBMITTED_C=0
let SUBMITTED_B=0
for ((JOBID=0;JOBID<=\$NUMCMDS;++JOBID));do
JOB_INDEX_PADDED=\`printf "%03d" \$JOBID\`
MYJOBQ=\""\$FILENAME".\$JOB_INDEX_PADDED\"
MYJOBB=\""\$FILENAMEBFLY".\$JOB_INDEX_PADDED\"
JOB_FILESIZE_Q=\$(stat -c%s \$MYJOBQ)
JOB_FILESIZE_B=\$(stat -c%s \$MYJOBB)
#if some Q have completed:
if [ -s \"\$MYJOBQ.completed\" ] ; then
JOB_COMPLETED_FILESIZE_Q=\$(stat -c%s \"\$MYJOBQ.completed\")
# if not all have completed then run both Q and B
if [ \"\$JOB_FILESIZE_Q\" -gt \"\$JOB_COMPLETED_FILESIZE_Q\" ] ; then
PBS_JOB4=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_CHRYSALIS.sh\" \`
if [[ ! \$PBS_JOB4 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_CHRYSALIS.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_C++
echo \$PBS_JOB4 >> \"\$JOB_CHRYSALIS.jobnames\"
echo \$PBS_JOB4 >> jobnumbers.out ;
PBS_JOB5=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED -W depend=afterok:$PBS_JOB4 \"\$JOB_BUTTERFLY.sh\" \`
if [[ ! \$PBS_JOB5 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED -W depend=afterok:$PBS_JOB4 \"\$JOB_BUTTERFLY.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_B++
echo \$PBS_JOB5 >> \"\$JOB_BUTTERFLY.jobnames\"
echo \$PBS_JOB5 >> jobnumbers.out ;
if [ \$(( \$JOBID % 20 )) -eq 0 ] ; then
echo Submitting up to 20 Quantify and/or Butterfly jobs
sleep 3 # be nice
fi
# else all Q have completed; have B completed?
else
# if at least some B have completed
if [ -s \"\$MYJOBB.completed\" ] ; then
JOB_COMPLETED_FILESIZE_B=\$(stat -c%s \"\$MYJOBB.completed\" )
# if not all, run them with no dependency (Q has completed)
if [ \"\$JOB_FILESIZE_B\" -gt \"\$JOB_COMPLETED_FILESIZE_B\" ] ; then
PBS_JOB5=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_BUTTERFLY.sh\" \`
if [[ ! \$PBS_JOB5 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_BUTTERFLY.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_B++
echo \$PBS_JOB5 >> \"\$JOB_BUTTERFLY.jobnames\"
echo \$PBS_JOB5 >> jobnumbers.out ;
if [ \$(( \$JOBID % 20 )) -eq 0 ] ; then
echo Submitting up to 20 Quantify and/or Butterfly jobs
sleep 3 # be nice
fi
fi
# else no Q have completed; run them without dependency
else
PBS_JOB5=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_BUTTERFLY.sh\" \`
if [[ ! \$PBS_JOB5 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_BUTTERFLY.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_B++
echo \$PBS_JOB5 >> \"\$JOB_BUTTERFLY.jobnames\"
echo \$PBS_JOB5 >> jobnumbers.out ;
if [ \$(( \$JOBID % 20 )) -eq 0 ] ; then
echo Submitting up to 20 Quantify and/or Butterfly jobs
sleep 3 # be nice
fi
fi
fi
# neither Q (and thus nor B) have ever ran successfully, submit both with a dependency
else
PBS_JOB4=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_CHRYSALIS.sh\" \`
if [[ ! \$PBS_JOB4 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED \"\$JOB_CHRYSALIS.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_C++
echo \$PBS_JOB4 >> \"\$JOB_CHRYSALIS.jobnames\"
echo \$PBS_JOB4 >> jobnumbers.out ;
PBS_JOB5=\`qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED -W depend=afterok:\$PBS_JOB4 \"\$JOB_BUTTERFLY.sh\" \`
if [[ ! \$PBS_JOB5 ]]; then
echo \"Submission for qsub -v JOB_INDEX_PADDED=\$JOB_INDEX_PADDED -W depend=afterok:\$PBS_JOB4 \"\$JOB_BUTTERFLY.sh\" FAILED. Aborting...\"
exit 255
fi
let SUBMITTED_B++
echo \$PBS_JOB5 >> \"\$JOB_BUTTERFLY.jobnames\"
echo \$PBS_JOB5 >> jobnumbers.out ;
if [ \$(( \$JOBID % 20 )) -eq 0 ] ; then
echo Submitting up to 20 Quantify and/or Butterfly jobs
sleep 3 # be nice
fi
fi
done
echo Submitted \$SUBMITTED_C Chrysalis and \$SUBMITTED_B Butterfly jobs
if [[ \$SUBMITTED_B == 0 && \$SUBMITTED_C == 0 ]]; then
echo \"No Trinity jobs need to be submitted \"
if [ -s "$OUTPUTDIR"/Trinity.fasta.complete ]; then
echo \"Trinity RNA-Seq assembly is complete! Result file is present as "$OUTPUTDIR"/Trinity.fasta \"
else
echo \"Proceeding with capturing the output with this command\"
echo ' find "$OUTPUTDIR"/chrysalis -name *allProbPaths.fasta -exec cat {} \\; > "$OUTPUTDIR"/Trinity.fasta '
find "$OUTPUTDIR"/chrysalis -name *allProbPaths.fasta -exec cat {} \\; > "$OUTPUTDIR"/Trinity.fasta
touch "$OUTPUTDIR"/Trinity.fasta.complete
echo DO: rm -f "$OUTPUTDIR"/bowtie.nameSorted.sam* "$OUTPUTDIR"/both.fa* "$OUTPUTDIR"/inchworm.kmer_count "$OUTPUTDIR"/iworm_* "$OUTPUTDIR"/target* "$OUTPUTDIR"/jellyfish* "$OUTPUTDIR"/scaffolding* "$OUTPUTDIR"/*.finished "$OUTPUTDIR"/mer_counts_*
fi
fi
"
######
##### Write the above script to a file for later execution
echo "${JOBSTRING4}" | cat -> "$JOBNAME4.sh"
|
