1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
# A shell "library" providing a function to run a given command on multiple
# arguments in parallel.
#
# For best results, make sure 'parallel' is installed.
#
if type parallel >/dev/null 2>&1; then
HAVE_PARALLEL=1
else
HAVE_PARALLEL=0
fi
if type xargs >/dev/null 2>&1; then
HAVE_XARGS=1
else
HAVE_XARGS=0
fi
if type getconf >/dev/null 2>&1; then
NCPUs="$(getconf _NPROCESSORS_ONLN)"
else
NCPUs="$(grep processor /proc/cpuinfo | wc -l)"
fi
_run_using_parallel() {
max_jobs=""
if [ $# -gt 2 ]; then
max_jobs="-j$3"
fi
if [ "$2" = "-" ]; then
parallel $max_jobs "$1 {}"
else
parallel $max_jobs "$1 {}" :::: "$2"
fi
return $?
}
_run_using_xargs() {
if [ $# -gt 2 ]; then
max_jobs="-P$3"
else
max_jobs="-P$NCPUs"
fi
if [ "$2" = "-" ]; then
xargs -n1 $max_jobs "$1"
else
xargs -n1 $max_jobs -a "$2" "$1"
fi
return $?
}
_run_using_for() {
if [ $# -gt 2 ]; then
job_slots="$3"
else
job_slots="$NCPUs"
fi
if [ "$2" = "-" ]; then
input_arg="/dev/stdin"
else
input_arg="$2"
fi
failure=0
while read item; do
if [ "$job_slots" = 0 ]; then
wait -n
if [ $? != 0 ] && [ $failure = 0 ]; then
failure=1
fi
job_slots="$(expr $job_slots + 1)"
fi
"$1" "$item" &
job_slots="$(expr $job_slots - 1)"
done < $input_arg
# wait for the jobs one by one and check the exit statuses (127 means there
# are no more jobs to wait for)
wait -n
exit_status=$?
while [ $exit_status != 127 ]; do
wait -n
exit_status=$?
if [ $exit_status != 0 ] && [ $exit_status != 127 ] && [ $failure = 0 ]; then
failure=1
fi
done
return $failure
}
run_in_parallel() {
# Run the given command with the read arguments in parallel
#
# $1 -- command to run
# $2 -- path to the file to read the argument items from,
# or "-" to load from STDIN
# $3 -- OPTIONAL, maximum number of parallel jobs to run,
# defaults to the number of CPUs
# return -- 0 if all runs of the command exited with 0, 1 otherwise
#
# Reads the arguments (one arg per line) and runs the given command in
# parallel with them. One argument per command. Uses 'parallel', 'xargs' or a
# for-loop with background jobs in respective order of preference.
# we need to know if $1 is a function because 'xargs' doesn't support
# functions at all and for 'parallel' we need to export the function
if type "$1" | head -n1 | grep "$1 is a function" >/dev/null; then
IS_A_FUNC=1
else
IS_A_FUNC=0
fi
if [ $HAVE_PARALLEL = 1 ]; then
if [ $IS_A_FUNC = 1 ]; then
export -f "$1"
fi
_run_using_parallel "$@"
if [ $? = 0 ]; then
return 0
else
return 1
fi
elif [ $HAVE_XARGS = 1 ] && [ $IS_A_FUNC != 1 ]; then
_run_using_xargs "$@"
if [ $? = 0 ]; then
return 0
else
return 1
fi
else
_run_using_for "$@"
return $?
fi
}
|
