File: native.sh

package info (click to toggle)
botch 0.21-8
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 1,298,428 kB
  • sloc: xml: 11,924,948; ml: 4,497; python: 3,620; sh: 1,269; makefile: 319
file content (581 lines) | stat: -rwxr-xr-x 22,010 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
#!/bin/sh -e

# to make sure that lines like "rm $tmp/*" doesn't remove root
set -u

PROGNAME=`basename $0`
CUSTOMTMP="no"
OUTDIR="./"
KEEP="no"
NUM_CORES="1"
LATEST="no"
SELFCONTAINED="no"
OPTUNIV="no"
NODROP="no"
STRONG="no"
NOFIXPOINT="no"
CLEAN="no"
TIMERS=
DEVELOP=
AVAILABLE=
ALLOWSRCMISMATCH=
IGNORESRCLESSBIN=
OPTGRAPH=
SAPSB=
ONLINE=
VERBOSE=
DROPBDINDEP=
DEBUG=

usage() {
	echo "usage: $PROGNAME [OPTIONS] [architecture Packages Sources]" >&2
	echo >&2
	echo " -h, --help             Show this help message and exit" >&2
	echo " -k, --keep             Keep the temporary files" >&2
	echo " -c, --self-contained   Do not work on the full repository but on a smaller" >&2
	echo "                        self-contained repository" >&2
	echo " -O, --optuniv          Calculate a self-contained repository with a minimal" >&2
	echo "                        number of source packages" >&2
	echo " -G, --optgraph         Calculate a dependency graph where each installation" >&2
	echo "                        set contains the minimal number of unavailable" >&2
	echo "                        binary packages" >&2
	echo " -T, --timers           time all program executions" >&2
	echo " -m, --allowsrcmismatch Allow binary packages to be associated with a source " >&2
	echo "                        package of a different version than themselves" >&2
	echo " -e, --ignoresrclessbin Ignore binary packages without a source package" >&2
	echo " -p, --no-drop          Do not drop known droppable build dependencies" >&2
	echo " -s, --strong           Calculate strong dependencies" >&2
	echo " -b, --sapsb            Calculate strong articulation points and bridges" >&2
	echo " -f, --no-fixpoint      Do not run fixpoint algorithm" >&2
	echo " -n, --clean            Clean the repository beforehand" >&2
	echo " -l, --latest           Select only the highest package versions" >&2
	echo " -w, --online           Produce stat.html for online consumption" >&2
	echo " -a, --available=FILE   Deb822 list of available binary packages" >&2
	echo " -o, --output=DIR       Output directory. Default is the current directory" >&2
	echo " -t, --tmp=DIR          Temporary directory. Default is created by mktemp(1). Implies --keep." >&2
	echo " -j, --jobs=NUM         Number of threads or processes to run in parallel" >&2
	echo "                        where applicable" >&2
	echo " -I, --deb-drop-b-d-indep   Drop Build-Depends-Indep dependencies" >&2
	echo " -v, --verbose          Be more verbose" >&2
	echo " -d, --debug            Maximum verbosity" >&2
	echo " -D, --develop          Execute tools from the source checkout instead of \$PATH" >&2
}

die() {
	echo " + error $@"
	if [ "$KEEP" = "yes" ]; then
		echo " + temporary files stored in $tmp" >&2
	else
		rm -f "$tmp"/*
		rm -fd $tmp
	fi
	exit 1
}

run() {
	if [ -n "$TIMERS" ]; then
		time --format "%e seconds" "$@"
	else
		"$@"
	fi
}

# clean up when sighup, sigint or sigterm is received
trap "die \"got signal\"" 1 2 15

getopt -T > /dev/null && exitcode=0 || exitcode=$?
if [ $exitcode -eq 4 ]; then
	# GNU enhanced getopt is available
	ARGS=`getopt --long help,keep,self-contained,optuniv,optgraph,timers,allowsrcmismatch,ignoresrclessbin,no-drop,strong,sapsb,no-fixpoint,clean,latest,online,available:,output:,tmp:,jobs:,deb-drop-b-d-indep,verbose,debug,develop --options hkcOGTmepsbfnlwa:o:t:j:IvdD -- "$@"`
else
	# Original getopt is available
	ARGS=`getopt hkcOGTmepsbfnlwa:o:t:j:IvdD "$@"`
fi

if [ $? -ne 0 ]; then
	exit 1
fi

eval set -- $ARGS

while [ $# -gt 0 ]; do
	case "$1" in
		-h | --help)     usage; exit 0;;
		-k | --keep)     KEEP="yes";;
		-c | --self-contained) SELFCONTAINED="yes";;
		-O | --optuniv)  OPTUNIV="yes";;
		-G | --optgraph) OPTGRAPH=--optgraph;;
		-T | --timers)   TIMERS=--timers;;
		-m | --allowsrcmismatch) ALLOWSRCMISMATCH=--allowsrcmismatch;;
		-e | --ignoresrclessbin) IGNORESRCLESSBIN=--ignoresrclessbin;; 
		-p | --no-drop)  NODROP="yes";;
		-s | --strong)   STRONG="yes";;
		-b | --sapsb)    SAPSB=--sapsb;;
		-f | --no-fixpoint) NOFIXPOINT="yes";;
		-n | --clean)    CLEAN="yes";;
		-l | --latest)   LATEST="yes";;
		-w | --online)   ONLINE="--online";;
		-a | --available) AVAILABLE="$2"; shift;;
		-o | --output)   OUTDIR="$2"; shift;;
		-t | --tmp)      CUSTOMTMP="yes"; KEEP="yes"; tmp="$2"; shift;;
		-j | --jobs)     NUM_CORES="$2"; shift;;
		-v | --verbose)  VERBOSE=--verbose;;
		-d | --debug)    set -x; DEBUG=--debug;;
		-D | --develop)  DEVELOP=--develop;;
		-I | --deb-drop-b-d-indep) DROPBDINDEP=--deb-drop-b-d-indep;;
		--)              shift; break;;
	esac
	shift
done

getocamlexec() {
	execname="$1"
	for ext in "native" "p.native" "byte" "d.byte"; do
		path="./$execname.$ext"
		if [ -f "$path" ]; then
			echo "$path"
			return
		fi
	done
	die "cannot find executable for $execname"
}

bin_coinstall=dose-deb-coinstall
bin_buildcheck=dose-builddebcheck
if [ -n "$DEVELOP" ]; then
	droppable=./droppable
	bin_annotate_strong=$(getocamlexec annotate-strong)
	bin_bin2src=$(getocamlexec bin2src)
	bin_build_fixpoint=$(getocamlexec build-fixpoint)
	bin_buildgraph_to_srcgraph=$(getocamlexec buildgraph2srcgraph)
	bin_clean_repository=$(getocamlexec clean-repository)
	bin_create_graph=$(getocamlexec create-graph)
	bin_find_fvs=$(getocamlexec find-fvs)
	bin_optuniv=$(getocamlexec optuniv)
	bin_partial_order=$(getocamlexec partial-order)
	bin_print_stats=$(getocamlexec print-stats)
	bin_src2bin=$(getocamlexec src2bin)
	bin_extract_scc=./tools/extract-scc.py
	bin_fasofstats=./tools/fasofstats.py
	bin_graph_info=./tools/graph-info.py
	bin_packages_difference=./tools/packages-difference.py
	bin_packages_intersection=./tools/packages-intersection.py
	bin_packages_union=./tools/packages-union.py
	bin_profile_build_fvs=./tools/profile-build-fvs.py
	bin_latest_version=./tools/latest-version.py
	bin_stat_html=./tools/stat-html.py
	bin_download_pkgsrc=./tools/download-pkgsrc.sh
	bin_buildgraph2packages=./tools/buildgraph2packages.py
	bin_y_u_no_bootstrap=./tools/y-u-no-bootstrap.sh
else
	droppable=/usr/share/botch/droppable
	bin_annotate_strong=botch-annotate-strong
	bin_bin2src=botch-bin2src
	bin_build_fixpoint=botch-build-fixpoint
	bin_buildgraph_to_srcgraph=botch-buildgraph2srcgraph
	bin_clean_repository=botch-clean-repository
	bin_create_graph=botch-create-graph
	bin_find_fvs=botch-find-fvs
	bin_optuniv=botch-optuniv
	bin_partial_order=botch-partial-order
	bin_print_stats=botch-print-stats
	bin_src2bin=botch-src2bin
	bin_extract_scc=botch-extract-scc
	bin_fasofstats=botch-fasofstats
	bin_graph_info=botch-graph-info
	bin_packages_difference=botch-packages-difference
	bin_packages_intersection=botch-packages-intersection
	bin_packages_union=botch-packages-union
	bin_profile_build_fvs=botch-profile-build-fvs
	bin_latest_version=botch-latest-version
	bin_stat_html=botch-stat-html
	bin_download_pkgsrc=botch-download-pkgsrc
	bin_buildgraph2packages=botch-buildgraph2packages
	bin_y_u_no_bootstrap=botch-y-u-no-bootstrap
fi

case $# in
	0)
		archpkgsrc=`$bin_download_pkgsrc`
		set $archpkgsrc
		NATIVEARCH="$1"
		packages="$2"
		sources="$3"
		;;
	3)
		NATIVEARCH="$1"
		packages="$2"
		sources="$3"
		;;
	*)
		usage
		exit 1
		;;
esac

if [ $CUSTOMTMP = "yes" ]; then
	mkdir -p "$tmp" || die "cannot mkdir $tmp"
else
	tmp=`mktemp --directory`
fi

mkdir -p "$OUTDIR" || die "cannot mkdir $OUTDIR"

if [ "$KEEP" != "yes" ] && [ -n "$(ls -A $tmp)" ]; then
	echo "$tmp is not empty and you did not specify --keep" >&2
	echo "refusing to run and delete that directory" >&2
	exit 1
fi

if [ "$LATEST" = "yes" ]; then
	# we can't just blindly select packages of latest version because this
	# might break versioned dependencies, therefore we do not do this with
	# the --clean option per default
	echo " + selecting latest packages" >&2

	run $bin_latest_version $VERBOSE "$packages" - > "$tmp/packages-$NATIVEARCH-latest"
	packages="$tmp/packages-$NATIVEARCH-latest"

	echo " + running grep-dctrl" >&2

	run grep-dctrl --not -X -FArchitecture all "$packages" > "$packages.noarchall" || die "grep-dctrl failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages.noarchall" "$sources" \
		> "$tmp/sources-latest" || die "bin2src failed"
	sources="$tmp/sources-latest"

fi


if [ "$CLEAN" = "yes" ]; then
	echo " + cleaning repository" >&2

	run $bin_clean_repository $VERBOSE $DROPBDINDEP --deb-native-arch=$NATIVEARCH \
		--progress $ALLOWSRCMISMATCH "$packages" "$sources" \
		> "$tmp/packages-$NATIVEARCH" || {
			$bin_y_u_no_bootstrap $ALLOWSRCMISMATCH $NATIVEARCH $VERBOSE $TIMERS $DEBUG --output="$OUTDIR" --keep --tmp="$tmp" $DEVELOP "$packages" "$sources" || true
			die "clean-repository failed"
		}
	packages="$tmp/packages-$NATIVEARCH"

	echo " + running grep-dctrl" >&2

	run grep-dctrl --not -X -FArchitecture all "$packages" > "$packages.noarchall" || die "grep-dctrl failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages.noarchall" "$sources" \
		> "$tmp/sources-clean" || die "bin2src failed"
	sources="$tmp/sources-clean"
fi

if [ "$OPTUNIV" = "yes" ]; then
	echo " + running grep-dctrl..." >&2

	run grep-dctrl --exact-match --field Package build-essential "$packages" \
		| ./tools/latest-version.py - - > $tmp/build-essential || die "grep-dctrl failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		$tmp/build-essential "$sources" > $tmp/build-essential-src || die "bin2src failed"

	echo " + running optuniv..." >&2

	run $bin_optuniv $VERBOSE $DROPBDINDEP --deb-native-arch=$NATIVEARCH \
		--bg "$sources" "$packages" $tmp/build-essential-src > "$tmp/opt-packages" \
		|| die "optuniv failed"
	packages="$tmp/opt-packages"

	echo " + running grep-dctrl" >&2

	run grep-dctrl --not -X -FArchitecture all "$packages" > "$packages.noarchall" || die "grep-dctrl failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
		"$packages.noarchall" "$sources" > "$tmp/opt-sources" \
		|| die "bin2src failed"
	sources="$tmp/opt-sources"

	echo " + running buildcheck..." >&2

	# if buildcheck succeeds and since the source list was generated from the
	# binary list, at this point we know that the pair of opt-packages and
	# opt-sources is indeed a valid self-contained repository
	run $bin_buildcheck $VERBOSE --explain --deb-drop-b-d-indep \
		--failures --deb-native-arch=$NATIVEARCH "$packages" \
		"$sources" || die "buildcheck failed"
fi

echo " + running grep-dctrl..." >&2

# these source packages are known to be cross compilable
#zcat "$sources" | \
#    grep-dctrl -P -e '^(eglibc|pcre3|attr|acl|zlib|hostname|libsepol|gpm|ifupdown|insserv|module-init-tools|python-defaults|libselinux|ncurses|bash|libpciaccess|libpng|slang|base-files|mawk|makedev|make|xz-utils|dpkg|netbase|base-files|time|manpages|libxau|libxdmcp|libpthread-stubs|diffutils|tar|cpio|diffutils|gmp|gdbm|sed|tzdata|base-passwd|coreutils|findutils|libnfnetlink|libffi|iptables|libelf|debconf|debianutils|ucf|xtrans|x11proto-kb|x11proto-input|x11proto-render|x11proto-xf86bigfont|x11proto-core|expat|net-tools|tcl8.5|tcp-wrappers|ubuntu-keyring|scowl|postgresql-common|openslp-dfsg|procps|psmisc|phpmyadmin|libusb|mcrypt|pixman|libice|bzip2|flex|db4.8|dash|iptables|keyutils|less|sysvinit|nano|db-defaults|grep|dbconfig-comon|linux-atm|util-linux|e2fsprogs|ustr|kmod|dbus|libsm|cunit|glib2.0|udev|libnih|json-c|openssl|initramfs-tools|libgcrypt11|libidn|wget|sysfsutils|iputils|ppl|binutils|busybox|libtasn1-3|libfribidi|libjson0|upstart|tcl8.5|sqlite3|db|cracklib2|pam|libcap2|libsemanage|shadow|iproute|ppl|cloog-ppl|libbsd|libedit|dbus|libgpg-error|popt|p11-kit|gnutls26|libxau|check|libxcb1|libx11|libfreetype|libfontconfig|libxrender|libxext|libxt|lzo2|cairo|libdatrie|libthai|libxft|pango1.0|libdrm|plymouth|perl|libjpeg-turbo|initramfs-tools|debconf|lsb|tzdata|makedev|pam|mountall|readline|cron|glib2.0|nspr|klibc|nano|less|gcc-4.7)$' > $OUTDIR/crossed-known
truncate --size=0 "$tmp/crossed-known"

zcat -f "$packages" | \
    run grep-dctrl -X \( \
    -FPackage build-essential --or \
    -FPackage apt --or \
    -FPackage sysvinit-core --or \
    -FPackage debhelper --or \
    -FEssential yes \
\) > "$tmp/minimal" || die "zcat failed"

echo " + running coinstall..." >&2

run $bin_coinstall $VERBOSE \
    --bg="$packages" --fg="$tmp/minimal" --deb-native-arch=$NATIVEARCH \
    > "$tmp/minimal-$NATIVEARCH" || die "coinstall failed"

echo " + sorting..." >&2

run $bin_packages_union $VERBOSE "$tmp/minimal-$NATIVEARCH" \
	"$tmp/minimal-$NATIVEARCH"  || die "union failed"

echo " + running grep-dctrl" >&2

run grep-dctrl --not -X -FArchitecture all "$tmp/minimal-$NATIVEARCH" > "$tmp/minimal-$NATIVEARCH.noarchall" || die "grep-dctrl failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	"$tmp/minimal-$NATIVEARCH.noarchall" "$sources" \
	> "$tmp/minimal-src" || die "bin2src failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/crossed-known" "$tmp/minimal-src" \
	"$tmp/crossed-src" || die "union failed"

echo " + running grep-dctrl..." >&2

zcat -f "$packages" | run grep-dctrl -FArchitecture all \
	> "$tmp/available-all" || die "grep-dctrl failed"

echo " + running src2bin..." >&2

run $bin_src2bin $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
    $IGNORESRCLESSBIN --bg "$sources" -A "$tmp/available-all" \
    "$packages" "$tmp/crossed-src" \
    > "$tmp/available-crossed" || die "src2bin failed"

if [ "$SELFCONTAINED" = "yes" ]; then
	echo " + union..." >&2

	run $bin_packages_union $VERBOSE "$tmp/available-crossed" \
		"$tmp/available-all" "$tmp/available" || die "union failed"

	echo " + creating a self-contained repository graph..." >&2

	# we ignore essential because it is already part of $tmp/minimal-$NATIVEARCH
	# this also means that we have to create the union of the results of this with
	# $tmp/minimal-$NATIVEARCH
	# the graph is started from all source packages that have been crossed before
	# and are thus creating the binary packages in $tmp/minimal-$NATIVEARCH
	run $bin_create_graph $VERBOSE $DROPBDINDEP --jobs=$NUM_CORES \
	    --deb-ignore-essential --progress --timers \
	    -A "$tmp/available" $ALLOWSRCMISMATCH --deb-native-arch=$NATIVEARCH \
	    --bg $sources $packages $tmp/crossed-src \
	    > "${OUTDIR}/selfcontained_repo.xml" || die "create graph failed"

	echo " + running buildgraph2packages" >&2

	run $bin_buildgraph2packages "${OUTDIR}/selfcontained_repo.xml" $packages \
		> "$tmp/minimal-closure" || die "buildgraph2packages failed"

	echo " + union..." >&2

	run $bin_packages_union $VERBOSE "$tmp/minimal-closure" \
		"$tmp/minimal-$NATIVEARCH" "$tmp/minimal-closure" \
		|| die "union failed"

	echo " + running grep-dctrl" >&2

	run grep-dctrl --not -X -FArchitecture all "$tmp/minimal-closure" > "$tmp/minimal-closure.noarchall" || die "grep-dctrl failed"

	echo " + running bin2src..." >&2

	run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	       "$tmp/minimal-closure.noarchall" "$sources" \
	       > "$tmp/minimal-closure-src" || die "bin2src failed"

	packages="$tmp/minimal-closure"
	sources="$tmp/minimal-closure-src"

	echo " + running buildcheck..." >&2

	# if buildcheck succeeds and since the source list was generated from the
	# binary list, at this point we know that the pair of minimal-closure and
	# minimal-closure-src is indeed a valid self-contained repository
	run $bin_buildcheck $VERBOSE --explain-minimal --deb-drop-b-d-indep \
		--failures --deb-native-arch=$NATIVEARCH "$packages" \
		"$sources" || die "buildcheck failed"

	echo " + intersecting..." >&2

	# the set of available packages must be a subset of minimal-closure
	run $bin_packages_intersection $VERBOSE "$tmp/available-crossed" "$packages" \
		"$tmp/available-crossed" || die "intersecting failed"

	echo " + running grep-dctrl..." >&2

	run grep-dctrl -FArchitecture all "$packages" \
		> "$tmp/available-all" || die "grep-dctrl failed"
fi

echo " + union..." >&2

if [ -n "$AVAILABLE" -a -f "$AVAILABLE" -a -s "$AVAILABLE" ]; then
	run $bin_packages_union $VERBOSE "$tmp/available-crossed" \
		"$tmp/available-all" "$AVAILABLE" \
		"$tmp/available-crossed-all" || die "union failed"
else
	run $bin_packages_union $VERBOSE "$tmp/available-crossed" \
		"$tmp/available-all" \
		"$tmp/available-crossed-all" || die "union failed"
fi

echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$packages" "$tmp/available-crossed-all" \
	"$tmp/not-available" || die "difference failed"

echo " + bin2src..." >&2
run $bin_bin2src $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	"$tmp/not-available" "$sources" \
	> "$tmp/tocompile" || die "bin2src failed"

if [ "$NOFIXPOINT" = "yes" ]; then
	cp "$tmp/available-crossed-all" "$tmp/available"
else
	echo " + running build-fixpoint..." >&2

	run $bin_build_fixpoint $VERBOSE $DROPBDINDEP --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	    -A "$tmp/available-crossed-all" --output-order="${OUTDIR}/order1.lst" "$packages" \
	    "$tmp/tocompile" \
	    > "$tmp/compilable-src" || die "fixpoint failed"

	echo " + running src2bin..." >&2

	run $bin_src2bin $VERBOSE --deb-native-arch=$NATIVEARCH $ALLOWSRCMISMATCH \
	    $IGNORESRCLESSBIN --bg "$sources" -A "$tmp/available-crossed-all" \
	    "$packages" "$tmp/compilable-src" \
	    > "$tmp/compilable" || die "src2bin failed"

	echo " + union..." >&2

	run $bin_packages_union $VERBOSE "$tmp/available-crossed-all" \
		"$tmp/compilable" "$tmp/available" || die "union failed"

	echo " + difference..." >&2
	run $bin_packages_difference $VERBOSE "$tmp/tocompile" \
		"$tmp/compilable-src" "$tmp/tocompile" || die "difference failed"
fi

# from this point on, everything that reads GraphML must read in the same
# Packages and Sources files to create the exact same cudf universe
UNIVERSE="--deb-native-arch=$NATIVEARCH --bg $sources $packages $tmp/tocompile"

echo " + running distgraph..." >&2

# we can use deb-ignore-essential here because the set of available packages
# was made sure to contain a coinstallation set of essential earlier in this
# script
run $bin_create_graph $VERBOSE $DROPBDINDEP $OPTGRAPH --jobs=$NUM_CORES \
    --deb-ignore-essential --progress --timers \
    -A "$tmp/available" $ALLOWSRCMISMATCH $UNIVERSE \
    > "${OUTDIR}/buildgraph.xml" || die "create graph failed"

echo " + running buildgraph2srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph.xml" || die "buildgraph2srcgraph failed"

if [ "$STRONG" = "yes" ]; then
	echo " + running annotate-strong..." >&2
	run $bin_annotate_strong $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph.xml" \
		"${OUTDIR}/srcgraph.xml" $UNIVERSE || die "annotate-strong failed"
fi

echo " + printing stats and finding feedback arc set..." >&2

if [ "$NODROP" = "yes" ]; then
	droplist=
else
	droplist="--remove-weak=${droppable}/weak-build-dependencies.list --remove-reduced=${droppable}/pehjota.list,${droppable}/thorsten-glaser.list,${droppable}/daniel-schepler.list,${droppable}/gentoo.list,${droppable}/wookey.list,${droppable}/selfcycles.list"
fi

run $bin_print_stats $VERBOSE $DROPBDINDEP --max-length=6 --max-length-fas=8 $ALLOWSRCMISMATCH \
	$SAPSB $droplist -A "$tmp/available" "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/srcgraph.xml" $UNIVERSE | ydump > "${OUTDIR}/stats.json" || die "print-stats failed"

echo " + generating html..." >&2

run $bin_stat_html $VERBOSE $ONLINE "${OUTDIR}/stats.json" > "${OUTDIR}/stats.html" || die "stat-html failed"

echo " + extracting the calculated feedback arc set..." >&2

run $bin_fasofstats $VERBOSE "${OUTDIR}/stats.json" | sort > "${OUTDIR}/remove.list" || die "fasofstats failed"

echo " + extract strongly connected components..." >&2

run $bin_extract_scc $VERBOSE --outdir=${OUTDIR} \
	--outfnameattr=cudfname --outfnamevert type:src \
	"${OUTDIR}/buildgraph.xml" cyclic \
	| sort > "${OUTDIR}/fas" || die "extract-scc failed"

echo " + calculate feedback vertex sets..." >&2

if [ "$NODROP" = "yes" ]; then
       alldrop="--remove-reduced=${OUTDIR}/remove.list"
else
       alldrop="$droplist,${OUTDIR}/remove.list"
fi

> "${OUTDIR}/feedback_vertex_set.list"
while read g; do
	echo " + processing $g..." >&2
	run $bin_find_fvs $VERBOSE $DROPBDINDEP $alldrop "${OUTDIR}/$g" $UNIVERSE \
		|| die "find-fvs failed"
done < "${OUTDIR}/fas" | sort > "${OUTDIR}/feedback_vertex_set.list"

echo " + profile build selected source packages..." >&2

run $bin_profile_build_fvs $VERBOSE $alldrop "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/feedback_vertex_set.list" \
	> "${OUTDIR}/buildgraph_acyclic.xml" || die "profile-build-fvs failed"

echo " + convert to srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph_acyclic.xml" || die "buildgraph2srcgraph failed"

echo " + calculate partial order..." >&2

run $bin_partial_order $VERBOSE $DROPBDINDEP "${OUTDIR}/srcgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/order2.lst" || die "partial-order failed"

echo " + build order is saved in ${OUTDIR}/order1.lst and ${OUTDIR}/order2.lst" >&2
echo " + an overview of the dependency situation is now available in ${OUTDIR}/stats.html" >&2
echo " + the data from which stats.html was generated is saved in ${OUTDIR}/stats.json" >&2
echo " + the build order was generated using the feedback arc set in ${OUTDIR}/remove.list" >&2
echo " + without the content of remove.list, the remaining strongly connected components are:" >&2

echo -e "graph.xml\ttype\tvertices\tedges\tscc"
while read g; do
	printf "$g\t"
	run $bin_graph_info $VERBOSE "${OUTDIR}/$g" || die "graph-info failed"
done < "${OUTDIR}/fas"

if [ "$KEEP" = "yes" ]; then
	echo " + temporary files stored in $tmp" >&2
else
	rm -f "$tmp"/*
	rm -fd $tmp
fi