#!/bin/sh -e

# to make sure that lines like "rm $tmp/*" doesn't remove root
set -u

PROGNAME=`basename $0`
CUSTOMTMP="no"
OUTDIR="./"
KEEP="no"
NUM_CORES="1"
DEVELOP="no"
TIMERS=
ALLOWSRCMISMATCH=
IGNORESRCLESSBIN=
SAPSB=
ONLINE=
OPTGRAPH=
DROPBDINDEP=
VERBOSE=

usage() {
	echo "usage: $PROGNAME [OPTIONS] [buildarch hostarch buildpackages sources]" >&2
	echo >&2
	echo " -h, --help       Show this help message and exit" >&2
	echo " -k, --keep       Keep the temporary files" >&2
	echo " -w, --online     Produce stat.html for online consumption" >&2
	echo " -o, --output=DIR Output directory. Default is the current directory" >&2
	echo " -t, --tmp=DIR    Temporary directory. Default is created by mktemp(1). Implies --keep." >&2
	echo " -G, --optgraph   Calculate a dependency graph where each installation" >&2
	echo "                  set contains the minimal number of unavailable" >&2
	echo "                  binary packages" >&2
	echo " -T, --timers           time all program executions" >&2
	echo " -m, --allowsrcmismatch Allow binary packages to be associated with a source " >&2
	echo "                        package of a different version than themselves" >&2
	echo " -e, --ignoresrclessbin Ignore binary packages without a source package" >&2
	echo " -b, --sapsb            Calculate strong articulation points and bridges" >&2
	echo " -j, --jobs=NUM   Number of threads or processes to run in parallel" >&2
	echo "                  where applicable" >&2
	echo " -I, --deb-drop-b-d-indep Drop Build-Depends-Indep dependencies" >&2
	echo " -v, --verbose    Be more verbose" >&2
	echo " -d, --debug      Maximum verbosity" >&2
	echo " -D, --develop    Execute tools from the source checkout instead of \$PATH" >&2
}

die() {
        echo " + error $@"
        if [ "$KEEP" = "yes" ]; then
                echo " + temporary files stored in $tmp" >&2
        else
                rm -f "$tmp"/*
                rm -fd $tmp
        fi
        exit 1
}

run() {
        if [ -n "$TIMERS" ]; then
                time --format "%e seconds" "$@"
        else
                "$@"
        fi
}

# clean up when sighup, sigint or sigterm is received
trap "die \"got signal\"" 1 2 15

getopt -T > /dev/null && exitcode=0 || exitcode=$?
if [ $exitcode -eq 4 ]; then
	# GNU enhanced getopt is available
	ARGS=`getopt --long help,output:,tmp:,optgraph,timers,allowsrcmismatch,ignoresrclessbin,sapsb,online,jobs:,deb-drop-b-d-indep,verbose,debug,develop --options ho:t:GTmebwj:IvdD -- "$@"`
else
	# Original getopt is available
	ARGS=`getopt ho:t:GTmebwj:IvdD "$@"`
fi

if [ $? -ne 0 ]; then
	exit 1
fi

eval set -- $ARGS

while [ $# -gt 0 ]; do
	case "$1" in
		-h | --help)    usage; exit 0;;
		-k | --keep)    KEEP="yes";;
		-w | --online)  ONLINE="--online";;
		-o | --output)  OUTDIR="$2"; shift;;
		-t | --tmp)     CUSTOMTMP="yes"; KEEP="yes"; tmp="$2"; shift;;
		-G | --optgraph) OPTGRAPH=--optgraph;;
		-T | --timers)   TIMERS=--timers;;
		-m | --allowsrcmismatch) ALLOWSRCMISMATCH=--allowsrcmismatch;;
		-e | --ignoresrclessbin) IGNORESRCLESSBIN=--ignoresrclessbin;;
		-b | --sapsb)    SAPSB=--sapsb;;
		-j | --jobs)    NUM_CORES="$2"; shift;;
		-I | --deb-drop-b-d-indep) DROPBDINDEP=--deb-drop-b-d-indep;;
		-v | --verbose) VERBOSE=--verbose;;
		-d | --debug)   set -x;;
		-D | --develop) DEVELOP="yes";;
		--)             shift; break;;
	esac
	shift
done

getocamlexec() {
	execname="$1"
	for ext in "native" "p.native" "byte" "d.byte"; do
		path="./$execname.$ext"
		if [ -f "$path" ]; then
			echo "$path"
			return
		fi
	done
	die "cannot find executable for $execname"
}

bin_coinstall=dose-deb-coinstall
bin_buildcheck=dose-builddebcheck
if [ "$DEVELOP" = "yes" ]; then
	droppable=./droppable
	bin_bin2src=$(getocamlexec bin2src)
	bin_build_fixpoint=$(getocamlexec build-fixpoint)
	bin_buildgraph_to_srcgraph=$(getocamlexec buildgraph2srcgraph)
	bin_clean_repository=$(getocamlexec clean-repository)
	bin_create_graph=$(getocamlexec create-graph)
	bin_find_fvs=$(getocamlexec find-fvs)
	bin_partial_order=$(getocamlexec partial-order)
	bin_print_stats=$(getocamlexec print-stats)
	bin_src2bin=$(getocamlexec src2bin)
	bin_buildcheck_more_problems=$(getocamlexec buildcheck-more-problems)
	bin_add_arch=./tools/add-arch.py
	bin_apply_ma_diff=./tools/apply-ma-diff.py
	bin_convert_arch=./tools/convert-arch.py
	bin_extract_scc=./tools/extract-scc.py
	bin_fasofstats=./tools/fasofstats.py
	bin_fix_cross_problems=./tools/fix-cross-problems.py
	bin_graph_info=./tools/graph-info.py
	bin_ma_diff=./tools/ma-diff.py
	bin_packages_difference=./tools/packages-difference.py
	bin_packages_union=./tools/packages-union.py
	bin_profile_build_fvs=./tools/profile-build-fvs.py
	bin_stat_html=./tools/stat-html.py
	bin_download_pkgsrc=./tools/download-pkgsrc.sh
	bin_buildgraph2packages=./tools/buildgraph2packages.py
else
	droppable=/usr/share/botch/droppable
	bin_bin2src=botch-bin2src
	bin_build_fixpoint=botch-build-fixpoint
	bin_buildgraph_to_srcgraph=botch-buildgraph2srcgraph
	bin_clean_repository=botch-clean-repository
	bin_create_graph=botch-create-graph
	bin_find_fvs=botch-find-fvs
	bin_partial_order=botch-partial-order
	bin_print_stats=botch-print-stats
	bin_src2bin=botch-src2bin
	bin_buildcheck_more_problems=botch-buildcheck-more-problems
	bin_add_arch=botch-add-arch
	bin_apply_ma_diff=botch-apply-ma-diff
	bin_convert_arch=botch-convert-arch
	bin_extract_scc=botch-extract-scc
	bin_fasofstats=botch-fasofstats
	bin_fix_cross_problems=botch-fix-cross-problems
	bin_graph_info=botch-graph-info
	bin_ma_diff=botch-ma-diff
	bin_packages_difference=botch-packages-difference
	bin_packages_union=botch-packages-union
	bin_profile_build_fvs=botch-profile-build-fvs
	bin_stat_html=botch-stat-html
	bin_download_pkgsrc=botch-download-pkgsrc
	bin_buildgraph2packages=botch-buildgraph2packages
fi

case $# in
	0)
		BUILDARCH=amd64
		HOSTARCH=armhf
		suite=sid
		date=20140101T000000Z
		$bin_download_pkgsrc $BUILDARCH 2014
		buildpackages=$suite-$BUILDARCH-packages-$date.gz
		hostpackages=$suite-$HOSTARCH-packages-$date.gz
		sources=$suite-sources-$date.gz
		;;
	4)
		BUILDARCH=$1
		HOSTARCH=$2
		buildpackages=$3
		sources=$4
		;;
	*)
		usage
		exit 1
		;;
esac

if [ $CUSTOMTMP = "yes" ]; then
	mkdir -p "$tmp" || die "cannot mkdir $tmp"
else
	tmp=`mktemp --directory`
fi

mkdir -p "$OUTDIR" || die "cannot mkdir $OUTDIR"

if [ "$KEEP" != "yes" ] && [ -n "$(ls -A $tmp)" ]; then
       echo "$tmp is not empty and you did not specify --keep" >&2
       echo "refusing to run and delete that directory" >&2
       exit 1
fi

echo " + running clean repository" >&2

# clean up buildpackages repository
$bin_clean_repository $VERBOSE $DROPBDINDEP --deb-native-arch=$BUILDARCH \
	--progress $buildpackages $sources \
	> $tmp/buildpackages_clean
buildpackages="$tmp/buildpackages_clean"

echo " + running grep-dctrl" >&2

run grep-dctrl --not -X -FArchitecture all "$buildpackages" > "$buildpackages.noarchall" || die "grep-dctrl failed"

echo " + running bin2src" >&2

run $bin_bin2src $VERBOSE --deb-native-arch=$BUILDARCH $buildpackages.noarchall $sources \
	> $tmp/sources_clean || die "bin2src failed"
sources="$tmp/sources_clean"

hostpackages="$tmp/hostpackages"

# we do this because version skew kills
run $bin_convert_arch $VERBOSE $BUILDARCH $HOSTARCH $buildpackages $hostpackages || die "convert arch failed"
# we do this because some of the source packages might not compile for our hostarch
run $bin_add_arch $VERBOSE $HOSTARCH $sources $tmp/tmp_sources || die "add arch failed"
mv $tmp/tmp_sources $sources

# we start off with making a smaller package selection by first calculating a
# selfcontained repository

zcat -f "$hostpackages" | \
    run grep-dctrl -X \( \
    -FPackage build-essential --or \
    -FPackage apt --or \
    -FPackage sysvinit-core --or \
    -FPackage debhelper --or \
    -FEssential yes \
\) > "$tmp/minimal" || die "zcat failed"

echo " + running coinstall..." >&2

run $bin_coinstall $VERBOSE \
    --bg="$hostpackages" --fg="$tmp/minimal" --deb-native-arch=$HOSTARCH \
    > "$tmp/minimal-$HOSTARCH" || die "coinstall failed"

echo " + running grep-dctrl" >&2

run grep-dctrl --not -X -FArchitecture all "$tmp/minimal-$HOSTARCH" > "$tmp/minimal-$HOSTARCH.noarchall" || die "grep-dctrl failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --deb-native-arch=$HOSTARCH $ALLOWSRCMISMATCH \
        "$tmp/minimal-$HOSTARCH.noarchall" "$sources" \
        > "$tmp/crossed-src" || die "bin2src failed"

echo " + running grep-dctrl..." >&2

run grep-dctrl -FArchitecture all "$hostpackages" \
	> "$tmp/available-all" || die "grep-dctrl failed"

echo " + creating a self-contained repository graph..." >&2

# we ignore essential because it is already part of $tmp/minimal-$HOSTARCH
# this also means that we have to create the union of the results of this with
# $tmp/minimal-$HOSTARCH
# the graph is started from all source packages that have been crossed before
# and are thus creating the binary packages in $tmp/minimal-$HOSTARCH
run $bin_create_graph $VERBOSE $DROPBDINDEP --jobs=$NUM_CORES \
	--deb-ignore-essential --progress --timers \
	-A "$tmp/available-all" $ALLOWSRCMISMATCH --deb-native-arch=$HOSTARCH \
	--bg $sources $hostpackages $tmp/crossed-src \
	> "${OUTDIR}/selfcontained_repo.xml" || die "create graph failed"

echo " + running buildgraph2packages" >&2

run $bin_buildgraph2packages "${OUTDIR}/selfcontained_repo.xml" "$hostpackages" \
	> "$tmp/minimal-closure" || die "buildgraph2packages failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/minimal-closure" \
	"$tmp/minimal-$HOSTARCH" "$tmp/minimal-closure" \
	|| die "union failed"

echo " + running grep-dctrl" >&2

run grep-dctrl --not -X -FArchitecture all "$tmp/minimal-closure" > "$tmp/minimal-closure.noarchall" || die "grep-dctrl failed"

echo " + running bin2src..." >&2

run $bin_bin2src $VERBOSE --deb-native-arch=$HOSTARCH $ALLOWSRCMISMATCH \
       "$tmp/minimal-closure.noarchall" "$sources" \
       > "$tmp/minimal-closure-src" || die "bin2src failed"

hostpackages="$tmp/minimal-closure"
sources="$tmp/minimal-closure-src"

echo " + running buildcheck..." >&2

# if buildcheck succeeds and since the source list was generated from the
# binary list, at this point we know that the pair of minimal-closure and
# minimal-closure-src is indeed a valid self-contained repository
run $bin_buildcheck $VERBOSE --explain-minimal --deb-drop-b-d-indep \
	--failures --deb-native-arch=$HOSTARCH "$hostpackages" \
	"$sources" || die "buildcheck failed"

# at this point the calculation of the selfcontained repository is complete
# we now alter the repository until all its source packages can satisfy their
# cross build dependencies

cat > "$tmp/crossbuild-essential-${HOSTARCH}" << END
Package: crossbuild-essential-${HOSTARCH}
Version: 1.0
Architecture: ${BUILDARCH}
END

# save the location of the old buildpackages to do a diff later
oldbuildpackages=$buildpackages

# if ma.diff already exists, apply it
if [ -s "${OUTDIR}/ma.diff" ]; then
	run $bin_apply_ma_diff $VERBOSE "${OUTDIR}/ma.diff" $buildpackages \
		$tmp/buildpackages || die "apply-ma-diff failed"
	buildpackages=$tmp/buildpackages
	run $bin_apply_ma_diff $VERBOSE "${OUTDIR}/ma.diff" $hostpackages \
		$tmp/minimal-closure-crossable || die "apply-ma-diff failed"
	hostpackages=$tmp/minimal-closure-crossable
	#run $bin_buildcheck --explain --failures --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $hostpackages $buildpackages "$tmp/crossbuild-essential-${HOSTARCH}" $sources
fi

# fix the remaining problems
cp $buildpackages $tmp/buildpackages-fixed
cp $hostpackages $tmp/minimal-closure-crossable-fixed

run $bin_buildcheck_more_problems $VERBOSE --explain --failures \
	--deb-drop-b-d-indep --deb-native-arch=$BUILDARCH \
	--deb-host-arch=$HOSTARCH $tmp/minimal-closure-crossable-fixed \
	$tmp/buildpackages-fixed "$tmp/crossbuild-essential-${HOSTARCH}" \
	$sources \
	> $tmp/buildcheck.yaml || [ $? -lt 64 ] || die "buildcheck failed"
run $bin_fix_cross_problems $VERBOSE $tmp/buildcheck.yaml $tmp/buildpackages-fixed \
	$tmp/minimal-closure-crossable-fixed || die "fix-cross-problems failed"
run $bin_ma_diff $VERBOSE $oldbuildpackages $tmp/buildpackages-fixed \
	| sort > "${OUTDIR}/ma.diff" || true # non-zero exit is not fatal here
run $bin_packages_union $VERBOSE $tmp/buildpackages-fixed $tmp/buildpackages-fixed || die "union failed"
run $bin_packages_union $VERBOSE $tmp/minimal-closure-crossable-fixed $tmp/minimal-closure-crossable-fixed || die "union failed"

buildpackages=$tmp/buildpackages-fixed
hostpackages=$tmp/minimal-closure-crossable-fixed

echo " + running grep-dctrl..." >&2

run grep-dctrl -FArchitecture all "$hostpackages" \
	> "$tmp/available-all" || die "grep-dctrl failed"

# calculate tocompile
echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$hostpackages" "$tmp/available-all" \
	"$tmp/not-available" || die "packages difference failed"

# after we have found the packages that have to be compiled, we do not
# need host architecture packages that are m-a:foreign anymore
# from this point on, if there are any foreign/host architecture and
# m-a:foreign binary packages around, then they will be used to satisfy
# dependencies instead of using the native/build architecture m-a:forein
# binary packages
# therefore we delete them so that they are not used for any dependency
# satisfaction anymore
# we delete all m-a:foreign packages from the host package list because
# arch:all packages come from the build package list
echo " + running grep-dctrl..." >&2
run grep-dctrl -X \( --not -FArchitecture all --and --not -FMulti-Arch foreign \) "$hostpackages" \
	> "$tmp/hostpackages-no-ma-foreign" || die "grep-dctrl failed"
hostpackages="$tmp/hostpackages-no-ma-foreign"

echo " + bin2src..." >&2
run $bin_bin2src $VERBOSE --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $ALLOWSRCMISMATCH \
        "$tmp/not-available" "$sources" \
        > "$tmp/tocompile" || die "bin2src failed"

# calculate overall available (incl. build arch)
echo " + union..." >&2
run $bin_packages_union $VERBOSE "$tmp/available-all" $buildpackages \
	"$tmp/crossbuild-essential-${HOSTARCH}" "$tmp/available-all-build" || die "union failed"

# running fixpoint
echo " + running build-fixpoint..." >&2

run $bin_build_fixpoint $VERBOSE $DROPBDINDEP --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH \
	$ALLOWSRCMISMATCH -A "$tmp/available-all-build" \
	--output-order="${OUTDIR}/order1.lst" "$buildpackages" \
	"$tmp/crossbuild-essential-${HOSTARCH}" "$hostpackages" \
	"$tmp/tocompile" \
	> "$tmp/compilable-src" || die "fixpoint failed"

echo " + running src2bin..." >&2

# no need to pass available packages here because there are no arch:all
# packages in the list of hostarch Packages file
run $bin_src2bin $VERBOSE --deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH $ALLOWSRCMISMATCH \
    $IGNORESRCLESSBIN --bg "$sources" "$hostpackages" "$tmp/compilable-src" \
    > "$tmp/compilable" || die "src2bin failed"

echo " + union..." >&2

run $bin_packages_union $VERBOSE "$tmp/available-all-build" "$tmp/compilable" \
	"$tmp/available" || die "union failed"

echo " + difference..." >&2
run $bin_packages_difference $VERBOSE "$tmp/tocompile" "$tmp/compilable-src" \
	"$tmp/tocompile" || die "difference failed"


# from this point on, everything that reads GraphML must read in the same
# Packages and Sources files to create the exact same cudf universe
UNIVERSE="--deb-native-arch=$BUILDARCH --deb-host-arch=$HOSTARCH --bg $sources $buildpackages "$tmp/crossbuild-essential-${HOSTARCH}" $hostpackages $tmp/tocompile"

# we can use deb-ignore-essential here because the set of available packages
# was made sure to contain a coinstallation set of essential earlier in this
# script
run $bin_create_graph $VERBOSE $DROPBDINDEP $OPTGRAPH --progress --timers \
	--jobs=$NUM_CORES --deb-ignore-essential \
	-A "$tmp/available" $ALLOWSRCMISMATCH $UNIVERSE \
	> "${OUTDIR}/buildgraph.xml" || die "create graph failed"

echo " + running buildgraph2srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph.xml" || die "buildgraph2srcgraph failed"

droplist="--remove-weak=${droppable}/weak-build-dependencies.list --remove-reduced=${droppable}/pehjota.list,${droppable}/thorsten-glaser.list,${droppable}/daniel-schepler.list,${droppable}/gentoo.list,${droppable}/wookey.list,${droppable}/selfcycles.list"

run $bin_print_stats $VERBOSE $DROPBDINDEP --max-length=4 --max-length-fas=8 $ALLOWSRCMISMATCH \
	$SAPSB $droplist -A "$tmp/available" "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/srcgraph.xml" $UNIVERSE | ydump > "${OUTDIR}/stats.json" || die "print stats failed"

echo " + generating html..." >&2

run $bin_stat_html $VERBOSE $ONLINE "${OUTDIR}/stats.json" > "${OUTDIR}/stats.html" || die "stat-html failed"

echo " + extracting the calculated feedback arc set..." >&2

run $bin_fasofstats $VERBOSE "${OUTDIR}/stats.json" | sort > "${OUTDIR}/remove.list" || die "fasofstats failed"

echo " + extract strongly connected components..." >&2

run $bin_extract_scc $VERBOSE --outdir=${OUTDIR} \
	--outfnameattr=cudfname --outfnamevert type:src \
	"${OUTDIR}/buildgraph.xml" cyclic \
	| sort > "${OUTDIR}/fas" || die "extract scc failed"

echo " + calculate feedback vertex sets..." >&2

alldrop="$droplist,${OUTDIR}/remove.list"

echo > "${OUTDIR}/feedback_vertex_set.list"
while read g; do
	run $bin_find_fvs $VERBOSE $DROPBDINDEP $alldrop "${OUTDIR}/$g" $UNIVERSE \
		|| die "find fvs failed"
done < "${OUTDIR}/fas" | sort > "${OUTDIR}/feedback_vertex_set.list"

echo " + profile build selected source packages..." >&2


run $bin_profile_build_fvs $VERBOSE $alldrop "${OUTDIR}/buildgraph.xml" \
	"${OUTDIR}/feedback_vertex_set.list" \
	> "${OUTDIR}/buildgraph_acyclic.xml" || die "profile-build-fvs failed"

echo " + convert to srcgraph..." >&2

run $bin_buildgraph_to_srcgraph $VERBOSE $DROPBDINDEP "${OUTDIR}/buildgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/srcgraph_acyclic.xml" || die "buildgraph2srcgraph failed"

echo " + calculate partial order..." >&2

run $bin_partial_order $VERBOSE $DROPBDINDEP "${OUTDIR}/srcgraph_acyclic.xml" $UNIVERSE \
	> "${OUTDIR}/order2.lst" || die "partial order failed"

echo " + build order is saved in ${OUTDIR}/order1.lst and ${OUTDIR}/order2.lst" >&2
echo " + an overview of the dependency situation is now available in ${OUTDIR}/stats.html" >&2
echo " + the data from which stats.html was generated is saved in ${OUTDIR}/stats.json" >&2
echo " + the build order was generated using the feedback arc set in ${OUTDIR}/remove.list" >&2
echo " + without the content of remove.list, the remaining strongly connected components are:" >&2

echo -e "graph.xml\ttype\tvertices\tedges\tscc"
while read g; do
	run $bin_graph_info $VERBOSE "${OUTDIR}/$g" || die "graph-info failed"
done < "${OUTDIR}/fas"

if [ "$KEEP" = "yes" ]; then
	echo " + temporary files stored in $tmp" >&2
else
	rm -f "$tmp"/*
	rm -fd $tmp
fi