#!/bin/bash -e

# See ``./devel --help``.

set -o pipefail

# Library implementation detection
detect:0:debian() { [ -f "/etc/debian_version" ]; }

_codename()
{
	local release
	release=$(lsb_release --release --short)
	if [ -z "${release}" ] || [ "${release}" = "n/a" ]; then
		printf "sid"
	else
		lsb_release --codename --short
	fi
}

# Profiles
declare -A MBD_PROFILES=(
	# dist selections
	[_debian]="export MBD_SETUP+=' --from-origins Debian --archives-from-origins --sources-from-origins';"                                             # Supported (distro-info) Debian codenames
	[_debian_all]="export MBD_SETUP+=' --from-origins Debian:all --archives-from-origins --sources-from-origins';"                                     # Known-working Debian codenames
	[_ubuntu]="export MBD_SETUP+=' --from-origins Ubuntu --archives-from-origins --sources-from-origins';"                                             # Supported (distro-info) Ubuntu codenames
	[_pureos]="export MBD_SETUP+=' --from-origins PureOS:all --archives-from-origins --sources-from-origins --distributions amber:arm64';"             # PureOS
	[_all]="export MBD_SETUP+=' --from-origins Debian:all Ubuntu:all --archives-from-origins --sources-from-origins';"                                 # All known-working (Debian and Ubuntu) codenames
	[_urold]="export MBD_SETUP+=' --sources squeeze lenny trusty';"                                                                                    # Urold, may still work with some special tweaks
	[_arm64]="export MBD_SETUP+=' --distributions $(_codename):arm64 --chroots $(_codename):arm64';"                                                   # Add arm64 arch (to test non-native arch under amd64)
	[_repos]="export MBD_SETUP+=' --repositories debdev::sid repo::sid';"                                                                              # Add special "debdev" and 'normal' "repo" repository

	# network options
	[_ssl]="export MBD_HTTPD_SSL=true;"                                                                                                                # Encrypted
	[_remote_localhost]="export MBD_SETUP+=' --remotes tcp:host=localhost:port=8068';"                                                                 # With remote 'localhost'
	[_remote_hostname]="export MBD_SETUP+=' --remotes tcp:host=$(hostname -f):port=8066'; export MBD_HOSTNAME=localhost; export MBD_HTTPD_PORT=8068;"  # Run as 'localhost:8068' with remote 'hostname'

	# installation OS workarounds
	[_buster]="export MBD_SKIP+=' codespell pylint pyflakes sphinx-build tidy build-ourselves';"                                                       # Hacks to run under buster
	[_bullseye]="export MBD_SKIP+=' pylint lintian'"                                                                                                   # Hacks to run under bullseye

	# Misc
	[_debug]="export MBD_COMMON_OPTIONS+=' --log-level=WARNING --log-level=mini_buildd.DEBUG';"                                                        # Normal debug mode (mini-buildd logs only)
	[_debug_all]="export MBD_COMMON_OPTIONS+=' --log-level=DEBUG'; export PYTHONWARNINGS='all'"                                                        # Full debug mode (all logs && python warnings)
	[_debug_django]="export MBD_EXTRA_OPTIONS+=' --debug=webapp'"                                                                                      # Enable django debug mode (templates)
)

_mbd_profile()  # <prof>
{
	local prof=${1}

	local code=${MBD_PROFILES[${prof}]}
	if [ -n "${code}" ]; then
		eval "${MBD_PROFILES[${prof}]}"
		logI "Using profile: ${prof}"
	else
		logE "No such profile: ${prof}"
		return 1
	fi
}

mbd_profile()  # [<prof0> <prof1> ...] <target_arg0> <target_arg1> ...
{
	# Load _profiles
	local prof
	local -i spos=0
	for prof in "${@}"; do
		[ "${prof:0:1}" == "_" ] || break  # profile args need to start with _
		_mbd_profile "${prof}"
		spos+=1
	done

	# Run w/ remaining (target) args
	./devel "${@:spos+1}"
}

mbd_supertestall()
{
	./devel profile _all      updatetestall | tee ../supertestall.log 2>&1
	./devel profile _all _ssl updatetestall | tee ../supertestall_ssl.log 2>&1
}

MBD_BROWSERS="firefox-esr firefox chromium google-chrome-stable google-chrome-beta google-chrome-unstable"
mbd_browser()
{
	local -a urls=("http://${MBD_HOSTNAME}:8066" "http://localhost:8068" "https://${MBD_HOSTNAME}:8066" "https://localhost:8068" "http://localhost:8000")
	local -A args=(
		[firefox]="--no-remote"
		[firefox-esr]="--no-remote"
	)
	${1} ${args[${1}]} "${urls[@]}" >devel.browser.log 2>&1 &
}

# Convenience: Run ipython so we can import directly from working dir (PYTHONPATH already set here)
mbd_ipython3() { ipython3; }

_bf_=$(tput bold) || true
_it_=$(tput sitm) || true
_red_=$(tput setaf 1) || true
_yellow_=$(tput setaf 3) || true
_blue_=$(tput setaf 4) || true
_reset_=$(tput sgr0) || true

log()
{
	local running=""
	[ -z "${MBD_RUNNING}" ] || running="${_bf_}${MBD_RUNNING}${_reset_}: "
	local -A color=(["I"]="${_blue_}" ["W"]="${_yellow_}" ["E"]="${_red_}")
	printf "$(date --rfc-3339=ns) ${color[${1}]}${1}${_reset_}: ${running}${2}\n" "${@:3}" >&2;
}
logI() { log "I" "${@}"; }
logW() { log "W" "${@}"; }
logE() { log "E" "${@}"; }

# don't accidentally run this on your host system.
check_devsys()
{
	if ! (ischroot || systemd-detect-virt --quiet); then
		printf "W: $0\n" >&2
		printf "W: Never use this script in production systems\n" >&2
		printf "W: This may be not a developer system (not chroot, not systemd container)\n" >&2
		read -p"Sure to continue (Ctrl-c to abort)?" DUMMY
	fi
}

# python local install helpers
mbd_pip_uninstall()
{
	local packages=${1:-$(pip3 freeze --user)}
	[ -z "${packages}" ] || pip3 uninstall --yes ${packages}
}

mbd_pip_install()
{
	local package
	for package in "${@}"; do
		pip3 install --upgrade --ignore-installed "${package}"
	done
}

mbd_installdeps()
{
	sudo apt-get update
	sudo apt-get --yes upgrade

	# Extras
	sudo apt-get install \
			 --no-install-recommends \
			 --yes \
			 \
			 lsb-release \
			 aptitude \
			 git \
			 git-buildpackage \
			 devscripts \
			 disorderfs \
			 diffoscope \
			 equivs \
			 pydocstyle \
			 pycodestyle \
			 isort \
			 pylint \
			 pyflakes3 \
			 python3-pip \
			 python3-wheel \
			 python3-keyrings.alt \
			 devscripts \
			 faketime \
			 tidy \
			 codespell \
			 sqlite3 \
			 moreutils \
			 curl \
			 siege \
			 debmirror \
			 apache2-utils \
			 ssl-cert \
			 libnss3-tools \
			 libdistro-info-perl \
			 arch-test \
			 firefox-esr

	# Extras: ftp client, preferring ftp-ssl.
	sudo apt-get install ftp-ssl || sudo apt-get install ftp

	# Build-Deps
	mk-build-deps --install --remove --root-cmd=sudo --tool="apt-get --yes --no-install-recommends"
}

# Force plain text backend for development
mbd_pythonkeyringtestconfig()
{
	local keyringrc="${HOME}/.config/python_keyring/keyringrc.cfg"
	[ -L "${keyringrc}" ] || { mkdir -p "$(dirname "${keyringrc}")" && ln -s -v -f "${MBD_PJPATH}/devel.keyringrc.cfg" "${keyringrc}"; }
}

# Add custom self-signed cert
mbd_setupcert()
{
	sudo ${MBD_PJPATH}/src/mini-buildd-self-signed-certificate create ${MBD_HOSTNAME}

	# Make trusted for system (ca-certificates)
	local cert="/etc/ssl/mini-buildd/certs/mini-buildd.crt"
	( cd "/usr/local/share/ca-certificates/" && sudo ln -s -f "${cert}" .)
	sudo update-ca-certificates --fresh

	# Helper to update certs to some known browsers
	./examples/mini-buildd-utils/ca-certificates2browser
}

declare -g -a _ON_EXIT=()
on_exit_run()
{
	local line
	for line in "${_ON_EXIT[@]}"; do
		if ${MBD_KEEP}; then
			logW "MBD_KEEP: Ignoring On Exit: %s" "${line}"
		else
			logI "On Exit: %s" "${line}"
			${line}
		fi
	done
	[ -z "${MBD_RUNNING}" ] || logE "Failed: $?"
}
on_exit()
{
	_ON_EXIT+=("${*}")
}
trap "on_exit_run" EXIT
# Avoid some esoteric cases where on_exit is not run (certain emacs compilation buffer kills)
trap "exit 2" INT
trap "exit 3" TERM

_check_prog()
{
	local path
	for path in $(printf "${PATH}" | tr ":" " "); do
		local prog="${path}/${1}"
		if [ -x "${prog}" ]; then
			logI "Found: ${prog}."
			return 0
		fi
	done
	logE "'${1}' not found in path; please install."
	logI "'./devel prepare-system' should install all deps needed."
	exit 1
}

# Extra python as pip user install (f.e., use 'pylint' here to use it instead of the (usually older) Debian version).
: ${MBD_PIPINSTALL:=""}

MBD_PJPATH="$(readlink -f $(dirname $0))"
MBD_PYPATH="${MBD_PJPATH}/src"
export PYTHONPATH="${MBD_PYPATH}"
MBD_SETUP_CFG="${MBD_PJPATH}/setup.cfg"
MBD_LINTIANRC="${MBD_PJPATH}/devel.lintianrc"
MBD_CODENAME=$(_codename)
declare -A _MBD_CODEVERSIONS=([buster]=10 [bullseye]=11 [bookworm]=12 [trixie]=13 [forky]=~FORKY [sid]=~SID)
[ -z "${MBD_PROFILES[_${MBD_CODENAME}]}" ] || _mbd_profile "_${MBD_CODENAME}"
MBD_CODEVERSION="${_MBD_CODEVERSIONS[${MBD_CODENAME}]}"

# Autogenerated by us
MBD_DPUT_CF="${MBD_PJPATH}/devel.dput.cf"
MBD_LAST_CHANGES="${MBD_PJPATH}/devel.last_changes"

#
# Environment (others)
#
# Use noninteractive by default -- install, configure && purge w/o confirmation (see --preserve-env=DEBIAN_FRONTEND in code below)
export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive}
export PYTHONWARNINGS=${PYTHONWARNINGS:-ignore}

#
# Configurable variables
#
MBD_HTTPD_SSL_DESC="Toggle 'https mode' (ssl key is managed via 'mini-buildd-self-signed-certificate'. Should just work after './devel prepare-system')."
: ${MBD_HTTPD_SSL:=false}
MBD_CONFIG+="MBD_HTTPD_SSL "

MBD_HTTPD_PORT_DESC="Custom port -- run a second instance on same network stack (i.e, use '8068' and configure ftp to be at '8069')."
: ${MBD_HTTPD_PORT:=8066}
MBD_CONFIG+="MBD_HTTPD_PORT "

MBD_COMMON_OPTIONS_DESC="Common options (used by all python-based tools)."
: ${MBD_COMMON_OPTIONS:=}
MBD_CONFIG+="MBD_COMMON_OPTIONS "

MBD_EXTRA_OPTIONS_DESC="Extra options for the mini-buildd call only."
: ${MBD_EXTRA_OPTIONS:=}
MBD_CONFIG+="MBD_EXTRA_OPTIONS "

MBD_HOSTNAME_DESC="Custom hostname -- for a second instance in ssl mode, you must have a different hostname for the certificate."
: ${MBD_HOSTNAME:=$(hostname -f || printf "localhost")}
MBD_CONFIG+="MBD_HOSTNAME "

MBD_SETUP_COMMON_DESC="Common args for API call 'setup', needed to complete testsuite. Rather don't touch."
: ${MBD_SETUP_COMMON:=--update all --pca all --archives-from-proxy --sources ${MBD_CODENAME} --distributions-from-sources --repositories-from-distributions test --chroots-from-sources }
MBD_CONFIG+="MBD_SETUP_COMMON "

MBD_SETUP_DESC="Extra args for API call 'setup'. Use '--archives-from-origins' if you have no local 'apt-cacher-ng' running."
: ${MBD_SETUP:=}
MBD_CONFIG+="MBD_SETUP "

MBD_KEEP_DESC="Don't run 'on exit' cleanups."
: ${MBD_KEEP:=false}
MBD_CONFIG+="MBD_KEEP "

MBD_SKIP_DESC="Skip these test cases (space separated list)."
: ${MBD_SKIP:=}
MBD_CONFIG+="MBD_SKIP "

if ${MBD_HTTPD_SSL}; then
	MBD_HTTPD_ENDPOINT="ssl:port=${MBD_HTTPD_PORT}:privateKey=/etc/ssl/mini-buildd/private/mini-buildd.key:certKey=/etc/ssl/mini-buildd/certs/mini-buildd.crt"
	MBD_HTTPD_PROTO="https"
else
	MBD_HTTPD_ENDPOINT="tcp6:port=${MBD_HTTPD_PORT}"
	MBD_HTTPD_PROTO="http"
fi
MBD_HTTPD_URL="${MBD_HTTPD_PROTO}://${MBD_HOSTNAME}:${MBD_HTTPD_PORT}"
MBD_HTTPD_ADMIN_URL="${MBD_HTTPD_PROTO}://admin@${MBD_HOSTNAME}:${MBD_HTTPD_PORT}"
MBD_DEBCONF="\
set mini-buildd/admin_password admin
set mini-buildd/options ${MBD_COMMON_OPTIONS} --hostname=${MBD_HOSTNAME} --http-endpoint=${MBD_HTTPD_ENDPOINT} ${MBD_EXTRA_OPTIONS}
set mini-buildd/pythonwarnings ${PYTHONWARNINGS}"

# Needed by ``gbp dch`` -- be sure it's not empty
: ${DEBEMAIL:="$(id --user --name)@${MBD_HOSTNAME}"}
export DEBEMAIL

mbd_api()       { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --auto-save-passwords --auto-confirm "${1}" "${MBD_HTTPD_ADMIN_URL}" "${@:2}"; }
mbd_codenames() { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --json status "${MBD_HTTPD_URL}" --with-repositories | jq --raw-output ".repositories.test.codenames[]"; }
mbd_events()    { ${MBD_PYPATH}/mini-buildd-events ${MBD_COMMON_OPTIONS} "${MBD_HTTPD_URL}" "${@:1}"; }
mbd_dput()
{
	mbd_api dput_conf >${MBD_DPUT_CF}
	${MBD_PYPATH}/mini-buildd-dput ${MBD_COMMON_OPTIONS} --config=${MBD_DPUT_CF} --force mini-buildd-${MBD_HOSTNAME%%.*} "${@:1}"
}

# Shortcuts to build one test/keyring package
mbd_testpackage() { mbd_api test_packages --sources="${1}" --distributions="${MBD_CODENAME}-test-unstable" "${@:2}"; }
mbd_mbd-test-cpp()     { mbd_testpackage "mbd-test-cpp" "${@}"; }
mbd_mbd-test-archall() { mbd_testpackage "mbd-test-archall" "${@}"; }
mbd_mbd-test-ftbfs()   { mbd_testpackage "mbd-test-ftbfs" "${@}"; }

mbd_archive-keyring()  { mbd_api keyring_packages --distributions="${MBD_CODENAME}-test-unstable" "${@}"; }

# pgrep pid && kill support
mbd_pid()
{
	local p pids=$(pgrep --uid=mini-buildd --full /usr/bin/python3.*/usr/sbin/mini-buildd)
	for p in ${pids}; do
		local root=$(sudo realpath -m "/proc/${p}/root")
		if [ "${root}" == "/" ]; then
			printf "%s\n" "${p}"
		else
			logW "Ignoring process %s from chroot %s" "${p}" "${root}"
		fi
	done
}

mbd_kill()
{
	local pid=$(mbd_pid) signal=${1:-SIGTERM}
	logI "Killing process ${pid:-*no mini-buildd process found*} (${signal})..."
	[ -z "${pid}" ] || sudo kill --signal=${signal} ${pid}

	# sysv (chroot) compat (else "service start" will not work)
	sudo rm -v -f /run/mini-buildd.pid
}

MBD_PYSOURCES=$(./pysources)
MBD_PYMODULES=$(./pysources modules)
MBD_PYSCRIPTS=$(./pysources scripts)

mbd_pyrpl()
{
	for m in ${MBD_PYSOURCES}; do
		rpl -F "${1}" "${2}" "${m}"
	done
}

mbd_service()
{
	sudo service mini-buildd "${1}"
}

mbd_changes() # [<arch>]
{
	local arch=${1-$(dpkg-architecture --query=DEB_BUILD_ARCH)}
	printf "../mini-buildd_$(dpkg-parsechangelog --show-field=version)_${arch}.changes"
}

mbd_curl-admin()
{
	local url="${MBD_HTTPD_URL}/accounts/login/"
	local cookies=./devel.curl_cookies
	local curl="curl --silent --cookie-jar ${cookies} --cookie ${cookies} --referer ${url}"
	rm -v -f "${cookies}"

	# Generate initial cookies file
	${curl} "${url}" >/dev/null
	local csrf="csrfmiddlewaretoken=$(grep csrftoken "${cookies}" | cut -f7)"

	# Login. Returns 302 w/ empty body on successful login. We test for the empty body.
	local login_body="./devel.curl_login"
	${curl} --data "${csrf}&username=admin&password=admin" --output "${login_body}" "${url}"
	if [ -s "${login_body}" ]; then
		logE "Curl login failed."
		return 1
	fi
	time ${curl} "${@}"
}

mbd_icons()
{
	logI "Icon names that need to be supported in used icon theme."
	logI "Naming oriented on: https://specifications.freedesktop.org/icon-naming-spec/icon-naming-spec-latest.html"
	grep --recursive --no-filename --only-matching 'mbd_icon[[:space:]]*[^[:space:]]*' src/mini_buildd/templates/ | sort --unique
}

# A very bad hack to hack HTML without having to install package
mbd_html-hack()
{
	local static_path="/usr/lib/python3/dist-packages/mini_buildd/static"
	local templates_path="/usr/lib/python3/dist-packages/mini_buildd/templates"
	case $1 in
		start)
			sudo adduser mini-buildd "$(id --group --name)"
			local path
			for path in ${static_path} ${templates_path}; do
				sudo rm -rfv "${path}"
				(cd $(dirname "${path}") && sudo ln -s -v "${MBD_PYPATH}/mini_buildd/$(basename "${path}")" .)
			done
			(
				cd "${MBD_PYPATH}/mini_buildd/static/"
				ln -s -v "/usr/share/javascript" . || true
				ln -s -v /usr/lib/python3/dist-packages/django/contrib/admin/static/admin . || true
			)
			mbd_service restart
			;;
		stop)
			for path in ${static_path} ${templates_path}; do
				sudo rm -v -f "${path}"
			done
			rm -v "${MBD_PYPATH}/mini_buildd/static/javascript" || true
			rm -v "${MBD_PYPATH}/mini_buildd/static/admin" || true
			;;
		check)
			if [ -L "${static_path}" -o -L "${templates_path}" ]; then
				logE "HTML hack active!"
				return 1
			fi
			;;
	esac
}

if ! mbd_html-hack check; then
	 read -p"Remove html hack (Ctrl-c to abort)?" DUMMY
	 mbd_html-hack stop
fi

mbd_makemigrations()
{
	python3 ./setup.py version_py
	./src/django-admin makemigrations mini_buildd
}

mbd_makedatamigrations()
{
	python3 ./setup.py version_py
	./src/django-admin makemigrations --empty mini_buildd
}

#
# Runner functions: mbd_run:<sequence>:<name>
#
# <sequence>: Three-digit sequence number; this number also determines in which order to run.
# All run in sequence order is the whole testsuite (which, in turn, may be run with different profiles, see supertestall as example).
#  0..: system prepare
#  1..: static code tests
#  2..: dynamic code tests
#  3..: build
#  4..: static build artefact tests
#  5..: deploy
#  6..: live tests
#  7..: web live tests

mbd_run:000:prepare-system()
{
	mbd_installdeps
	mbd_pythonkeyringtestconfig
	mbd_pip_uninstall
	mbd_pip_install ${MBD_PIPINSTALL}
}

mbd_run:100:version_py()  # Needed once to get (at least) version.py setup
{
	python3 ./setup.py version_py
}

mbd_run:105:codespell()
{
	local ups=$(codespell --ignore-words="./devel.codespell.ignore-words" --exclude-file="./devel.codespell.ignore-lines" --quiet-level=2 ./devel ${MBD_PYSOURCES} manual/*.rst)
	if [ -n "${ups}" ]; then
		logE "codespell failed:\n${ups}"
		return 1
	fi
}

mbd_run:110:pycodestyle()  # See setup.cfg
{
	_check_prog pycodestyle
	pycodestyle ${MBD_PYSOURCES}
}

# Note: Output not parsable; see https://github.com/PyCQA/pydocstyle/issues/460
mbd_run:115:pydocstyle()  # See setup.cfg
{
	_check_prog pydocstyle
	pydocstyle ${MBD_PYSOURCES}
}

mbd_run:117:isort()  # See setup.cfg
{
	_check_prog isort
	if [ "${1}" == "fix" ]; then
		isort ${MBD_PYSOURCES}
	else
		isort --check ${MBD_PYSOURCES}
	fi
}

MBD_PYLINT_MSG_TEMPLATE="{path}:{line}: {C}: [{msg_id}({symbol}), {obj}] {msg}"
mbd_run:120:pylint()  # See setup.cfg  # See https://github.com/PyCQA/pylint/issues/6943
{
	_check_prog pylint
	pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" ${MBD_PYSOURCES}
}

mbd_run:121:pylint-migrations()
{
	_check_prog pylint
	pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" \
				 --disable="invalid-name,duplicate-code,line-too-long" \
				 src/mini_buildd/migrations/
}

mbd_run:135:pyflakes()  # See setup.cfg
{
	_check_prog pyflakes3
	pyflakes3 ${MBD_PYSOURCES}
}

mbd_run:200:pydoctests()
{
	./pydoctests
}

mbd_run:201:migrations-from-20x()
{
	local sqlite_path="src/test-data/config.sqlite"
	sqlite3 "${sqlite_path}" ".read ${sqlite_path}.20x.sql"
	./src/run-migrations --log-level=DEBUG ${sqlite_path}
	rm -v -f "${sqlite_path}"
}

# Same call as in debian/rules, but warnings are errors
mbd_run:205:sphinx-build()
{
	python3 ./setup.py version_py
	sphinx-build -N -bhtml -W ./manual/ ./build/sphinx/html/
}

mbd_run:300:changelog()
{
	# Checking changelog (must be unchanged)...
	git diff-index --exit-code HEAD debian/changelog
	on_exit git checkout debian/changelog

	## Variant w/: --snapshot --distribution=xxx working:
	# gbp dch --snapshot --auto \
	#		--new-version="$(dpkg-parsechangelog --show-field=Version)~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \
	#		--distribution="${MBD_CODENAME}-test-experimental" --force-distribution

	# Try to get snapshot changelog so package might be uploaded to testsuite mini-buildd later for testing
	# gbp dch --auto   # This gets pretty slow if your are in a longer dev battle stint with many commits
	gbp dch --since HEAD~~~~ \
			--new-version="$(dpkg-parsechangelog --show-field=Version).dev$(date --utc +'%Y%m%d%H%M%S')~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \
			--distribution=${MBD_CODENAME}-test-experimental --force-distribution
	mbd_changes >"${MBD_LAST_CHANGES}"
}

mbd_run:305:build()
{
	DEB_BUILD_OPTIONS+="nocheck" debuild --no-lintian -us -uc
}

mbd_run:400:lintian()
{
	local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes)
	logI "Lintian-checking: %s" "${changes}"
	local result=$(lintian --cfg="${MBD_LINTIANRC}" "${@}" "${changes}")
	printf "%s\n---\n" "${result}"
	if grep "^[EW]:" <<< ${result}; then
		logE "Lintian FAILED"
		return 1
	fi
}

# This also checks "full" package building (with doc and check)
mbd_run:405:debrepro()
{
	debrepro
}

mbd_run:500:remove()
{
	mbd_service stop || true
	sudo --preserve-env=DEBIAN_FRONTEND dpkg --${1:-remove} mini-buildd mini-buildd-utils mini-buildd-doc python3-mini-buildd python-mini-buildd mini-buildd-common
}

mbd_run:505:purge()
{
	mbd_run:500:remove purge
}

mbd_run:510:install()
{
	! ischroot || mbd_kill  # In chroot environment, apt will ignore stopping the (old) service, which may lead to errors upgrading

	sudo --preserve-env=DEBIAN_FRONTEND apt-get --yes --allow-downgrades install $(mbd_changes)
	! ${MBD_HTTPD_SSL} || mbd_setupcert  # setup certificates late to guarantee user 'mini-buildd' is available and ownership of cert files can be properly set
	sudo sed -i "s|^\([A-Z_]\+\)=.*|\1=\"\"|g" /etc/default/mini-buildd   # Deprive any FOO_BAR=value lines of their value (else ``debian/mini-buildd.config`` honors (old) values from file as user customizations)
	printf "%s" "${MBD_DEBCONF}" | sudo debconf-communicate
	sudo --preserve-env=DEBIAN_FRONTEND dpkg-reconfigure mini-buildd
}

# Workaround: ``apt-get --install-recommends --install-suggests <FILE_ON_DISK>`` does not seem to work (see install() above)
mbd_run:511:install-recommends-suggests()
{
	sudo aptitude --assume-yes install '~Rrecommends:^mini-buildd' '~Rsuggests:^mini-buildd'
}

mbd_run:515:restart()
{
	mbd_service restart
	sleep 0.8  # Try to avoid "connection refused" warnings in most cases

	local -i try=1
	while ! mbd_api status; do
		if [ ${try} -gt 5 ]; then
			logE "Can't get status even after ${try} tries"
			return 1
		fi
		try+=1
		sleep 1
	done
}

mbd_run:600:setup()
{
	mbd_api setup ${MBD_SETUP_COMMON} ${MBD_SETUP}
}

mbd_run:601:spameventsqueue()
{
	local i
	for i in {1..50}; do
		mbd_events >>/dev/null 2>&1 &
	done
	sleep 10                                                                    # Wait some time to let all clients build up network connection
	curl --max-time 5 --retry 1 --silent --output /dev/null "${MBD_HTTPD_URL}"  # If it does not reply in time, assume error: twisted would stall (TCP connect yes, but no handling)
	mbd_service restart                                                         # This should terminate all clients
	sleep 10                                                                    # Be sure mini-buildd is up again for following tests
}

mbd_run:605:keyring-packages()
{
	mbd_api keyring_packages
}

mbd_run:610:test-packages()
{
	mbd_api test_packages --with-check
}

mbd_run:615:testsuite-packages()
{
	local location="./share/testsuite-packages/" dst changes

	# Clean out possible debris
	(cd "${location}" && git clean -x -d -f)

	# Rebuild DSCs when needed
	for dst in $(find ${location} -maxdepth 1 -mindepth 1 -type d); do
		local source=$(basename "${dst}")
		[ -z "${1}" ] || [ "${source}" == "${1}" ] || continue
		sed --in-place --expression="1 s/sid/${MBD_CODENAME}/" --expression="1 s/~SID/${MBD_CODEVERSION}/" "${dst}/debian/changelog"
		(
			cd ${dst}
			dpkg-buildpackage -us -uc
		)
		sed --in-place --expression="1 s/${MBD_CODENAME}/sid/" --expression="1 s/${MBD_CODEVERSION}/~SID/" "${dst}/debian/changelog"
	done

	# Tests: Upload and expect event
	for changes in ${location}/*${1}*.changes; do
		local after=$(date --rfc-3339=ns)
		local source=$(basename ${changes} | cut -d'_' -f1)
		local expect=$(cut -d'-' -f1 <<<${source} | tr '[:lower:]' '[:upper:]')
		local failOn="FAILED REJECTED"
		case ${expect} in
			REJECTED)
				failOn="INSTALLED BUILT"
				;;
		esac
		[ -z "${1}" ] || [ "${source}" == "${1}" ] || continue
		logI "Testing ${source} (expect=${expect})"

		# Try package removal (make test not fail if it's already installed from a previous run)
		if [ "${expect}" == "INSTALLED" ]; then
			mbd_api remove "${source}" "${MBD_CODENAME}-test-unstable" --without-rollback >/dev/null || true
		fi

		case ${source} in
			installed-portext)
				cp "${location}/installed-portext_1.0.0.dsc" "${location}/installed-portext_1.0.0.tar.gz" "/tmp/"
				mbd_api port_ext --allow-unauthenticated "file:///tmp/installed-portext_1.0.0.dsc" "${MBD_CODENAME}-test-unstable"
				;;
			*)
				# Standard dput, dput-ng can't do ftps.
				mbd_dput "${changes}"
				;;
		esac
		mbd_events --source="${source}" --after="${after}" --exit-on ${expect} --fail-on ${failOn}
		if [ "${expect}" == "INSTALLED" ]; then
			case ${source} in
				installed-port)
					# Try to check for success on internal ports where possible
					local c codenames=$(mbd_codenames)  # Available codenames on running mbd
					for c in bookworm bullseye; do  # Needs to be same as in installed-port's ``changelog``
						# Note: On rebuilds (c == MBD_CODENAME), the events check won't work (i.e., would always immediately succeed on the original package)
						if [ "${c}" != "${MBD_CODENAME}" ] && grep "^${c}\$" <<<${codenames}; then
							logI "Waiting for internal ${c} port..."
							mbd_events --source "${source}" --distribution "${c}-test-unstable" --after "${after}" --exit-on ${expect} --fail-on ${failOn}
							mbd_api remove "${source}" "${c}-test-unstable" --without-rollback
						else
							logW "Skipping check for internal ${c} port (rebuilt or codename not available)"
						fi
					done
					;;
			esac
		fi
	done
}

mbd_run:620:build-ourselves()
{
	local after=$(date --rfc-3339=ns)
	local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes)
	mbd_dput ${changes}
	mbd_events --source="mini-buildd" --distribution="${MBD_CODENAME}-test-experimental" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED
}

mbd_run:625:build-migrate-remove()
{
	__build_migrate()
	{
		local after=$(date --rfc-3339=ns)
		mbd_mbd-test-cpp
		mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-unstable" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED

		mbd_api migrate --full "mbd-test-cpp" "${MBD_CODENAME}-test-unstable"
		mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-testing" --after="${after}" --exit-on MIGRATED
		mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-stable" --after="${after}" --exit-on MIGRATED
	}
	__build_migrate
	__build_migrate

	# Build && migrated twice: There should be rollback0 for all three suites
	local ls=$(mbd_api ls "mbd-test-cpp")
	grep "${MBD_CODENAME}-test-unstable-rollback0" <<<${ls}
	grep "${MBD_CODENAME}-test-testing-rollback0" <<<${ls}
	grep "${MBD_CODENAME}-test-stable-rollback0" <<<${ls}

	# Test removals (don't remove stable as we do test installs later)
	mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable-rollback0"
	mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable"
}

mbd_run:630:bogus-ftp-uploads()
{
	local -i ftp_port=$((MBD_HTTPD_PORT+1))
	# test packager.py bogus upload handling
	_upload()
	{
		timeout 5 ftp -inv ${MBD_HOSTNAME} ${ftp_port} <<EOF || true  # timeout, true: openssl 3 'unexpected eof while reading' from pyftpd -- still receives the file, but won't close connection, making this halt
user "anonymous" "dummypass"
cd incoming/
binary
put "${1}" "$(basename "${1}")"
bye
EOF
	}

	# Broken but basically acceptable changes. This should log event REJECTED.
	local reject="/tmp/reject.changes"
	cat <<EOF >"${reject}"
Distribution: kaputt
Version: 1.2.3
Source: kaputt
Architecture: kaputt
EOF
	local after=$(date --rfc-3339=ns)
	_upload "${reject}"
	mbd_events --source="kaputt" --after="${after}" --exit-on REJECTED

	# Completely unparsable changes. This file should just be removed by mini-buildd (no feasible way to test that automatically, though).
	local invalid="/tmp/invalid.changes"
	cat <<EOF >"${invalid}"
kaputt
EOF
	_upload "${invalid}"
}

mbd_run:635:api-getters()
{
	# Consume
	mbd_api status
	mbd_api pub_key
	mbd_api dput_conf
	mbd_api sources_list
	mbd_api ls "mbd-test-cpp" --repositories "test"
	mbd_api show "mbd-test-cpp" --repositories "test" --codenames "${MBD_CODENAME}"

	# Maintain
	mbd_api start
	mbd_api uploaders
	mbd_api snapshot_ls "${MBD_CODENAME}-test-stable"
}

_test_install()
{
	on_exit sudo dpkg --purge "${1}"
	sudo apt-get "${@:2}" install "${1}"
}

mbd_run:640:apt-tofu-bootstrap()
{
	sudo MBD_CODENAME=${MBD_CODENAME} ./src/mini-buildd-bootstrap-apt "${MBD_HTTPD_URL}" auto

	local sources_list="/etc/apt/sources.list.d/mbd-testsuite.list"
	on_exit sudo rm -v -f "${sources_list}"
	on_exit sudo apt-get update

	# Add sources.list && test apt update
	mbd_api sources_list --codenames "${MBD_CODENAME}" | sudo tee "${sources_list}"
	sudo apt-get update

	# install previously built test package
	_test_install mbd-test-cpp
}

mbd_run:645:apt-snapshot()
{
	# Test snapshot gen, del and sources.list
	local snap="TESTSUITE_$(date --iso-8601=seconds)"
	mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}"
	mbd_api snapshot_delete "${MBD_CODENAME}-test-stable" "${snap}"
	mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}"
	mbd_api snapshot_ls "${MBD_CODENAME}-test-stable"
	mbd_api sources_list --repositories "test" --codenames "${MBD_CODENAME}" --suites "stable" --snapshot "${snap}"
}

mbd_test_urls()
{
	local uri
	for uri in \
		/mini_buildd/ \
		/mini_buildd/api/ \
		/mini_buildd/api/status/?output=html \
		/mini_buildd/api/search/?pattern=mbd\&output=html \
		/mini_buildd/api/ls/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \
		/mini_buildd/api/show/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \
		$(for route in events repositories builders crontab manual api setup sitemap; do printf "/mini_buildd/%s/ " "${route}"; done) \
		/mini_buildd/repositories/test/dists/${MBD_CODENAME}-test-stable/ \
		$(for route in events repositories; do printf "/mini_buildd/%s-dir/ " "${route}"; done) \
		/accounts/login/ \
		/admin/ \
		/admin/mini_buildd/source/ \
		/static/manual/index.html \
		/static/manual/abstract.html \
		/static/manual/administrator.html \
		/static/manual/consumer.html \
		/static/manual/developer.html \
		/static/manual/roadmap.html \
		/static/manual/auto_admonitions.html \
		; do
		printf "%s\n" "${MBD_HTTPD_URL}${uri}"
	done
}

mbd_run:700:tidy()
{
	local url html="./devel.tidy.html"
	while IFS="" read -r url; do
		logI "Testing HTML: ${html} from '${url}'"
		mbd_curl-admin --fail --silent --location --output "${html}" "${url}"

		local mute=""
		mute+="PROPRIETARY_ATTRIBUTE,"                                       # With django 3, tidy complains about 'proprietary attribute "autocapitalize"', but this seems to be valid html5.
		mute+="TRIM_EMPTY_ELEMENT,"                                          # Just too picky.
		mute+="TAG_NOT_ALLOWED_IN,"                                          # Cries about "<script> isn't allowed in <table>", but this isn't true for HTML5 afaik. And me wants!
		mute+="MISSING_ENDTAG_BEFORE,INSERTING_TAG,"                         # django 4.2(admin/color_theme_toggle.html) uses <button><div/></button> (not sure if valid, but we should not fail on django upstream)
		mute+="BLANK_TITLE_ELEMENT,"                                         # New w/ tidy 5.8.0, no fix known
		if grep --quiet "/manual/" <<<${url}; then
			mute+="MISSING_ATTR_VALUE,"                                        # HTML created by sphinx (manual) has this a lot, and seems nothing we can do about it.
			mute+="UNKNOWN_ELEMENT,DISCARDING_UNEXPECTED,"                     # sphinx 7.3.7 seems to use new "search" element
		fi

		local result=$(tidy --quiet yes --force-output yes --gnu-emacs yes --mute-id yes -output /dev/null --mute ${mute} "${html}" 2>&1)

		if [ -n "${result}" ]; then
			printf "Tidy says: \"%s\"\n" "${result}"
			return 1
		fi
	done <<<$(mbd_test_urls)
}

mbd_run:710:httpd-benchmark()
{
	siege --benchmark --time=10s --file=<(mbd_test_urls) --json-output
}

mbd_run:720:debmirror()
{
	mbd_api debmirror --codenames "${MBD_CODENAME}" --suites stable
}

mbd_sequence()  # [<levelRegex>=00] [<name>] <hr>
{
	local levelRegex="${1:-00}" name="${2}" hr="${3}"
	for func in $(declare -F | cut -d" " -f3- | grep "^mbd_run:${levelRegex}:${name}" | sort || true); do
		if [ -n "${hr}" ]; then
			printf "%s " "$(cut -d: -f3 <<<${func})"
		else
			printf "%s " "${func}"
		fi
	done
}

mbd_skip()  # <func>
{
	local short=$(cut -d: -f3 <<<${1}) s
	for s in ${MBD_SKIP}; do
		[ "${short}" != "${s}" ] || return 0
	done
	return 1
}

mbd_run()  # [<levelRegex>=00] [<name>] [<customArgs>...]
{
	local levelRegex="${1}" name="${2}"
	local -a info=()
	local func totalStartStamp=$(date +%s)
	local -i count=0
	for func in $(mbd_sequence "${levelRegex}" "${name}"); do
		MBD_RUNNING="${func}"
		local startStamp=$(date +%s)
		if mbd_skip "${func}"; then
			logI "SKIPPED (in skip list: ${MBD_SKIP})"
		else
			logI "Starting..."
			${func} "${@:3}"
			logI "Completed."
		fi
		MBD_RUNNING=""

		count+=1
		info+=("$(printf "OK (%03d seconds): %s" "$(($(date +%s) - startStamp))" "${func}")")
	done
	if ((count <= 0)); then
		logE "No runs for this sequence filter: ${levelRegex} ${name}"
		return 1
	fi
	logI "Sequence results (%03d seconds):" "$(($(date +%s) - totalStartStamp))"
	logI "%s" "${info[@]}"
	logI "OK, %s runs succeeded ($(date))." "${count}"
}

# Shortcuts
declare -A MBD_RUN_SHORTCUTS=(
	["check"]='1..'
	["update"]='\(300\|305\|510\|515\)'
	["updatecheck"]='\(1..\|300\|305\|510\|515\)'
	["updatetest"]='\([12]..\|300\|305\|510\|515\|4..\)'
	["updatetestall"]='.*'
)
# We can't iterate through the associative array in the given order later, so we at least want a sorted key list as helper
MBD_RUN_SHORTCUTS_SORTED="$(printf '%s\n' "${!MBD_RUN_SHORTCUTS[@]}" | sort -n)"

# Some extra targets
mbd_service.log()
{
	if systemctl is-system-running; then
		journalctl --unit mini-buildd --follow
	else
		tail --follow=name --retry /var/log/mini-buildd.log | ts
	fi
}

mbd_access.log()
{
	tail --follow=name --retry /var/lib/mini-buildd/var/log/access.log
}

mbd_grep()
{
	grep --recursive \
			 --exclude-dir=".git" \
			 --exclude="changelog" \
			 --binary-files="without-match" "${@}"
}

mbd_pylintgeneratedmembers()
{
	# Hack to update identifiers with false-positive "has no xxx member" error (mostly django).
	# Implies you already have a this as _last_ option in ``setup.cfg``.
	{
		for o in $(pylint src/mini_buildd/ | grep "has no.*member" | cut -d"'" -f4 | sort | uniq); do
			printf "\t${o},\n"
		done
	} >>setup.cfg
}

# Try to reproduce deadlock
# This try w/ events does not yet reproduce the deadlock; saving for later.
mbd_deadlock()
{
	local -i count=0
	while mbd_events --source mbd-test-cpp --stop >/dev/null; do
		# echo ${count}; count+=1;
		sleep 0;
	done
}

mbd_bash-completion()
{
	local non_runners=$(declare -F | cut -d" " -f3 | grep "^mbd_" | grep --invert-match "^mbd_run" | cut -b5-)
	local profiles="${!MBD_PROFILES[*]}"
	printf "complete -W '%s' ./devel\n" "$(printf '%s ' "${!MBD_RUN_SHORTCUTS[@]}") $(mbd_sequence "..." "" "hr") ${non_runners} ${profiles} ${MBD_BROWSERS}"
}

mbd_run-lintian()
{
	mbd_run:300:changelog
	mbd_run:305:build
	mbd_run:400:lintian
}

# run-profile [callgrind]: Run with python profiling enabled && show stats after termination
# * needs ``python3-yappi`` (for multithreading)
# * optional: will use ``kcachegrind`` if called with ``callgrind`` argument
mbd_run-profile()
{
	local format=${1:-pstat} profile=~mini-buildd/mini-buildd.profile
	mbd_service stop
	sudo su - mini-buildd --command="yappi --output-format=${format} --output-file=${profile} /usr/sbin/mini-buildd" || true
	logI "Profiling data: ${profile} (${format})"
	case ${format} in
		"pstat")
			python3 -c "import pstats; pstats.Stats('${profile}').sort_stats('cumtime').print_stats('mini_buildd');" | less
			;;
		"callgrind")
			kcachegrind ${profile}
			;;
	esac
}

main()
{
	if [ -z "${1}" ]; then
		local p="./$(basename "${0}")"
		cat <<EOF
Usage: [<env>] ${p} <shortcut-or-runner-or-special> | run <groupRegex><levelRegex>${_reset} [<customArgs>...]

Helper script to develop/debug mini-buildd.

Prerequisites
-------------
 * Only run in *DEVELOPMENT* systems (it does DO THINGS, and maybe w/o asking)
 * 'sudo' should be configured
 * Known-Pre-Depends: lsb-release
 * eval \$(./devel bash-completion) is your friend

Test suite configuration via environment
----------------------------------------
$(for VAR in ${MBD_CONFIG}; do VAR_DESC="${VAR}_DESC"; printf "%s\n  %s\n" "${VAR}='${!VAR}'" "${!VAR_DESC}"; done)

HINT: Combinable profiles exist for common scenarios: ./devel profile _<TAB>

Shortcut commands
-----------------
$(for s in ${MBD_RUN_SHORTCUTS_SORTED}; do printf "  ${_it}${p} %-15s${_reset}: %s(%s)\n" "${s}" "$(mbd_sequence "${MBD_RUN_SHORTCUTS[${s}]}" "" "hr")" "${MBD_RUN_SHORTCUTS[${s}]}"; done)

Runners
-------
 ${p} $(mbd_sequence "..." "" "hr" | tr " " "|")

Examples
--------
${_it}Install snapshot Debian packages${_reset}:
 $ ./devel prepare-system  # Only 1st time
 $ ./devel update

${_it}Run full testuite (leaves you with a fresh mini-buildd fully set up, any previous install WILL BE PURGED!)${_reset}:
 $ ./devel updatetestall               # w/ default setup
 $ ./devel profile _all updatetestall  # w/ all known sources

${_it}Local setup to test remotes (on two systems w/ same network stack like chroots)${_reset}:
 $ ./devel profile _debian _remote_hostname update   # chroot 0
 $ ./devel profile _debian _remote_localhost update  # chroot 1

${_it}Follow service logs${_reset}:
 $ ${p} service.log  : Follow daemon logs.

${_it}Run all fully-automated tests possible${_reset}:
 $ ${p} supertestall
EOF
	else
		check_devsys
		local shortcut="${MBD_RUN_SHORTCUTS[${1}]}"
		if [ -n "${shortcut}" ]; then
			mbd_run "${shortcut}" "" "${@:2}"
		else
			local func="mbd_${1}"  # direct function
			if [ "$(type -t "${func}")" = "function" ]; then
				${func} "${@:2}"
			else
				mbd_run '.*' "${1}" "${@:2}"
			fi
		fi
	fi
}

main "${@}"