#!/bin/bash
#
# Regina - A Normal Surface Theory Calculator
# Python Test Suite Runner
#
# Copyright (c) 2007-2025, Ben Burton
# For further details contact Ben Burton (bab@debian.org).
#
# Usage: testall
#
# Runs the entire Python test suite.  This script searches for files
# named *.test in or beneath the current directory, runs them through
# regina-python, and compares the results byte-by-byte with the
# corresponding *.out files.  Any mismatches are considered to be test
# failures.
#
# If the output is expected to differ between pybind11 2.x.y (used with
# Python 3.11 and earlier) vs pybind11 3.x.y (used with Python 3.12 and
# later), there should instead be two output files: *.out.v2 and *.out.v3.
# If *.out is present, this will take priority over *.out.v2 and/or *.out.v3.
#
# This script MUST be run from the python/testsuite directory in the
# build tree.  It runs regina-python directly out of the build tree.
# It does not require regina to be installed on the system (and indeed
# it ignores any installation that it might find).
#
# Because tests might need to access other files within the source tree
# (which might not be the same as the build tree), the corresponding
# python/testsuite directory within the source tree will be passed as an
# argument to each test script.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# As an exception, when this program is distributed through (i) the
# App Store by Apple Inc.; (ii) the Mac App Store by Apple Inc.; or
# (iii) Google Play by Google Inc., then that store may impose any
# digital rights management, device limits and/or redistribution
# restrictions that are required by its terms of service.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

set -e

# To see the output, you need to run:
#     make test ARGS="-V"

executable=../regina-python
testdir='@BASH_CMAKE_SOURCE_DIR@/python/testsuite'
testoutdir=`mktemp -d -t regina.XXX`
pybind11_version='@REGINA_PYBIND11_VERSION@'

echo "---------------------------------------"
echo "Test suite for Regina's python bindings"
echo "---------------------------------------"
echo

if ! test -f ../../CMakeCache.txt -a -f "$executable"; then
	echo "ERROR: You do not appear to be within the python/testsuite directory"
	echo "       in the build tree.  This script must be run from directly"
	echo "       within the build tree (where you ran cmake)."
	echo "       Please change into the python/testsuite directory in the"
	echo "       build tree and try again."
	exit 1
fi

# Clobber environment variables that the user might already have set.
export REGINA_VERBOSITY=
export REGINA_PYLIBDIR=
export REGINA_HOME=

broken=
found=

# We need to loop through all files $testdir/*.test .
# Because $testdir may contain spaces, we use a slightly tortured loop
# with the source 'find' statement at the end (not the beginning).
# See http://mywiki.wooledge.org/BashFAQ/020 for details.
#
# Note that both "find -print0" and "sort -z" (which are part of our machinery
# to handle spaces correctly) are implemented in both GNU and BSD utilities.
#
while IFS= read -r -d $'\0' i; do
	found=1
	testname=`basename "$i"`
	expected="${i/.test/.out}"
	if [ ! -e "$expected" ]; then
		expected="$expected.v$pybind11_version"
	fi
	observed="$testoutdir/${testname/.test/.out}"
	filter="$testdir/${testname/.test/.filter}"

	# Run the test.
	#
	# Since we have "set -e", we need "&& dummy=" to prevent this line to just
	# exit the bash script when the test has non-zero exit code.
	# We use && as opposed to || in order to preserve the exit code from
	# the first command.
	#
	# The code that unsets TERM fixes an issue seen some years ago on
	# Fedora, where (as a result of a more exotic TERM setting) unprintable
	# characters were being dumped at the beginning of the python output.
	#
	if [ -e "$filter" ]; then
		echo -n "Running $testname (with filter) ... "
		TERM= PYTHONIOENCODING=utf8 \
			"$executable" "$i" "$testdir" | "$filter" > "$observed" 2>&1 && dummy=
	else
		echo -n "Running $testname ... "
		TERM= PYTHONIOENCODING=utf8 \
			"$executable" "$i" "$testdir" > "$observed" 2>&1 && dummy=
	fi

	# Catch exitcode.
	exitcode=$?

	# Branch based on exit code.
	if [ "$exitcode" -ne 0 ];  then
		echo "FAILED (exit code $exitcode)"
		echo "    Output was:"
		echo "        ..."
		tail -n 5 $observed | sed -e 's/^/        /'
		broken=1
	else
		if ! ( diff --strip-trailing-cr "$expected" "$observed" > /dev/null ); then
			echo "FAILED (output differs from expected)"
			echo "    Diff was:"
			diff --strip-trailing-cr -u \
				--label 'Expected output' --label 'Actual output' \
				"$expected" "$observed" | head -n200 | sed -e 's/^/        /'
			echo "        ..."
			broken=1
		else
			echo "ok"
			rm -f "$observed"
		fi
	fi
done < <(find "$testdir" -name "*.test" -print0 | sort -z)

if ! test -n "$found"; then
	echo "ERROR: No tests were found!"
	echo
	echo "       Please check your installation, and please ensure that"
	echo "       you are running this script from within the python/testsuite"
	echo "       directory of the build tree."
	echo
	echo "       Note that the build tree might not be the same as the source"
	echo "       tree.  The build tree is where you ran the configure script."
	exit 1
fi

if test -n "$broken"; then
	echo
	echo "One or more tests failed."
	echo "Please see the list above for details."
	echo
	echo "The full output file(s) produced by Regina are in the directory:"
	echo "    $testoutdir"
	exit 1
else
	rmdir "$testoutdir"
fi

echo
echo "All tests passed!"
exit 0
