#!/bin/bash
#
# Regina - A Normal Surface Theory Calculator
# Python Test Suite Runner
#
# Copyright (c) 2007-2025, Ben Burton
# For further details contact Ben Burton (bab@debian.org).
#
# Usage: testbasic
#
# Runs most but not all of the Python test suite using a basic C++ executable
# with an embedded Python interpreter and an in-built timeout feature.
#
# Like testall, this script works through all *.test files in or beneath
# the current directory; however, unlike testall it skips those tests
# whose names are in a hard-coded blacklist.  Similar to testall, this
# script runs each test through the basic embedded Python interpreter and
# compares the results byte-by-byte with the corresponding *.out files
# (or, if the output differs between pybind11 versions, *.out.v2 and *.out.v3).
# Any mismatches or timeouts are considered to be test failures.
#
# This script MUST be run from the python/testsuite directory in the
# build tree.  It runs everything directly out of the build tree.
# It does not require regina to be installed on the system (and indeed
# it ignores any installation that it might find).
#
# Unlike testall, the corresponding python/testsuite directory will *not*
# be passed to the test scripts.  Any scripts that require this information
# should not be used with testbasic.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# As an exception, when this program is distributed through (i) the
# App Store by Apple Inc.; (ii) the Mac App Store by Apple Inc.; or
# (iii) Google Play by Google Inc., then that store may impose any
# digital rights management, device limits and/or redistribution
# restrictions that are required by its terms of service.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

set -e

# To see the output, you need to run:
#     make test ARGS="-V"

executable=./basic
testdir='@BASH_CMAKE_SOURCE_DIR@/python/testsuite'
testoutdir=`mktemp -d -t regina.XXX`
pybind11_version='@REGINA_PYBIND11_VERSION@'

# REGINA_HOME can be deduced from the executable path, but on Windows this
# deduction gives the wrong style of path (windows vs unix style) for use
# with the embedded basic Python interpreter.  Fix it explicitly here.
export REGINA_HOME=../../engine

# Cases that expect the python/testsuite directory as an argument:
blacklist="file i18n memory1 orb pdf repr snappea"

# The following cases implicitly import the threading module, which leads to a
# harmless but noisy ignored exception upon shutdown under python 3.12 and
# above.  See: https://github.com/pybind/pybind11/issues/2197
blacklist="$blacklist docstrings"

echo "----------------------------------------------------------------"
echo "Test suite for python bindings with a basic embedded interpreter"
echo "----------------------------------------------------------------"
echo

if ! test -f ../../CMakeCache.txt -a -f "$executable"; then
	echo "ERROR: You do not appear to be within the python/testsuite directory"
	echo "       in the build tree.  This script must be run from directly"
	echo "       within the build tree (where you ran cmake)."
	echo "       Please change into the python/testsuite directory in the"
	echo "       build tree and try again."
	exit 1
fi

# Make sure Python will be able to import regina.
os='@CMAKE_SYSTEM_NAME@'
case "$os" in
	Darwin )
		export DYLD_LIBRARY_PATH="../../engine:$DYLD_LIBRARY_PATH"
		export PYTHONPATH="..:$PYTHONPATH"
		;;
	Windows )
		# The PYTHONHOME directory is incorrectly hard-coded in msys2.
		# Work out the correct value by asking the python interpreter directly.
		PYTHON='@BASH_Python_EXECUTABLE@'
		if [ ! -e "$PYTHON" ]; then
			echo "ERROR: Python interpreter not found: $PYTHON"
			exit 1
		fi
		export PYTHONHOME="`"$PYTHON" -c 'import sys; print(sys.prefix)'`"
		if [ ! -d "$PYTHONHOME" ]; then
			echo "ERROR: Python home not found: $PYTHONHOME"
			exit 1
		fi

		export PATH="../../engine:$PATH"
		export PYTHONPATH="..;$PYTHONPATH"
		;;
	* )
		export LD_LIBRARY_PATH="../../engine:$LD_LIBRARY_PATH"
		export PYTHONPATH="..:$PYTHONPATH"
		;;
esac

# Python 3.10.[0-2] cannot process several of our test scripts due to a
# breakage in the line-by-line compilation (this was fixed with the
# introduction of the constant PyCF_ALLOW_INCOMPLETE_INPUT in Python 3.10.3).
# Note: these *are* still tested with testall (which runs within a normal python
# interpreter); just not testbasic (which embeds python in a C++ program).
pyver=`"$executable" -v`
case "$pyver" in
	3.10.[0-2]*)
		echo "WARNING: Skipping several tests due to broken line-by-line"
		echo "         compilation in embedded python 3.10.[0-2]."
		echo
		blacklist="$blacklist embeddings faces flype groups hypersurfaces"
		blacklist="$blacklist iterators listview presentations refs skeleton"
		blacklist="$blacklist standardtri"
		;;
	*)
		;;
esac

found=
broken=

# Decide on a suitable timeout (measured in seconds) for our tests.
case "$os" in
	Windows )
		# The Windows simulator is *enormously* slow.
		# Give it half an hour (gulp).
		timeout=1800
		;;
	* )
		# Linux builds may also be running under hardware emulation.
		# Give each test ten minutes, which is *way* more than it should need.
		timeout=600
		;;
esac

# We need to loop through all files $testdir/*.test .
# Because $testdir may contain spaces, we use a slightly tortured loop
# with the source 'find' statement at the end (not the beginning).
# See http://mywiki.wooledge.org/BashFAQ/020 for details.
#
# Note that both "find -print0" and "sort -z" (which are part of our machinery
# to handle spaces correctly) are implemented in both GNU and BSD utilities.
#
while IFS= read -r -d $'\0' i; do
	found=1
	testname=`basename "$i"`
	expected="${i/.test/.out}"
	if [ ! -e "$expected" ]; then
		expected="$expected.v$pybind11_version"
	fi
	observed="$testoutdir/${testname/.test/.out}"
	filter="$testdir/${testname/.test/.filter}"

	blacklisted=0
	for b in $blacklist; do
		if [ "$testname" = "$b.test" ]; then
			blacklisted=1
			break
		fi
	done
	if [ "$blacklisted" = 1 ]; then
		echo "Skipping $testname"
		continue
	fi

	# We do not test with a separate execution thread for now, since the
	# current implementation of PythonInterpreter requires code to be executed
	# from the same thread that created the interpreter.
	#
	# for multithreading in 0 1; do
	for multithreading in 0; do
		# Run the test.
		#
		# Since we have "set -e", we need "&& dummy=" to prevent this line to
		# just exit the bash script when the test has non-zero exit code.
		# We use && as opposed to || in order to preserve the exit code from
		# the first command.
		#
		# The code that unsets TERM fixes an issue seen some years ago on
		# Fedora, where (as a result of a more exotic TERM setting) unprintable
		# characters were being dumped at the beginning of the python output.
		#
		if [ -e "$filter" ]; then
			if [ "$multithreading" = 0 ]; then
				# echo -n "Running $testname (with filter, main thread) ... "
				echo -n "Running $testname (with filter) ... "
				TERM= PYTHONIOENCODING=utf8 \
					"$executable" -t "$timeout" "$i" | "$filter" > "$observed" 2>&1 && dummy=
			else
				echo -n "Running $testname (with filter, separate thread) ... "
				TERM= PYTHONIOENCODING=utf8 \
					"$executable" -m -t "$timeout" "$i" | "$filter" > "$observed" 2>&1 && dummy=
			fi
		else
			if [ "$multithreading" = 0 ]; then
				# echo -n "Running $testname (main thread) ... "
				echo -n "Running $testname ... "
				TERM= PYTHONIOENCODING=utf8 \
					"$executable" -t "$timeout" "$i" > "$observed" 2>&1 && dummy=
			else
				echo -n "Running $testname (separate thread) ... "
				TERM= PYTHONIOENCODING=utf8 \
					"$executable" -m -t "$timeout" "$i" > "$observed" 2>&1 && dummy=
			fi
		fi

		# Catch exitcode.
		exitcode=$?

		# Branch based on exit code.
		if [ "$exitcode" -ne 0 ];  then
			echo "FAILED (exit code $exitcode)"
			echo "    Output was:"
			echo "        ..."
			tail -n 5 $observed | sed -e 's/^/        /'
			broken=1
		else
			if ! ( diff --strip-trailing-cr "$expected" "$observed" > /dev/null ); then
				echo "FAILED (output differs from expected)"
				echo "    Diff was:"
				diff --strip-trailing-cr -u \
					--label 'Expected output' --label 'Actual output' \
					"$expected" "$observed" | head -n200 | sed -e 's/^/        /'
				echo "        ..."
				broken=1
			else
				echo "ok"
				rm -f "$observed"
			fi
		fi
	done
done < <(find "$testdir" -name "*.test" -print0 | sort -z)

if ! test -n "$found"; then
	echo "ERROR: No tests were found!"
	echo
	echo "       Please check your installation, and please ensure that"
	echo "       you are running this script from within the python/testsuite"
	echo "       directory of the build tree."
	echo
	echo "       Note that the build tree might not be the same as the source"
	echo "       tree.  The build tree is where you ran the configure script."
	exit 1
fi

if test -n "$broken"; then
	echo
	echo "One or more tests failed."
	echo "Please see the list above for details."
	echo
	echo "The full output file(s) produced by Regina are in the directory:"
	echo "    $testoutdir"
	exit 1
else
	rmdir "$testoutdir"
fi

echo
echo "All tests passed!"
exit 0