#!/bin/bash
#
# Regina - A Normal Surface Theory Calculator
# Python Test Suite Runner
#
# Copyright (c) 2007-2016, Ben Burton
# For further details contact Ben Burton (bab@debian.org).
#
# Usage: testall
#
# Runs the entire Python test suite.  This script searches for files
# named *.test in or beneath the current directory, runs them through
# regina-python, and compares the results byte-by-byte with the
# corresponding *.out files.  Any mismatches are considered to be test
# failures.
#
# This script MUST be run from the python/testsuite directory in the
# build tree.  It runs regina-python directly out of the build tree.
# It does not require regina to be installed on the system (and indeed
# it ignores any installation that it might find).
#
# Because tests might need to access other files within the source tree
# (which might not be the same as the build tree), the corresponding
# python/testsuite directory within the source tree will be passed as an
# argument to each test script.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# As an exception, when this program is distributed through (i) the
# App Store by Apple Inc.; (ii) the Mac App Store by Apple Inc.; or
# (iii) Google Play by Google Inc., then that store may impose any
# digital rights management, device limits and/or redistribution
# restrictions that are required by its terms of service.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
# MA 02110-1301, USA.

set -e

# To see the output, you need to run:
#     make test ARGS="-V"

# Change verbose to 1 if you wish to see the full command lines.
verbose=0

executable=../regina-python
testdir="@PROJECT_SOURCE_DIR@/python/testsuite"
testoutdir=`mktemp -d -t regina.XXX`

echo "---------------------------------------"
echo "Test suite for Regina's python bindings"
echo "---------------------------------------"
echo

if ! test -f ../../CMakeCache.txt -a -f "$executable"; then
	echo "ERROR: You do not appear to be within the python/testsuite directory"
	echo "       in the build tree.  This script must be run from directly"
	echo "       within the build tree (where you ran the configure script)."
	echo "       Please change into the python/testsuite directory in the"
	echo "       build tree and try again."
	exit 1
fi

# Clobber environment variables that the user might already have set.
export REGINA_VERBOSITY=
export REGINA_PYLIBDIR=
export REGINA_HOME=

broken=
found=

for i in `find "$testdir" -name "*.test" | sort`; do
	found=1
	testname=`basename "$i"`
	baseline="${i/.test/.out}"
	testout="$testoutdir/${testname/.test/.out}"
	echo -n "Running $testname ... "

	# Run the test.
	#
	# Since we have "set -e", we need "&& dummy=" to prevent this line to just
	# exit the bash script when the test has non-zero exit code.
	# We use && as opposed to || in order to preserve the exit code from
	# the first command.
	#
	# The code that unsets TERM fixes an issue seen some years ago on
	# Fedora, where (as a result of a more exotic TERM setting) unprintable
	# characters were being dumped at the beginning of the python output.
	#
	TERM= "$executable" "$i" "$testdir" > "$testout" 2>&1 && dummy=

	# Catch exitcode.
	exitcode=$?

	# Branch based on exit code.
	if [ "$exitcode" -ne 0 ];  then
		echo "FAILED (exit code $exitcode)"
		if [ "$verbose" -eq 1 ]; then
			echo "    Command was:"
			echo "        " \""$executable"\" \""$i"\" \""$testdir"\" ">" \""$testout"\" "2>&1"
		fi
		echo "    Output was:"
		echo "        ..."
		tail -n 5 $testout | sed -e 's/^/        /'
		broken=1
	else
		if ! ( diff --strip-trailing-cr "$baseline" "$testout" > /dev/null ); then
			echo "FAILED (output differs from expected)"
			if [ "$verbose" -eq 1 ]; then
				echo "    Command to diff:"
				echo "        " diff "--strip-trailing-cr" \""$baseline"\" \""$testout"\"
			fi
			echo "    Diff was:"
			diff --strip-trailing-cr -u \
				--label 'Expected output' --label 'Actual output' \
				"$baseline" "$testout" | head -n10 | sed -e 's/^/        /'
			echo "        ..."
			broken=1
		else
			echo "ok"
			rm -f "$testout"
		fi
	fi
done

if ! test -n "$found"; then
	echo "ERROR: No tests were found!"
	echo
	echo "       Please check your installation, and please ensure that"
	echo "       you are running this script from within the python/testsuite"
	echo "       directory of the build tree."
	echo
	echo "       Note that the build tree might not be the same as the source"
	echo "       tree.  The build tree is where you ran the configure script."
	exit 1
fi

if test -n "$broken"; then
	echo
	echo "One or more tests failed."
	echo "Please see the list above for details."
	echo
	echo "The full output file(s) produced by Regina are in the directory:"
	echo "    $testoutdir"
	exit 1
else
	rmdir "$testoutdir"
fi

echo
echo "All tests passed!"
exit 0
