############################################################################
 #                                                                          #
 #                            REPORTS.PY                                    #
 #                                                                          #
 #           Copyright (C) 2008 - 2010 Ada Core Technologies, Inc.          #
 #                                                                          #
 # This program is free software: you can redistribute it and/or modify     #
 # it under the terms of the GNU General Public License as published by     #
 # the Free Software Foundation, either version 3 of the License, or        #
 # (at your option) any later version.                                      #
 #                                                                          #
 # This program is distributed in the hope that it will be useful,          #
 # but WITHOUT ANY WARRANTY; without even the implied warranty of           #
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
 # GNU General Public License for more details.                             #
 #                                                                          #
 # You should have received a copy of the GNU General Public License        #
 # along with this program.  If not, see <http://www.gnu.org/licenses/>     #
 #                                                                          #
 ############################################################################

"""This package contains various classes to manipulate data generated by the
testsuites. We assume a testsuite result is contained in a directory. The
directory should have the following structure::

    root_dir/
        results (contains the test results)
        <test_name>.note (a note associated with test <test_name> - optional)
        <test_name>.out  (actual test output - optional)
        <test_name>.expected (expected test output - optional)

The format of the results file is the following::

    TEST1_NAME:STATUS[:MESSAGE]
    TEST2_NAME:STATUS[:MESSAGE]
    ...

STATUS is one of following string::

    DIFF, PROBLEM, FAILED,
    CRASH,
    INVALID_TEST,
    XFAIL, SKIP,
    UOK,
    DEAD,
    PASSED, OK, NOT-APPLICABLE, TENTATIVELY_PASSED

MESSAGE is an optional message
"""

from gnatpython.main import Main
from gnatpython.fileutils import split_file, FileUtilsError
import os.path

# This following lists regroup the status in different categories. A given
# status might appear in several categories. For convenience, the following
# code is using only categories, so that adding new test status result only
# in adding it in the following lists.
FAIL = ['DIFF', 'PROBLEM', 'FAILED']
CRASH = ['CRASH']
INVALID = ['INVALID_TEST']
XFAIL = ['XFAIL', 'SKIP']
UPASS = ['UOK']
DEAD = ['DEAD']
PASS = ['PASSED', 'OK', 'NOT-APPLICABLE', 'TENTATIVELY_PASSED']
FAIL_OR_CRASH = CRASH + FAIL
NON_DEAD = PASS + UPASS + XFAIL + INVALID + FAIL + CRASH


class TestResult(object):
    """Class that holds test information

    ATTRIBUTES
      dir: result directory. We expect to find all test related file in it
      name: test name
      status: test status
      msg: associated message
    """

    def __init__(self, dir, name, status, msg):
        """TestResult constructor

        PARAMETERS
          dir: result directory
          name: the test name
          status: test status
          msg: test message

        RETURN VALUE
          a TestResult object
        """
        self.dir = dir
        self.name = name
        self.status = status
        self.msg = msg

    def __get_file(self, ext):
        """Internal function that retrieves a file associated with a test

        PARAMETERS
          ext: extension of the file to look at

        RETURN VALUE
          the content of the file or None
        """
        filename = self.dir + '/' + self.name + ext
        if os.path.isfile(filename):
            fd = open(filename, 'rb')
            result = fd.read()
            fd.close()
            return result
        else:
            return None

    def get_note(self):
        """Retrieve the note associated with the test

        PARAMETERS
          None

        RETURN VALUE
          A string. If there is no note associated with the test, the null
          string is returned
        """
        note = self.__get_file('.note')
        if note is None:
            return ''
        else:
            return note.strip()

    def get_expected_output(self):
        """Retrieve test actual output

        PARAMETERS
          None

        RETURN VALUE
          test actual output or None
        """
        return self.__get_file('.expected')

    def get_actual_output(self):
        """Retrieve test expected output

        PARAMETERS
          None

        RETURN VALUE
          test expected output or None
        """
        return self.__get_file('.out')

    def __str__(self):
        return '%s:%s:%s' % (self.name, self.status, self.msg)


class Report(object):
    """Class that holds a complete testsuite result"""

    def __init__(self, dir):
        """Report constructor

        PARAMETERS
          dir: the directory that contains the testsuite results

        RETURN VALUE
          a Report Object
        """
        self.dir = dir
        self.result_db = {}

        if self.dir is not None:
            assert os.path.isdir(self.dir), "invalid result directory"

            result_list = split_file(dir + '/results')
            result_list = (k.split(':', 2) for k in result_list)

            for item in result_list:
                msg = ''
                if len(item) > 2:
                    msg = item[2]

                self.result_db[item[0]] = \
                  TestResult(self.dir, item[0], item[1], msg)

    def select(self, kind=None, return_set=False):
        """Retrieve the list/set of tests that match a given list of status

        PARAMETERS
          kind: None or a list of status. If None the complete list of test
                names will be returned
          return_set: if True then a set object is returned. Otherwise a
                sorted list

        RETURN VALUE
          a list or a set of test names
        """

        result = None
        if kind is None:
            result = set([k for k in self.result_db])
        else:
            result = set([k for k in self.result_db \
                           if self.result_db[k].status in kind])

        if not return_set:
            result = list(result)
            result.sort()

        return result

    def test(self, name):
        """Get a test object from the testsuite

        PARAMETERS
          name: name of the test we want to retrieve

        RETURN VALUE
          a TestResult object
        """
        return self.result_db[name]


# A few constant use by ReportDiff.select method
IN_ONE = 0
IN_BOTH = 1
IN_NEW_ONLY = 2
IN_OLD_ONLY = 3


class ReportDiff(object):
    """Class that allows comparison between two testsuite reports

    ATTRIBUTES
      new: a Report object that contains the new results
      old: a Report object that contains the old results
    """

    def __init__(self, dir, old_dir=None):
        """ReportDiff constructor

        PARAMETERS
          dir: the directory of the new testsuite result
          old_dir: the directory of the old testsuite result

        RETURN VALUE
          a ReportDiff object
        """
        self.new = Report(dir)

        if old_dir is not None and not (
            os.path.isdir(old_dir) and os.path.isfile(old_dir + '/results')):
            # There is no old report. Skip it.
            old_dir = None
        try:
            self.old = Report(old_dir)
        except FileUtilsError:
            self.old = Report(None)

    def select(self, kind=None, status=IN_BOTH, old_kind=None):
        """Do a query

        PARAMETERS
          kind: a list of status or None used to filter the tests in the new
            report
          status: a selector IN_ONE, IN_BOTH, IN_NEW_ONLY or IN_OLD_ONLY.
          old_kind: a list of status used to filter in the old report. If None
            the parameter kind is reused

        RETURN VALUE
          a sorted list of test names
        """

        if old_kind is None:
            old_kind = kind
        new = self.new.select(kind, return_set=True)
        old = self.old.select(old_kind, return_set=True)
        result = None
        if status == IN_BOTH:
            result = new & old
        elif status == IN_NEW_ONLY:
            result = new - old
        elif status == IN_OLD_ONLY:
            result = old - new
        else:
            result = old | new

        return result

    def txt_image(self, filename):
        if isinstance(filename, str):
            fd = open(filename, 'wb+')
        else:
            fd = filename

        def output_list(tests, tmpl_msg, output_if_null=False, from_old=False):
            if len(tests) > 0 or output_if_null:
                tmpl_msg = '---------------- %d %s\n' % (len(tests), tmpl_msg)
                fd.write('\n')
                fd.write(tmpl_msg)
                for test in tests:
                    if from_old:
                        fd.write('%s\n' % self.old.test(test))
                    else:
                        fd.write('%s\n' % self.new.test(test))

        def test_diff(test_name):
            test_obj = self.new.test(test_name)
            test_note = test_obj.get_note()
            test_out = test_obj.get_actual_output()
            test_exp = test_obj.get_expected_output()

            fd.write("================ Bug %s %s\n" % (test_name, test_note))
            if test_exp is None:
                fd.write('---------------- unexpected output\n')
                if test_out is not None:
                    fd.write(test_out)
            else:
                fd.write('---------------- expected output\n')
                if test_exp is not None:
                    fd.write(test_exp)
                fd.write('---------------- actual output\n')
                if test_out is not None:
                    fd.write(test_out)

        constant = self.select(FAIL, IN_BOTH)
        xfail_tests = self.new.select(XFAIL)
        uok_tests = self.new.select(UPASS)

        fixed_tests = self.select(PASS, IN_BOTH, FAIL_OR_CRASH)
        invalid_tests = self.new.select(INVALID)
        new_dead_tests = self.select(DEAD, IN_BOTH, NON_DEAD)
        removed_tests = self.select(status=IN_OLD_ONLY)
        crash_tests = self.new.select(CRASH)
        dead_tests = self.new.select(DEAD)
        complete_tests = self.new.select()
        diff_tests = self.new.select(FAIL)
        new_regressions = self.select(FAIL_OR_CRASH, IN_NEW_ONLY)
        non_dead_tests = self.new.select(NON_DEAD)

        fd.write('Out of %d tests:\n' % len(complete_tests))
        fd.write('%5d executed test(s) (non dead)\n' % len(non_dead_tests))
        fd.write('%5d crash(es) detected\n' % len(crash_tests))
        fd.write('%5d other potential regression(s)\n' % len(diff_tests))
        fd.write('%5d expected regression(s)\n' % len(xfail_tests))
        fd.write('%5d unexpected passed test(s)\n' % len(uok_tests))
        fd.write('%5d invalid test(s)\n' % len(invalid_tests))
        fd.write('%5d new dead test(s)\n' % len(new_dead_tests))
        fd.write('%5d test(s) removed\n' % len(removed_tests))

        output_list(new_regressions, 'new regression(s)', True)
        output_list(constant, 'already detected regression(s)', True)
        output_list(xfail_tests, 'expected regression(s)')
        output_list(uok_tests, 'unexpected passed test(s)')
        output_list(fixed_tests, 'fixed regression(s)')
        output_list(invalid_tests, 'invalid test(s)')
        output_list(new_dead_tests, 'new dead test(s)')
        output_list(removed_tests, 'test(s) removed', from_old=True)

        fd.write('\n---------------- differences in output\n')
        for test in self.new.select(FAIL):
            test_diff(test)

        if len(xfail_tests) > 0:
            fd.write('\n---------------- XFAIL differences in output\n')
            for test in xfail_tests:
                test_diff(test)

        if isinstance(filename, str):
            fd.close()