#!/usr/bin/env python3
# pylint: disable=C0301,R0902,R0914,R0912,R0913,R0915,R1702,C0302
# SPDX-License-Identifier: (GPL-2.0 OR MIT)

## Copyright (C) 2023    Intel Corporation                 ##
## Author: Mauro Carvalho Chehab <mchehab@kernel.org>      ##
##                                                         ##
## Allow keeping inlined test documentation and validate   ##
## if the documentation is kept updated.                   ##

"""Maintain test plan and test implementation documentation on IGT."""

import glob
import json
import os
import re
import sys

MIN_PYTHON = (3, 6)
if sys.version_info < MIN_PYTHON:
    sys.exit("Python %s.%s or later is required.\n" % MIN_PYTHON) # pylint: disable=C0209

#
# ancillary functions to sort dictionary hierarchy
#
def _sort_per_level(item):
    if "level" not in item[1]["_properties_"]:
        return item[0]

    return "%05d_%05d_%s" % (item[1]["_properties_"]["level"], item[1]["_properties_"]["sublevel"], item[0])   # pylint: disable=C0209

def _sort_using_array(item, array):
    ret_str = ''
    for field in array:
        if field in item[1]:
            ret_str += '_' + field + '_' + item[1][field]

    if ret_str == '':
        ret_str="________"

    return ret_str

#
# Ancillary logic to allow plurals on fields
#
# As suggested at https://stackoverflow.com/questions/18902608/generating-the-plural-form-of-a-noun/19018986
#

def _plural(field):

    """
    Poor man's conversion to plural.

    It should cover usual English plural rules, although it is not meant
    to cover exceptions (except for a few ones that could be useful on actual
    fields).

    """

    match = re.match(r"(.*)\b(\S+)", field)
    if match:
        ret_str = match.group(1)
        word = match.group(2)

        if word.isupper():
            ret_str += word
        elif word in ["of", "off", "on", "description", "todo"]:
            ret_str += word
        elif word.endswith('ed'):
            ret_str += word
        elif word[-1:] in ['s', 'x', 'z']:
            ret_str += word + 'es'
        elif word[-2:] in ['sh', 'ch']:
            ret_str += word + 'es'
        elif word.endswith('fe'):
            ret_str += word[:-2] + 'ves'
        elif word.endswith('f'):
            ret_str += word[:-1] + 'ves'
        elif word.endswith('y'):
            ret_str += word[:-1] + 'ies'
        elif word.endswith('o'):
            ret_str += word + 'es'
        elif word.endswith('us'):
            ret_str += word[:-2] + 'i'
        elif word.endswith('on'):
            ret_str += word[:-2] + 'a'
        elif word.endswith('an'):
            ret_str += word[:-2] + 'en'
        else:
            ret_str += word + 's'

        return ret_str

    return field

#
# TestList class definition
#
class TestList:

    """
    Parse and handle test lists with test/subtest documentation, in the
    form of C comments, with two meta-tags (by default, TEST and SUBTEST),
    and a set of `field: value` items:

        /**
         * TEST: Check if new IGT test documentation logic functionality is working
         * Category: Software build block
         * Sub-category: documentation
         * Functionality: test documentation
         * Issue: none
         * Description: Complete description of this test
         *
         * SUBTEST: foo
         * Description: do foo things
         * 	description continuing on another line
         *
         * SUBTEST: bar
         * Description: do bar things
         * 	description continuing on another line
         */

        /**
         * SUBTEST: test-%s-binds-%s-with-%ld-size
         * Description: Test arg[2] arg[1] binds with arg[3] size
         *
         * SUBTEST: test-%s-%ld-size
         * Description: Test arg[1] with %arg[2] size
         *
         * arg[1]:
         *
         * @large:	large
         * 		something
         * @mixed:	mixed
         * 		something
         * @small:	small
         * 		something
         *
         * arg[2]:	buffer size
         * 		of some type
         *
         * arg[3]:
         *
         * @binds:			foo
         * @misaligned-binds:		misaligned
         * @userptr-binds:		userptr
         * @userptr-misaligned-binds:	userptr misaligned
         */

    It is also possible to define a list of values that will actually be
    used by the test and no string replacement is needed.
    This is done by using one or multiple arg[n].values lines:

        /**
         * SUBTEST: unbind-all-%d-vmas
         * Description: unbind all with %arg[1] VMAs
         *
         * arg[1].values: 2, 8
         * arg[1].values: 16, 32
         */

        /**
         * SUBTEST: unbind-all-%d-vmas
         * Description: unbind all with %arg[1] VMAs
         *
         * arg[1].values: 64, 128
         */

    Such feature is specially useful for numeric parameters

    The valid `field` values are defined on a JSON configuration file.

    The TEST documents the common fields present on all tests. Typically,
    each IGT C test file should contain just one such field.

    The SUBTEST contains the fields that are specific to each subtest.

    Note: when igt_simple_main is used, there will be a single nameless
    subtest. On such case, "SUBTEST:" is still needed, but without a
    test name on it, e. g., it would be documented as:

        /**
         * TEST: some test that uses igt_simple_main
         * Category: Software build block
         * Sub-category: documentation
         * Functionality: test documentation
         * Issue: none
         * Description: Complete description of this test
         *
         * SUBTEST:
         * Description: do foo things
         */

    Some IGT tests may have strings or integer wildcards, like:
        test-%s-%ld-size

    Those are declared using igt_subtest_f() and similar macros.

    The wildcard arguments there need to be expanded. This is done by
    defining arg[1] to arg[n] at the same code comment that contains the
    SUBTEST as those variables are locally processed on each comment line.

    This script needs a configuration file, in JSON format, describing the
    fields which will be parsed for TEST and/or SUBTEST tags.

    An example of such file is:

    {
        "files": [ "tests/driver/*.c" ],
        "exclude_files": [ "tests/driver/*-helper.c" ],
        "fields": {
            "Category": {
                "Sub-category": {
                    "Functionality": {
                    }
                }
            },
            "Issue": {
                "_properties_": {
                    "description": "If the test is used to solve an issue, point to the URL containing the issue."
                }
            },
            "Description" : {
                "_properties_": {
                    "description": "Provides a description for the test/subtest."
                }
            }
        }
    }

    So, the above JSON config file expects tags like those:

    TEST: foo
    Description: foo

    SUBTEST: bar
    Category: Hardware
    Sub-category: EU
    Description: test bar on EU

    SUBTEST: foobar
    Category: Software
    Type: ioctl
    Description: test ioctls
    """

    def __init__(self, config_fname = None,
                 include_plan = False, file_list = None,
                 igt_build_path = None,
                 config_dict = None, sources_path = None,
                 test_tag = "TEST", subtest_tag = "SUBTESTS?",
                 main_name = "igt", planned_name = "planned",
                 subtest_separator = "@"):
        self.doc = {}
        self.test_number = 0
        self.config = None
        self.filenames = file_list
        self.plan_filenames = []
        self.props = {}
        self.igt_build_path = igt_build_path
        self.level_count = 0
        self.field_list = {}
        self.title = None
        self.filters = {}
        self.subtest_separator = subtest_separator

        # Set a default value
        driver_name = main_name

        self.internal_fields =  [ '_summary_', '_arg_', '_subtest_line_' ]

        # Exclusive or: either one is needed
        if bool(config_fname) == bool(config_dict):
            sys.exit("Error: either config filename or config dict shall be used")

        if main_name:
            main_name += subtest_separator

        if planned_name:
            planned_name += subtest_separator

        implemented_class = None

        if config_fname:
            with open(config_fname, 'r', encoding='utf8') as handle:
                self.config = json.load(handle)

            config_origin = config_fname
            cfg_path = os.path.realpath(os.path.dirname(config_fname)) + "/"
            driver_name = re.sub(r'(.*/)?([^\/]+)/.*', r'\2', config_fname).capitalize()

        else:
            self.config = config_dict
            config_origin = "config dict"
            cfg_path = "./"

        if "drivers" in self.config:
            self.drivers = self.config["drivers"]
        else:
            self.drivers = [driver_name]

        if sources_path:
            cfg_path = os.path.realpath(sources_path) + "/"

        if not self.config:
            sys.exit("Error: configuration is empty!")

        self.__add_field(None, 0, 0, self.config["fields"])

        sublevel_count = [ 0 ] * self.level_count

        for field, item in self.props.items():
            if "sublevel" in item["_properties_"]:
                level = item["_properties_"]["level"]
                sublevel = item["_properties_"]["sublevel"]
                if sublevel > sublevel_count[level - 1]:
                    sublevel_count[level - 1] = sublevel

            field_lc = field.lower()
            self.field_list[field_lc] = field
            field_plural = _plural(field_lc)
            if field_lc != field_plural:
                self.field_list[field_plural] = field

        if include_plan:
            self.props["Class"] = {}
            self.props["Class"]["_properties_"] = {}
            self.props["Class"]["_properties_"]["level"] = 1
            self.props["Class"]["_properties_"]["sublevel"] = sublevel_count[0] + 1

        # Remove non-multilevel items, as we're only interested on
        # hierarchical item levels here
        for field, item in self.props.items():
            if "sublevel" in item["_properties_"]:
                level = item["_properties_"]["level"]
                if sublevel_count[level - 1] == 1:
                    del item["_properties_"]["level"]
                    del item["_properties_"]["sublevel"]

            update = self.props[field]["_properties_"].get("update-from-file")
            if update:
                # Read testlist files if any
                if "include" in update:
                    testlist = {}
                    for value in update["include"]:
                        for name in value.keys():
                            match_type = update.get("include-type", "subtest-match")

                            self.read_testlist(match_type, main_name, field,
                                               testlist, name,
                                               cfg_path + value[name])

                    update["include"] = testlist

                # Read blocklist files if any
                if "exclude" in update:
                    testlist = {}
                    for value in update["exclude"]:
                        for name in value.keys():
                            match_type = update.get("exclude-type", "subtest-match")

                            self.read_testlist(match_type, main_name, field,
                                               testlist, name,
                                               cfg_path + value[name])

                    update["exclude"] = testlist

        if "_properties_" in self.props:
            del self.props["_properties_"]

        has_implemented = False
        if not self.filenames:
            self.filenames = []
            exclude_files = []
            files = self.config["files"]
            exclude_file_glob = self.config.get("exclude_files", [])
            for cfg_file in exclude_file_glob:
                cfg_file = cfg_path + cfg_file
                for fname in glob.glob(cfg_file):
                    exclude_files.append(fname)

            for cfg_file in files:
                cfg_file = cfg_path + cfg_file
                for fname in glob.glob(cfg_file):
                    if fname in exclude_files:
                        continue
                    self.filenames.append(fname)
                    has_implemented = True
        else:
            for cfg_file in self.filenames:
                if cfg_file:
                    has_implemented = True

        has_planned = False
        if include_plan and "planning_files" in self.config:
            implemented_class = "Implemented"
            files = self.config["planning_files"]
            for cfg_file in files:
                cfg_file = cfg_path + cfg_file
                for fname in glob.glob(cfg_file):
                    self.plan_filenames.append(fname)
                    has_planned = True

        planned_class = None
        if has_implemented:
            if has_planned:
                planned_class = "Planned"
                self.title = "Planned and implemented "
            else:
                self.title = "Implemented "
        else:
            if has_planned:
                self.title = "Planned "
            else:
                sys.exit("Need file names to be processed")

        self.title += self.config.get("name", "tests for " + driver_name + " driver")

        # Parse files, expanding wildcards
        field_re = re.compile(r"(" + '|'.join(self.field_list.keys()) + r'):\s*(.*)', re.I)

        for fname in self.filenames:
            if fname == '':
                continue

            self.__add_file_documentation(fname, main_name,
                                          implemented_class, field_re,
                                          test_tag, subtest_tag, config_origin)

        if include_plan:
            for fname in self.plan_filenames:
                self.__add_file_documentation(fname, planned_name,
                                              planned_class, field_re,
                                              test_tag, subtest_tag, config_origin)

    #
    # ancillary methods
    #

    def __add_field(self, name, sublevel, hierarchy_level, field):

        """ Flatten config fields into a non-hierarchical dictionary """

        for key in field:
            if key not in self.props:
                self.props[key] = {}
                self.props[key]["_properties_"] = {}

            if name:
                if key == "_properties_":
                    if key not in self.props:
                        self.props[key] = {}
                    self.props[name][key].update(field[key])

                    sublevel += 1
                    hierarchy_level += 1
                    if "sublevel" in self.props[name][key]:
                        if self.props[name][key]["sublevel"] != sublevel:
                            sys.exit(f"Error: config defined {name} as sublevel {self.props[key]['sublevel']}, but wants to redefine as sublevel {sublevel}")

                    self.props[name][key]["level"] = self.level_count
                    self.props[name][key]["sublevel"] = sublevel

                    continue
            else:
                self.level_count += 1

            self.__add_field(key, sublevel, hierarchy_level, field[key])

    def read_testlist(self, match_type, base_name, field, testlist,
                      name, filename):

        """
        Read a list of tests with a common value from a file.

        Tests on this list are matched only for actual tests, not
        for planned ones.
        """

        match_type_regex = set(["regex", "regex-ignorecase"])
        match_type_str = set(["subtest-match"])
        match_types = match_type_regex | match_type_str

        if match_type not in match_types:
            sys.exit(f"Error: invalid match type '{match_type}' for {field}")

        if match_type == "regex":
            flags = 0
        elif match_type == "regex-ignorecase":
            flags = re.IGNORECASE

        base = r"^\s*({}[^\s\{}]+)(\S*)\s*(\#.*)?$"
        regex = re.compile(base.format(base_name, self.subtest_separator))

        if name not in testlist:
            testlist[name] = []
        with open(filename, 'r', newline = '', encoding = 'utf8') as fp:
            for line in fp:
                match = regex.match(line)
                if not match:
                    continue

                test = match.group(1)
                subtest = match.group(2)
                test_name = f"{test}{subtest}"
                if not test_name.endswith("$"):
                    test_name += r"(\@.*)?$"

                if match_type in match_type_regex:
                    testlist[name].append(re.compile(test_name, flags))
                    continue

                subtests = test_name.split(self.subtest_separator)[:3]

                prefix=""
                for subtest in subtests:
                    subtest = prefix + subtest
                    prefix = subtest + self.subtest_separator

                testlist[name].append(re.compile(f"{subtest}(@.*)?"))

    def __filter_subtest(self, test, subtest, field_not_found_value):

        """ Apply filter criteria to subtests """

        for filter_field, regex in self.filters.items():
            if filter_field in subtest:
                if not regex.search(subtest[filter_field]):
                    return True
            elif filter_field in test:
                if not regex.search(test[filter_field]):
                    return True
            else:
                return field_not_found_value

        # None of the filtering rules were applied
        return False

    def update_testlist_field(self, subtest_dict):

        """process include and exclude rules used by test lists read from files"""

        expand = re.compile(r",\s*")

        for field in self.props.keys(): # pylint: disable=C0201,C0206
            if "_properties_" not in self.props[field]:
                continue

            update = self.props[field]["_properties_"].get("update-from-file")
            if not update:
                continue

            default_value = update.get("default-if-not-excluded")
            append_value = update.get("append-value-if-not-excluded")

            testname = subtest_dict["_summary_"]

            value = subtest_dict.get(field)
            if value:
                values = set(expand.split(value))
            else:
                values = set()

            if append_value:
                include_names = set(expand.split(append_value))
                values.update(include_names)

            for names, regex_array in update.get("include", {}).items():
                name = set(expand.split(names))
                for regex in regex_array:
                    if regex.fullmatch(testname):
                        values.update(name)
                        break

            # If test is at a global blocklist, ignore it
            set_default = True
            for names, regex_array in update.get("exclude", {}).items():
                deleted_names = set(expand.split(names))
                for regex in regex_array:
                    if regex.fullmatch(testname):
                        if sorted(deleted_names) == sorted(values):
                            set_default = False
                        values -= deleted_names

            if default_value and set_default and not values:
                values.update([default_value])

            if values or self.props[field]["_properties_"].get("mandatory"):
                subtest_dict[field] = ", ".join(sorted(values))

    def expand_subtest(self, fname, test_name, test, allow_inherit, with_lines = False, with_subtest_nr = False):

        """Expand subtest wildcards providing an array with subtests"""

        subtest_array = []

        for subtest in self.doc[test]["subtest"].keys():
            summary = test_name
            if self.doc[test]["subtest"][subtest]["_summary_"] != '':
                summary += self.subtest_separator + self.doc[test]["subtest"][subtest]["_summary_"]
            if not summary:
                continue

            num_vars = summary.count('%')
            file_ln = self.doc[test]["_subtest_line_"][subtest]

            # Handle trivial case: no wildcards
            if num_vars == 0:
                subtest_dict = {}

                subtest_dict["_summary_"] = summary

                for k in sorted(self.doc[test]["subtest"][subtest].keys()):
                    if k in self.internal_fields:
                        continue

                    if not allow_inherit:
                        if k in self.doc[test] and self.doc[test]["subtest"][subtest][k] == self.doc[test][k]:
                            continue

                    subtest_dict[k] = self.doc[test]["subtest"][subtest][k]

                self.update_testlist_field(subtest_dict)

                if with_lines:
                    subtest_dict["line"] = file_ln

                if with_subtest_nr:
                    subtest_dict["subtest_nr"] = subtest

                subtest_array.append(subtest_dict)

                continue

            # Handle summaries with wildcards

            # Convert subtest arguments into an array
            arg_array = {}
            arg_ref = self.doc[test]["subtest"][subtest]["_arg_"]

            for arg_k in self.doc[test]["_arg_"][arg_ref].keys():
                arg_array[arg_k] = []
                if int(arg_k) > num_vars:
                    continue

                for arg_el in sorted(self.doc[test]["_arg_"][arg_ref][arg_k].keys()):
                    arg_array[arg_k].append(arg_el)

            size = len(arg_array)

            if size < num_vars:
                sys.exit(f"{fname}:subtest {summary} needs {num_vars} arguments but only {size} are defined.")

            for j in range(0, num_vars):
                if arg_array[j] is None:
                    sys.exit(f"{fname}:subtest{summary} needs arg[{j}], but this is not defined.")


            # convert numeric wildcards to string ones
            summary = re.sub(r'%(d|ld|lld|i|u|lu|llu)','%s', summary)

            # Expand subtests
            pos = [ 0 ] * num_vars
            args = [ 0 ] * num_vars
            arg_map = [ 0 ] * num_vars
            while 1:
                for j in range(0, num_vars):
                    arg_val = arg_array[j][pos[j]]
                    args[j] = arg_val

                    if arg_val in self.doc[test]["_arg_"][arg_ref][j]:
                        arg_map[j] = self.doc[test]["_arg_"][arg_ref][j][arg_val]
                        if re.match(r"\<.*\>", self.doc[test]["_arg_"][arg_ref][j][arg_val]):
                            args[j] = "<" + arg_val + ">"
                    else:
                        arg_map[j] = arg_val

                arg_summary = summary % tuple(args)

                # Store the element
                subtest_dict = {}
                subtest_dict["_summary_"] = arg_summary

                for field in sorted(self.doc[test]["subtest"][subtest].keys()):
                    if field in self.internal_fields:
                        continue

                    sub_field = self.doc[test]["subtest"][subtest][field]
                    sub_field = re.sub(r"%?\barg\[(\d+)\]", lambda m: arg_map[int(m.group(1)) - 1], sub_field) # pylint: disable=W0640

                    if not allow_inherit:
                        if field in self.doc[test]:
                            if sub_field in self.doc[test][field] and sub_field == self.doc[test][field]:
                                continue

                    subtest_dict[field] = sub_field

                self.update_testlist_field(subtest_dict)

                if with_lines:
                    subtest_dict["line"] = file_ln

                if with_subtest_nr:
                    subtest_dict["subtest_nr"] = subtest

                subtest_array.append(subtest_dict)

                # Increment variable inside the array
                arr_pos = 0
                while pos[arr_pos] + 1 >= len(arg_array[arr_pos]):
                    pos[arr_pos] = 0
                    arr_pos += 1
                    if arr_pos >= num_vars:
                        break

                if arr_pos >= num_vars:
                    break

                pos[arr_pos] += 1

        return subtest_array

    def expand_dictionary(self, subtest_only):

        """ prepares a dictionary with subtest arguments expanded """

        test_dict = {}

        for test in self.doc:                   # pylint: disable=C0206
            fname = self.doc[test]["File"]
            base_name = self.doc[test]["_base_name_"]

            name = re.sub(r'.*/', '', fname)
            name = re.sub(r'\.[\w+]$', '', name)
            name = base_name + name

            if not subtest_only:
                test_dict[name] = {}

                for field in self.doc[test]:
                    if field in self.internal_fields:
                        continue

                    test_dict[name][field] = self.doc[test][field]
                dic = test_dict[name]
            else:
                dic = test_dict

            subtest_array = self.expand_subtest(fname, name, test, subtest_only)
            for subtest in subtest_array:
                if self.__filter_subtest(self.doc[test], subtest, True):
                    continue

                summary = subtest["_summary_"]

                dic[summary] = {}
                for field in sorted(subtest.keys()):
                    if field in self.internal_fields:
                        continue
                    dic[summary][field] = subtest[field]

        return test_dict

    #
    # Output methods
    #

    def print_rest_flat(self, filename = None, return_string = False): # pylint: disable=R1710

        """Print tests and subtests ordered by tests"""

        out = "=" * len(self.title) + "\n"
        out += self.title + "\n"
        out += "=" * len(self.title) + "\n\n"

        for test in sorted(self.doc.keys()):
            fname = self.doc[test]["File"]
            base_name = self.doc[test]["_base_name_"]

            name = re.sub(r'.*/', '', fname)
            name = re.sub(r'\.[ch]', '', name)
            name = base_name + name

            tmp_subtest = self.expand_subtest(fname, name, test, False)

            # Get subtests first, to avoid displaying tests with
            # all filtered subtests
            subtest_array = []
            for subtest in tmp_subtest:
                if self.__filter_subtest(self.doc[test], subtest, True):
                    continue
                subtest_array.append(subtest)

            if not subtest_array:
                continue

            out += len(name) * '=' + "\n"
            out += name + "\n"
            out += len(name) * '=' + "\n\n"

            for field in sorted(self.doc[test].keys()):
                if field == "subtest":
                    continue
                if field in self.internal_fields:
                    continue

                out += f":{field}: {self.doc[test][field]}\n"

            for subtest in subtest_array:

                out += "\n" + subtest["_summary_"] + "\n"
                out += len(subtest["_summary_"]) * '=' + "\n\n"

                for field in sorted(subtest.keys()):
                    if field in self.internal_fields:
                        continue

                    out += f":{field}:" + subtest[field] + "\n"

                out += "\n"

            out += "\n\n"

        if filename:
            with open(filename, "w", encoding='utf8') as handle:
                handle.write(out)
        elif not return_string:
            print(out)
        else:
            return out

    def get_spreadsheet(self, expand_fields = None):

        """
        Return a bidimentional array with the test contents.

        Its output is similar to a spreadsheet, so it can be used by a
        separate python file that would create a workbook's sheet.
        """

        if not expand_fields:
            expand_fields = {}

        expand_field_name = {}
        sheet = []
        row = 0
        sheet.append([])
        sheet[row].append('Test name')

        subtest_dict = self.expand_dictionary(True)

                # Identify the sort order for the fields
        fields_order = []
        fields = sorted(self.props.items(), key = _sort_per_level)
        for item in fields:
            if item[0] in expand_fields.keys():
                expand_field_name[item[0]] = set()
                continue
            fields_order.append(item[0])
            sheet[row].append(item[0])

        # Receives a flat subtest dictionary, with wildcards expanded
        subtest_dict = self.expand_dictionary(True)

        subtests = sorted(subtest_dict.items(),
                          key = lambda x: _sort_using_array(x, fields_order))

        for subtest, fields in subtests:
            row += 1
            sheet.append([])

            sheet[row].append(subtest)

            for field in fields_order:
                if field in fields:
                    sheet[row].append(fields[field])
                else:
                    sheet[row].append('')

            for field in expand_fields.keys():
                names = fields.get(field)
                if not names:
                    continue

                names = set(re.split(r",\s*", names))
                expand_field_name[field].update(names)

        # Add expanded fields, if any
        for item in sorted(expand_fields.keys(), key=str.lower):
            prefix = expand_fields[item]

            if item not in expand_field_name:
                continue

            for field in sorted(list(expand_field_name[item]), key=str.lower):
                row = 0
                sheet[row].append(prefix + field)

                for subtest, fields in subtests:
                    row += 1
                    value = ""
                    names = fields.get(item)
                    if names:
                        names = set(re.split(r",\s*", names))

                        if field in names:
                            value = "Yes"

                    sheet[row].append(value)

        return sheet

    def print_nested_rest(self, filename = None, return_string = False):  # pylint: disable=R1710

        """Print tests and subtests ordered by tests"""

        handler = None
        if filename:
            handler = open(filename, "w", encoding='utf8') # pylint: disable=R1732
            sys.stdout = handler

        out = "=" * len(self.title) + "\n"
        out += self.title + "\n"
        out += "=" * len(self.title) + "\n\n"

        # Identify the sort order for the fields
        fields_order = []
        fields = sorted(self.props.items(), key = _sort_per_level)
        for item in fields:
            fields_order.append(item[0])

        # Receives a flat subtest dictionary, with wildcards expanded
        subtest_dict = self.expand_dictionary(True)

        subtests = sorted(subtest_dict.items(),
                          key = lambda x: _sort_using_array(x, fields_order))

        # Use the level markers below
        level_markers='=-^_~:.`"*+#'

        # Print the data
        old_fields = [ '' ] * len(fields_order)

        for subtest, fields in subtests:
            # Check what level has different message
            marker = 0
            for cur_level in range(0, len(fields_order)):  # pylint: disable=C0200
                field = fields_order[cur_level]
                if "level" not in self.props[field]["_properties_"]:
                    continue
                if field in fields:
                    if old_fields[cur_level] != fields[field]:
                        break
                    marker += 1

            # print hierarchy
            for i in range(cur_level, len(fields_order)):
                if "level" not in self.props[fields_order[i]]["_properties_"]:
                    continue
                if fields_order[i] not in fields:
                    continue

                if marker >= len(level_markers):
                    sys.exit(f"Too many levels: {marker}, maximum limit is {len(level_markers):}")

                title_str = fields_order[i] + ": " + fields[fields_order[i]]

                out += title_str + "\n"
                out += level_markers[marker] * len(title_str) + "\n\n"
                marker += 1

            out += "\n``" + subtest + "``" + "\n\n"

            # print non-hierarchy fields
            for field in fields_order:
                if "level" in self.props[field]["_properties_"]:
                    continue

                if field in fields:
                    out += f":{field}: {fields[field]}" + "\n"

            # Store current values
            for i in range(cur_level, len(fields_order)):
                field = fields_order[i]
                if "level" not in self.props[field]["_properties_"]:
                    continue
                if field in fields:
                    old_fields[i] = fields[field]
                else:
                    old_fields[i] = ''

            out += "\n"

        if filename:
            with open(filename, "w", encoding='utf8') as handle:
                handle.write(out)
        elif not return_string:
            print(out)
        else:
            return out

    def print_json(self, out_fname):

        """Adds the contents of test/subtest documentation form a file"""

        # Receives a dictionary with tests->subtests with expanded subtests
        test_dict = self.expand_dictionary(False)

        with open(out_fname, "w", encoding='utf8') as write_file:
            json.dump(test_dict, write_file, indent = 4)

    #
    # Add filters
    #
    def add_filter(self, filter_field_expr):

        """ Add a filter criteria for output data """

        match = re.match(r"(.*)=~\s*(.*)", filter_field_expr)
        if not match:
            sys.exit(f"Filter field {filter_field_expr} is not at <field> =~ <regex> syntax")
        field = match.group(1).strip().lower()
        if field not in self.field_list:
            sys.exit(f"Field '{field}' is not defined")
        filter_field = self.field_list[field]
        regex = re.compile("{0}".format(match.group(2).strip()), re.I) # pylint: disable=C0209
        self.filters[filter_field] = regex

    #
    # Subtest list methods
    #

    def get_subtests(self, sort_field = None, expand = None, with_order = False):

        """Return an array with all subtests"""

        subtests = {}
        subtests[""] = []

        order = None

        if expand:
            expand = re.compile(expand)

        if sort_field:
            if sort_field.lower() not in self.field_list:
                sys.exit(f"Field '{sort_field}' is not defined")
            sort_field = self.field_list[sort_field.lower()]

            if with_order:
                if "_properties_" in self.props[sort_field]:
                    if "order" in self.props[sort_field]["_properties_"]:
                        order = self.props[sort_field]["_properties_"]["order"]

        subtest_array = []
        for test in sorted(self.doc.keys()):
            fname = self.doc[test]["File"]
            base_name = self.doc[test]["_base_name_"]

            test_name = re.sub(r'.*/', '', fname)
            test_name = re.sub(r'\.[ch]', '', test_name)
            test_name = base_name + test_name

            subtest_array += self.expand_subtest(fname, test_name, test, True)

            subtest_array.sort(key = lambda x : x.get('_summary_'))

            for subtest in subtest_array:
                if self.__filter_subtest(self.doc[test], subtest, True):
                    continue

                if sort_field:
                    if sort_field in subtest:
                        if expand:
                            test_list = expand.split(subtest[sort_field])

                            for test_elem in test_list:
                                if test_elem not in subtests:
                                    subtests[test_elem] = []
                                if order:
                                    subtests[test_elem].append((subtest["_summary_"], test_list))
                                else:
                                    subtests[test_elem].append(subtest["_summary_"])
                        else:
                            if subtest[sort_field] not in subtests:
                                subtests[subtest[sort_field]] = []
                                if order:
                                    subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]]))
                                else:
                                    subtests[subtest[sort_field]].append(subtest["_summary_"])
                    else:
                        if order:
                            subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]]))
                        else:
                            subtests[""].append(subtest["_summary_"])

                else:
                    if order:
                        subtests[test_elem].append((subtest["_summary_"], [subtest[sort_field]]))
                    else:
                        subtests[""].append(subtest["_summary_"])

        if order:
            for group, tests in subtests.items():
                prefix_tests = []
                suffix_tests = []
                middle_tests = []
                is_prefix = True
                for k in order:
                    if k == "__all__":
                        is_prefix = False
                        continue
                    for test in tests:
                        if k in test[1]:
                            if is_prefix:
                                prefix_tests.append(test[0])
                            else:
                                suffix_tests.append(test[0])
                for test in tests:
                    if test[0] not in prefix_tests and test[0] not in suffix_tests:
                        middle_tests.append(test[0])
                subtests[group] = prefix_tests + middle_tests + suffix_tests

        return subtests

    def get_testlist(self):

        """ Return a list of tests as reported by --list-subtests """
        tests = []
        no_binaries = self.get_not_compiled()
        for name in self.filenames:
            binary = re.sub(r"\.c$", "", name.split('/')[-1])
            if binary in no_binaries:
                continue

            testlist = binary + ".testlist"
            fname = os.path.join(self.igt_build_path, "tests", testlist)

            if not os.path.isfile(fname):
                print(f"{testlist}: Testlist file not found.")
                continue

            with open(fname, 'r', encoding='utf8') as handle:
                for line in handle:
                    tests.append(line.rstrip("\n"))

        return sorted(tests)

    def get_not_compiled(self):

        """ Return a list of tests which were not compiled """
        no_binaries = []
        for name in self.filenames:
            test_basename = re.sub(r"\.c$", "", name.split('/')[-1])
            fname = os.path.join(self.igt_build_path, "tests", test_basename)

            if not os.path.isfile(fname):
                no_binaries.append(test_basename)

        return sorted(no_binaries)

    #
    # Validation methods
    #
    def check_tests(self):

        """Compare documented subtests with the IGT test list"""

        if not self.igt_build_path:
            sys.exit("Need the IGT build path")

        if self.filters:
            print("NOTE: test checks are affected by filters")

        mandatory_fields = set()
        for field, item in self.props.items():
            if item["_properties_"].get("mandatory"):
                mandatory_fields.add(field)

        doc_subtests = set()

        args_regex = re.compile(r'\<[^\>]+\>')

        subtests = self.expand_dictionary(True)
        for subtest, data in sorted(subtests.items()):
            subtest = self.subtest_separator.join(subtest.split(self.subtest_separator)[:3])
            subtest = args_regex.sub(r'\\d+', subtest)

            for field in mandatory_fields:
                if field not in data:
                    print(f"Warning: {subtest} {field} documentation is missing")

            doc_subtests.add(subtest)

        # Get a list of tests from
        run_subtests = set(self.get_testlist())

        not_compiled = set(self.get_not_compiled())

        # Compare sets

        run_missing = list(sorted(run_subtests - doc_subtests))
        doc_uneeded = list(sorted(doc_subtests - run_subtests))
        doc_uneeded_build = [t for t in doc_uneeded if t.split('@')[1] not in not_compiled]

        if doc_uneeded_build or run_missing:
            for test_name in run_missing:
                print(f'ERROR: Missing documentation for {test_name}')

            for test_name in doc_uneeded_build:
                print(f'ERROR: Unneeded documentation for {test_name}')

            print('Please refer: docs/test_documentation.md for more details')
            sys.exit(1)

    #
    # File handling methods
    #

    def __add_file_documentation(self, fname, base_name, implemented_class,
                                 field_re, test_tag, subtest_tag, config_origin):

        """Adds the contents of test/subtest documentation form a file"""

        current_test = None
        current_subtest = None

        handle_section = ''
        current_field = ''
        arg_number = 1
        cur_arg = -1
        cur_arg_element = 0
        has_test_or_subtest = 0

        test_regex = re.compile(test_tag + r':\s*(.*)')
        subtest_regex = re.compile('^' + subtest_tag + r':\s*(.*)')

        with open(fname, 'r', encoding='utf8') as handle:
            arg_ref = None
            current_test = ''
            subtest_number = 0

            for file_ln, file_line in enumerate(handle):
                file_line = file_line.rstrip()

                if re.match(r'^\s*\*$', file_line):
                    continue

                if re.match(r'^\s*\*/$', file_line):
                    handle_section = ''
                    current_subtest = None
                    arg_ref = None
                    cur_arg = -1
                    has_test_or_subtest = 0

                    continue

                if re.match(r'^\s*/\*\*$', file_line):
                    handle_section = '1'
                    continue

                if not handle_section:
                    continue

                file_line = re.sub(r'^\s*\* ?', '', file_line)

                # Handle only known sections
                if handle_section == '1':
                    current_field = ''

                    # Check if it is a new TEST section
                    match = test_regex.match(file_line)
                    if match:
                        has_test_or_subtest = 1
                        current_test = self.test_number
                        self.test_number += 1

                        handle_section = 'test'

                        self.doc[current_test] = {}
                        self.doc[current_test]["_arg_"] = {}
                        self.doc[current_test]["_summary_"] = match.group(1)
                        self.doc[current_test]["_base_name_"] = base_name
                        self.doc[current_test]["File"] = fname
                        self.doc[current_test]["subtest"] = {}
                        self.doc[current_test]["_subtest_line_"] = {}

                        if implemented_class:
                            self.doc[current_test]["Class"] = implemented_class
                        current_subtest = None

                        continue

                # Check if it is a new SUBTEST section
                match = subtest_regex.match(file_line)
                if match:
                    has_test_or_subtest = 1
                    current_subtest = subtest_number
                    subtest_number += 1

                    current_field = ''
                    handle_section = 'subtest'

                    # subtests inherit properties from the tests
                    self.doc[current_test]["subtest"][current_subtest] = {}
                    for field in self.doc[current_test].keys():
                        if field in self.internal_fields:
                            continue
                        if field == "File":
                            continue
                        if field == "subtest":
                            continue
                        if field == "_properties_":
                            continue
                        if field == "Description":
                            continue
                        self.doc[current_test]["subtest"][current_subtest][field] = self.doc[current_test][field]

                    self.doc[current_test]["subtest"][current_subtest]["_summary_"] = match.group(1)
                    self.doc[current_test]["_subtest_line_"][current_subtest] = file_ln

                    if not arg_ref:
                        arg_ref = arg_number
                        arg_number += 1
                        self.doc[current_test]["_arg_"][arg_ref] = {}

                    self.doc[current_test]["subtest"][current_subtest]["_arg_"] = arg_ref

                    continue

                # It is a known section. Parse its contents
                match = field_re.match(file_line)
                if match:
                    current_field = self.field_list[match.group(1).lower()]
                    match_val = match.group(2)

                    if handle_section == 'test':
                        self.doc[current_test][current_field] = match_val
                    else:
                        self.doc[current_test]["subtest"][current_subtest][current_field] = match_val

                    cur_arg = -1

                    continue

                if not has_test_or_subtest:
                    continue

                # Use hashes for arguments to avoid duplication
                match = re.match(r'arg\[(\d+)\]:\s*(.*)', file_line)
                if match:
                    current_field = ''
                    if arg_ref is None:
                        sys.exit(f"{fname}:{file_ln + 1}: arguments should be defined after one or more subtests, at the same comment")

                    cur_arg = int(match.group(1)) - 1
                    if cur_arg not in self.doc[current_test]["_arg_"][arg_ref]:
                        self.doc[current_test]["_arg_"][arg_ref][cur_arg] = {}

                    cur_arg_element = match.group(2)

                    if match.group(2):
                        # Should be used only for numeric values
                        self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] = "<" + match.group(2) + ">"

                    continue

                match = re.match(r'\@(\S+):\s*(.*)', file_line)
                if match:
                    if cur_arg >= 0:
                        current_field = ''
                        if arg_ref is None:
                            sys.exit(f"{fname}:{file_ln + 1}: arguments should be defined after one or more subtests, at the same comment")

                        cur_arg_element = match.group(1)
                        self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] = match.group(2)

                    else:
                        print(f"{fname}:{file_ln + 1}: Warning: invalid argument: @%s: %s" %
                            (match.group(1), match.group(2)))

                    continue

                # We don't need a multi-lined version here
                match = re.match(r'arg\[(\d+)\]\.values:\s*(.*)', file_line)
                if match:
                    cur_arg = int(match.group(1)) - 1

                    if cur_arg not in self.doc[current_test]["_arg_"][arg_ref]:
                        self.doc[current_test]["_arg_"][arg_ref][cur_arg] = {}

                    values = match.group(2).replace(" ", "").split(",")
                    for split_val in values:
                        if split_val == "":
                            continue
                        self.doc[current_test]["_arg_"][arg_ref][cur_arg][split_val] = split_val

                    continue

                # Handle multi-line field contents
                if current_field:
                    match = re.match(r'\s+(.*)', file_line)
                    if match:
                        if handle_section == 'test':
                            dic = self.doc[current_test]
                        else:
                            dic = self.doc[current_test]["subtest"][current_subtest]

                        if dic[current_field] != '':
                            dic[current_field] += " "
                        dic[current_field] += match.group(1)
                        continue

                # Handle multi-line argument contents
                if cur_arg >= 0 and arg_ref is not None:
                    match = re.match(r'\s+\*?\s*(.*)', file_line)
                    if match:
                        match_val = match.group(1)

                        match = re.match(r'^(\<.*)\>$',self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element])
                        if match:
                            self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] = match.group(1) + ' ' + match_val + ">"
                        else:
                            if self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] != '':
                                self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] += ' '
                            self.doc[current_test]["_arg_"][arg_ref][cur_arg][cur_arg_element] += match_val
                    continue

                file_line.rstrip(r"\n")
                sys.exit(f"{fname}:{file_ln + 1}: Error: unrecognized line. Need to add field at %s?\n\t==> %s" %
                         (config_origin, file_line))

    def show_subtests(self, sort_field):

        """Show subtests, allowing sort and filter a field """

        if sort_field:
            test_subtests = self.get_subtests(sort_field)
            for val_key in sorted(test_subtests.keys()):
                if not test_subtests[val_key]:
                    continue
                if val_key == "":
                    print("not defined:")
                else:
                    print(f"{val_key}:")
                    for sub in test_subtests[val_key]:
                        print (f"  {sub}")
        else:
            for sub in self.get_subtests(sort_field)[""]:
                print (sub)

    def gen_testlist(self, directory, sort_field):

        """Generate testlists from the test documentation"""


        # NOTE: currently, it uses a comma for multi-value delimitter
        test_subtests = self.get_subtests(sort_field, r",\s*", with_order = True)

        if not os.path.exists(directory):
            os.makedirs(directory)

        for test in test_subtests.keys():  # pylint: disable=C0201,C0206
            if not test_subtests[test]:
                continue

            testlist = test.lower()
            if testlist == "":
                fname = "other"
            elif testlist == "bat":         # Basic Acceptance Test
                fname = "fast-feedback"
            else:
                fname = testlist

            fname = re.sub(r"[^\w\d]+", "-", fname)
            fname = directory + "/" + fname + ".testlist"

            with open(fname, 'w', encoding='utf8') as handler:
                for sub in test_subtests[test]:
                    handler.write (f"{sub}\n")
                print(f"{fname} created.")