#!/usr/bin/env python # # Copyright (c), 2016-2020, SISSA (International School for Advanced Studies). # All rights reserved. # This file is distributed under the terms of the MIT License. # See the file 'LICENSE' in the root directory of the present # distribution, or http://opensource.org/licenses/MIT. # # @author Davide Brunato # """ This script runs tests concerning the W3C XML Schema 1.1 test suite. """ import unittest import argparse import os.path import warnings from pathlib import Path from typing import Optional from xml.etree import ElementTree try: import lxml.etree as lxml_etree except ImportError: lxml_etree = None from xmlschema import validate, XMLSchema10, XMLSchema11, \ XMLSchemaException, XMLSchemaValidationError TEST_SUITE_NAMESPACE = "http://www.w3.org/XML/2004/xml-schema-test-suite/" XLINK_NAMESPACE = "http://www.w3.org/1999/xlink" XSD_VERSION_VALUES = {'1.0 1.1', '1.0', '1.1'} ADMITTED_VALIDITY = {'valid', 'invalid', 'indeterminate'} #### # Tests that are incompatible with XSD meta-schema validation or that are postponed SKIPPED_TESTS = { ## # Skip for typos in test data '../msData/regex/reZ003v.xml', # 13589 '../ibmData/instance_invalid/S3_10_6/s3_10_6ii01.xsd', # 14911 '../ibmData/instance_invalid/S3_10_6/s3_10_6ii02.xsd', # 14912 '../ibmData/instance_invalid/S3_10_6/s3_10_6ii04.xsd', # 14914 '../ibmData/instance_invalid/S3_10_1/s3_10_1ii08.xsd', # 15360 '../ibmData/instance_invalid/S3_10_1/s3_10_1ii09.xsd', # 15361 ## # Invalid schemas marked as valid '../msData/additional/addB194.xsd', # invalid xml:lang='enu' is a typo? '../msData/particles/particlesZ001.xsd', # Invalid XSD 1.0 schema (valid with XSD 1.1) '../msData/simpleType/stE110.xsd', # Circular xs:union declaration '../saxonData/Missing/missing001.xsd', # missing type (this may be valid in 'lax' mode?) '../saxonData/Missing/missing002.xsd', # missing substitution group '../saxonData/Missing/missing003.xsd', # missing type and substitution group '../saxonData/Missing/missing006.xsd', # missing list item type '../msData/annotations/annotF001.xsd', # xml.xsd allows xml:lang="" for un-declarations ## # XSD 1.0 limited URI (see RFC 2396 + RFC 2732) '../msData/datatypes/Facets/anyURI/anyURI_a001.xsd', '../msData/datatypes/Facets/anyURI/anyURI_a003.xsd', '../msData/datatypes/Facets/anyURI/anyURI_b004.xsd', '../msData/datatypes/Facets/anyURI/anyURI_b006.xsd', ## # Uncertain cases (disputed tests) '../msData/group/groupH021.xsd', # 8679: Unclear invalidity stated for XSD 1.0 ... # # Two uncertain equivalent cases related with element # substitution and its equivalence with a choice group. # Xerces says these are both invalid with XSD 1.0 and valid with XSD 1.1. # # http://www.w3.org/Bugs/Public/show_bug.cgi?id=4146 # http://www.w3.org/Bugs/Public/show_bug.cgi?id=4147 # '../msData/element/elemZ026.xsd', # 8541: bug id=4147 '../msData/particles/particlesV020.xsd', # 10942: bug id=4147 ## # 7295: Inapplicable test on URI (the first resource is not reachable anymore) # https://www.w3.org/Bugs/Public/show_bug.cgi?id=4126 '../msData/datatypes/Facets/anyURI/anyURI_a004.xml', ## # Signed ad invalid, but valid because it depends on implementation choices or platform. # https://www.w3.org/Bugs/Public/show_bug.cgi?id=4133 '../msData/schema/schG3.xml', '../msData/schema/schG6_a.xsd', # Valid because the ns import is done once, validation fails. '../msData/schema/schG11_a.xsd', # Valid because the ns import is done once, validation fails. '../msData/schema/schG12.xml', '../msData/element/elemZ031.xsd', # Valid because Python has arbitrary large integers ## # Invalid XML tests '../sunData/combined/xsd005/xsd005.n05.xml', # 3984: Invalid if lxml is used (xsi:type and duplicate prefix) '../msData/additional/test93490_4.xml', # 4795: https://www.w3.org/Bugs/Public/show_bug.cgi?id=4078 '../msData/datatypes/gMonth002.xml', # 8017: gMonth bogus: conflicts with other invalid schema tests '../msData/datatypes/gMonth004.xml', # 8019: (http://www.w3.org/Bugs/Public/show_bug.cgi?id=6901) '../wgData/sg/e1.xml', # 14896: wrong href for valid instanceTest name="e1bis.xml" ## # Unicode version related '../msData/regex/reJ11.xml', '../msData/regex/reJ13.xml', '../msData/regex/reJ19.xml', '../msData/regex/reJ21.xml', '../msData/regex/reJ23.xml', '../msData/regex/reJ25.xml', '../msData/regex/reJ29.xml', '../msData/regex/reJ31.xml', '../msData/regex/reJ33.xml', '../msData/regex/reJ35.xml', '../msData/regex/reJ61.xml', '../msData/regex/reJ69.xml', '../msData/regex/reJ75.xml', '../msData/regex/reJ77.xml', '../msData/regex/reL98.xml', '../msData/regex/reL99.xml', '../msData/regex/reM98.xml', '../msData/regex/reN99.xml', '../msData/regex/reS21.xml', '../msData/regex/reS42.xml', '../msData/regex/reT63.xml', '../msData/regex/reT84.xml', # http://www.w3.org/Bugs/Public/show_bug.cgi?id=4113 '../msData/regex/reV16.xml', '../msData/regex/reV17.xml', '../msData/regex/reV18.xml', '../msData/regex/reV19.xml', '../msData/regex/reV20.xml', '../msData/regex/reV21.xml', '../msData/regex/reV22.xml', '../msData/regex/reV23.xml', '../msData/regex/reV24.xml', '../msData/regex/reV33.xml', '../msData/regex/reV34.xml', '../msData/regex/reV35.xml', '../msData/regex/reV36.xml', '../msData/regex/reV37.xml', '../msData/regex/reV38.xml', '../msData/regex/reV39.xml', '../msData/regex/reV40.xml', '../msData/regex/reV41.xml', '../msData/regex/reV42.xml', '../msData/regex/reV43.xml', # Tests with \W pattern and characters belonging to the M category ## # Skip for missing XML version 1.1 implementation '../saxonData/XmlVersions/xv001.v01.xml', # 14850 '../saxonData/XmlVersions/xv003.v01.xml', # 14852 '../saxonData/XmlVersions/xv004.xsd', # 14853 non-BMP chars allowed in names in XML 1.1+ '../saxonData/XmlVersions/xv005.v01.xml', # 14854 '../saxonData/XmlVersions/xv006.v01.xml', # 14855 invalid character  (valid in XML 1.1) '../saxonData/XmlVersions/xv006.n02.xml', # 14855 invalid character 𐀀 (valid in XML 1.1) '../saxonData/XmlVersions/xv007.v01.xml', # 14856 '../saxonData/XmlVersions/xv008.v01.xml', # 14857 '../saxonData/XmlVersions/xv008.n01.xml', '../saxonData/XmlVersions/xv009.v02.xml', # 14858 '../saxonData/XmlVersions/xv009.n02.xml', '../saxonData/XmlVersions/xv009.n03.xml', '../saxonData/XmlVersions/xv100.i.xml', # 14859 '../saxonData/XmlVersions/xv100.c.xml', # 14860 } XSD10_SKIPPED_TESTS = { # Invalid schemas marked as valid '../msData/simpleType/stE072.xsd', # 13868: a union derived from ID with a fixed value '../msData/simpleType/stE072.xml', } XSD11_SKIPPED_TESTS = { # Valid schemas marked ad invalid '../msData/regex/reK86.xsd', # \P{Is} is valid in regex for XSD 1.1 '../msData/regex/reK87.xsd', # \P{Is} is valid in regex for XSD 1.1 '../msData/particles/particlesZ033_g.xsd', # valid in XSD 1.1 (invalid for engine limitation) '../saxonData/CTA/cta0043.xsd', # Only a warning for type table difference on restriction ## # XSD 1.1 schema composition (dynamic schema load) # (See bullet 4 of G.1.15: https://www.w3.org/TR/xmlschema11-1/#ch_schemacomp) '../msData/additional/test93490_5.xml', # 4796 '../msData/additional/test93490_7.xml', # 4798 '../msData/additional/test93490_8.xml', # 4799 '../msData/additional/test93490_10.xml', # 4801 '../msData/additional/test93490_12.xml', # 4803 # TODO: Parse ENTITY declarations in DOCTYPE before enforce checking '../saxonData/Id/id017.n01.xml', # 14571-14575 '../saxonData/Id/id018.n01.xml', '../saxonData/Id/id018.n02.xml', '../saxonData/Id/id019.n01.xml', '../saxonData/Id/id019.n02.xml', '../saxonData/Id/id020.n01.xml', '../saxonData/Id/id020.n02.xml', '../saxonData/Id/id021.n01.xml', '../saxonData/Id/id021.n02.xml', } DO_NOT_USE_META_SCHEMA = { '../msData/additional/test264908_1.xsd', } DO_NOT_USE_FALLBACK_LOCATIONS = { '../msData/wildcards/wildZ001.xml', } # Total files counters total_xsd_files = 0 total_xml_files = 0 def fetch_xsd_test_suite(): parent = os.path.dirname xmlschema_test_dir = parent(os.path.abspath(__file__)) xmlschema_base_dir = parent(xmlschema_test_dir) suite_file = os.path.join(parent(xmlschema_base_dir), 'xsdtests/suite.xml') if os.path.isfile(suite_file): return suite_file else: raise FileNotFoundError("can't find the XSD suite index file suite.xml ...") def skip_message(source_href, group_num, version='each'): if source_href.endswith('.xsd'): msg = "Skip test number {} with schema {!r} for {} version ..." else: msg = "Skip test number {} with file {!r} for {} version ..." print(msg.format(group_num, source_href, version)) def create_w3c_test_group_case(args, filename, group_elem, group_num, xsd_version='1.0'): """ Creates a test class for a W3C test group. :param args: parsed command line arguments. :type args: argparse.Namespace. :param filename: the filename of the testSet that owns the testGroup. :param group_elem: the Element instance of the test group. :param group_num: a positive integer to distinguish and order test groups. :param xsd_version: if '1.1' uses XSD 1.1 validator class, otherwise uses the XSD 1.0 validator. """ def get_test_conf(elem): schema_test = elem.tag.endswith('schemaTest') if schema_test: tag = '{%s}schemaDocument' % TEST_SUITE_NAMESPACE else: tag = '{%s}instanceDocument' % TEST_SUITE_NAMESPACE try: source_href = elem.find(tag).get('{%s}href' % XLINK_NAMESPACE) except AttributeError: return else: if not schema_test and source_href.endswith('.testSet'): return if source_href in SKIPPED_TESTS: if args.numbers: skip_message(source_href, group_num) return # Normalize and check file path source_path = os.path.normpath(os.path.join(os.path.dirname(filename), source_href)) if not os.path.isfile(source_path): print("ERROR: file %r not found!" % source_path) return test_conf = {} for version in xsd_version.split(): if 'version' in elem.attrib and version not in elem.attrib['version']: continue elif version not in args.version: continue elif source_href in XSD10_SKIPPED_TESTS and version == '1.0' or \ source_href in XSD11_SKIPPED_TESTS and version == '1.1': if args.numbers: skip_message(source_href, group_num, version) continue for e in elem.findall('{%s}expected' % TEST_SUITE_NAMESPACE): if 'version' not in e.attrib: test_conf[version] = e.attrib['validity'] elif e.attrib['version'] == version or \ e.attrib['version'] == 'full-xpath-in-CTA': test_conf[version] = e.attrib['validity'] break if version not in test_conf: msg = "ERROR: Missing expected validity for XSD version %s in %r of test group %r" print(msg % (version, elem, name)) return elif test_conf[version] not in ADMITTED_VALIDITY: msg = "ERROR: Wrong validity=%r attribute for XSD version %s in %r test group %r" print(msg % (test_conf[version], version, elem, name)) return elif test_conf[version] not in args.expected: test_conf.pop(version) elif test_conf[version] == 'indeterminate': if args.verbose: print("WARNING: Skip indeterminate test group %r" % name) test_conf.pop(version) if test_conf: test_conf['source'] = source_path if schema_test: if not source_path.endswith('.xml'): test_conf['sources'] = [ os.path.normpath( os.path.join(os.path.dirname(filename), schema_href.get('{%s}href' % XLINK_NAMESPACE)) ) for schema_href in elem.findall(tag) ] if source_href in DO_NOT_USE_META_SCHEMA: nonlocal use_meta use_meta = False if source_href in DO_NOT_USE_FALLBACK_LOCATIONS: nonlocal use_fallback use_fallback = False return test_conf if group_num == 1: return # Skip introspection tests that have several failures due to schema mismatch. elif args.numbers and group_num not in args.numbers: return name = group_elem.attrib['name'] group_tests = [] global total_xsd_files global total_xml_files use_meta = True use_fallback = True # Get schema/instance path for k, child in enumerate(group_elem.iterfind('{%s}schemaTest' % TEST_SUITE_NAMESPACE)): if k: print("ERROR: multiple schemaTest definition in group %r" % name) return config = get_test_conf(child) if not config: return group_tests.append(config) total_xsd_files += 1 if args.xml: for child in group_elem.iterfind('{%s}instanceTest' % TEST_SUITE_NAMESPACE): if 'version' in child.attrib and child.attrib['version'] not in args.version: continue config = get_test_conf(child) if config: group_tests.append(config) total_xml_files += 1 if not group_tests: if len(args.expected) > 1 and args.xml: print("ERROR: Missing both schemaTest and instanceTest in test group %r" % name) return class TestGroupCase(unittest.TestCase): elem = group_elem @classmethod def get_add_info(cls, error: Optional[XMLSchemaException] = None, *files): if len(args.numbers) != 1: return str(error) if error is not None else '' if error is None: msg = '' elif isinstance(error, XMLSchemaValidationError) and error.stack_trace: msg = f"\n\n{error}\n{error.stack_trace}" else: msg = f"\n\n*** Error description ***\n\n{error}" ElementTree.register_namespace('', TEST_SUITE_NAMESPACE) case_info = ElementTree.tostring(cls.elem).decode('utf-8') msg = f"{msg}\n\n*** Case info ***\n\n{case_info}" for filepath in map(Path, files): with filepath.open() as fp: msg = f"{msg}\n\n*** {filepath.name} ***\n\n{fp.read()}" return msg @unittest.skipIf(group_tests[0]['source'].endswith('.xml'), 'No schema test') def test_xsd_schema(self): def create_schema_instance(): with warnings.catch_warnings(): warnings.simplefilter('ignore') return schema_class( source=item['sources'] or source, use_meta=use_meta, use_fallback=use_fallback, ) for item in filter(lambda x: x['source'].endswith('.xsd'), group_tests): source = item['source'] rel_path = os.path.relpath(source) for version, expected in \ sorted(filter(lambda x: not x[0].startswith('source'), item.items())): if args.lxml and version == '1.0': if expected == 'invalid': with self.assertRaises(lxml_etree.XMLSchemaParseError): schema_tree = lxml_etree.parse(source) lxml_etree.XMLSchema(schema_tree) else: schema_tree = lxml_etree.parse(source) lxml_etree.XMLSchema(schema_tree) continue schema_class = XMLSchema11 if version == '1.1' else XMLSchema10 if expected == 'invalid': message = f"schema {rel_path} should be invalid with XSD " \ f"{version} {self.get_add_info(None, rel_path)}" with self.assertRaises(XMLSchemaException, msg=message): create_schema_instance() else: try: schema = create_schema_instance() except XMLSchemaException as err: schema = None message = \ f"schema {rel_path} should be valid with XSD {version}" \ f", but an error is raised: {self.get_add_info(err, source)}" else: message = None self.assertIsInstance(schema, schema_class, msg=message) @unittest.skipIf( group_tests[0]['source'].endswith('.xsd') and len(group_tests) == 1, 'No instance tests' ) def test_xml_instances(self): if group_tests[0]['source'].endswith('.xsd'): schema = group_tests[0]['source'] schemas = group_tests[0]['sources'] else: schema = None schemas = [] def validate_xml_instance(): with warnings.catch_warnings(): warnings.simplefilter('ignore') if len(schemas) <= 1: validate(source, schema=schema, cls=schema_class, use_meta=use_meta, use_fallback=use_fallback) else: schema_instance = schema_class( source=schemas, use_meta=use_meta, use_fallback=use_fallback ) schema_instance.validate(source) for item in filter(lambda x: not x['source'].endswith('.xsd'), group_tests): source = item['source'] rel_path = os.path.relpath(source) for version, expected in sorted(filter(lambda x: x[0] != 'source', item.items())): schema_class = XMLSchema11 if version == '1.1' else XMLSchema10 if expected == 'invalid': message = f"instance {rel_path} should be invalid with XSD {version}:" \ f"{self.get_add_info(None, *schemas, rel_path)}" with self.assertRaises(XMLSchemaException, msg=message): validate_xml_instance() else: try: self.assertIsNone(validate_xml_instance()) except XMLSchemaException as err: error = f"instance {rel_path} should be valid with XSD " \ f"{version} but an {err.__class__.__name__} is raised" msg = self.get_add_info(err, *schemas, rel_path) self.assertIsNone(error, msg=msg) if not any(g['source'].endswith('.xsd') for g in group_tests): del TestGroupCase.test_xsd_schema if not any(g['source'].endswith('.xml') for g in group_tests): del TestGroupCase.test_xml_instances TestGroupCase.elem = group_elem TestGroupCase.__name__ = TestGroupCase.__qualname__ = str( 'TestGroupCase{:05}_{}'.format(group_num, name.replace('-', '_')) ) return TestGroupCase def w3c_tests_factory(argv=None): import sys if argv is None: argv = sys.argv[1:] def xsd_versions(value): if value not in ('1.0', '1.1', '1.0 1.1', '1.1 1.0'): raise argparse.ArgumentTypeError("%r is not an XSD version string" % value) return value def number_or_interval(value): try: return int(value) except ValueError: if '-' not in value: raise try: start, stop = value.split('-') return [int(start), int(stop)] except ValueError: pass raise def iter_numbers(numbers): for x in numbers: if isinstance(x, int): yield x elif isinstance(x, list) and len(x) == 2: yield from range(x[0], x[1] + 1) parser = argparse.ArgumentParser(prog=__name__, add_help=True) parser.add_argument('-v', '--verbose', default=False, action='store_true') parser.add_argument('--version', dest='version', type=xsd_versions, default='1.0 1.1', help="Run only tests related to a specific XSD version.") parser.add_argument('--xml', default=False, action='store_true', help="Include XML validation tests.") parser.add_argument('--lxml', default=False, action='store_true', help="Use lxml's XMLSchema validator for XSD 1.0 tests.") parser.add_argument('--valid', dest='expected', default=ADMITTED_VALIDITY, const='valid', action='store_const', help="Run only expected 'valid' tests.") parser.add_argument('--invalid', dest='expected', const='invalid', action='store_const', help="Run only expected 'invalid' tests.") parser.add_argument('--unknown', dest='expected', const='indeterminate', action='store_const', help="Run only expected 'indeterminate' tests.") parser.add_argument('numbers', metavar='TEST_NUMBER', type=number_or_interval, nargs='*', help='Runs only specific tests, selected by numbers.') args = parser.parse_args(args=argv) # Clean argv of xmlschema-only arguments sys.argv[:] = [__name__, '-v'] if '-v' in sys.argv else [__name__] args.numbers = [x for x in iter_numbers(args.numbers)] if lxml_etree is None and args.lxml: print("Ignore --lxml option: library lxml is not available ...") args.lxml = False quiet = __name__ == '__main__' index_path = fetch_xsd_test_suite() index_dir = os.path.dirname(index_path) suite_xml = ElementTree.parse(index_path) test_classes = {} testgroup_num = 0 if args.verbose and not quiet: print("\n>>>>> ADD TEST GROUPS FROM TESTSET FILES <<<<<\n") for testset_elem in suite_xml.iter("{%s}testSetRef" % TEST_SUITE_NAMESPACE): href_attr = testset_elem.attrib.get("{%s}href" % XLINK_NAMESPACE, '') testset_file = os.path.join(index_dir, href_attr) testset_groups = 0 testset = ElementTree.parse(testset_file) testset_version = testset.getroot().get('version', '1.0 1.1') if testset_version not in XSD_VERSION_VALUES: if not quiet: msg = "Testset file %r has an invalid version=%r, skip ..." print(msg % (href_attr, testset_version)) continue for testgroup_elem in testset.iter("{%s}testGroup" % TEST_SUITE_NAMESPACE): testgroup_num += 1 testgroup_version = testgroup_elem.get('version', testset_version) if testgroup_version == 'full-xpath-in-CTA': # skip full XPath test for the moment ... if args.verbose and not quiet: print("Skip full XPath test %r ..." % testgroup_elem.get('name')) continue elif testgroup_version not in XSD_VERSION_VALUES: if not quiet: _msg = "Test group %r has an invalid version=%r, skip ..." print(_msg % (testgroup_elem.get('name'), testgroup_version)) continue elif testgroup_version not in testset_version: if args.verbose and not quiet: _msg = "Warning: Test group %r version=%r is not included " \ "in test set version=%r" print(_msg % (testgroup_elem.get('name'), testgroup_version, testset_version)) cls = create_w3c_test_group_case( args=args, filename=testset_file, group_elem=testgroup_elem, group_num=testgroup_num, xsd_version=testgroup_version, ) if cls is not None: test_classes[cls.__name__] = cls testset_groups += 1 if args.verbose and testset_groups and not quiet: print(f"Added {testset_groups} test groups from {href_attr}") if test_classes and not quiet: print("\n+++ Number of classes under test: %d +++" % len(test_classes)) if total_xml_files: print("+++ Number of XSD schemas under test: %d +++" % total_xsd_files) print("+++ Number of XML files under test: %d +++" % total_xml_files) print() if args.verbose and not quiet: print("\n>>>>> RUN TEST GROUPS <<<<<\n") return test_classes if __name__ == '__main__': from xmlschema.testing import run_xmlschema_tests globals().update(w3c_tests_factory()) run_xmlschema_tests("running W3C XSD 1.0/1.1 tests")