Origin: upstream, 7b9c4e8412eac6fbda24092ad0d3577e9437dfe3 From: Rob Herring Date: Fri, 14 Apr 2023 14:01:26 -0500 Subject: Switch to jsonschema 4.18+ We've been stuck on jsonschema 4.17 because 4.18 changed in non-compatible ways with how RefResolver worked. Upstream was not willing to fix the issues claiming our usage was incorrect. The RefResolver interface is deprecated anyways so it isn't really worth the effort to fix or find a work-around when we need to move to the Registry interface anyways. In 4.17, the Draft2019-09 meta-schema was (silently) broken due to '$recursiveRef' not being handled, so we were stuck using Draft7 validator in some places. Now those are all updated to Draft2019-09. Setting '$schema' to our meta-schema seems to not be how extending the meta-schema is supposed to be done. '$schema' is supposed to be which dialect of json-schema we are using (Draft 2019-09) rather than our extended meta-schema. Going against this seems to cause problems with referencing the core schema within our schema, so that has been dropped in favor of validating against both meta-schemas separately. Further reworking of how our meta-schema is structured is needed. 4.18+ has been out for some time now and is what distros are packaging. Also, 4.17 seems to be broken with python 3.14 and a fix by upstream seems unlikely. So now is the time to switch. Signed-off-by: Rob Herring --- dtschema/meta-schemas/base.yaml | 12 --- dtschema/schema.py | 139 ++++++++++++++++++-------------- dtschema/validator.py | 50 +++++------- pyproject.toml | 2 +- test/schemas/bad-example.yaml | 2 +- test/test-dt-validate.py | 4 +- 6 files changed, 103 insertions(+), 106 deletions(-) diff --git a/dtschema/meta-schemas/base.yaml b/dtschema/meta-schemas/base.yaml index 44f869f4..6a5de383 100644 --- a/dtschema/meta-schemas/base.yaml +++ b/dtschema/meta-schemas/base.yaml @@ -8,7 +8,6 @@ $schema: https://json-schema.org/draft/2019-09/schema description: Metaschema for devicetree binding documentation allOf: - - $ref: http://json-schema.org/draft-07/schema# - $ref: http://devicetree.org/meta-schemas/keywords.yaml# properties: @@ -43,17 +42,6 @@ properties: type: string format: email - select: - description: '"select" must contain a valid json-schema' - allOf: - - $ref: http://json-schema.org/draft-07/schema# - - oneOf: - - type: object - properties: - properties: true - required: true - - type: boolean - propertyNames: enum: [ $id, $schema, title, description, examples, required, allOf, anyOf, oneOf, definitions, $defs, additionalProperties, dependencies, dependentRequired, diff --git a/dtschema/schema.py b/dtschema/schema.py index 85f63d57..22c045b8 100644 --- a/dtschema/schema.py +++ b/dtschema/schema.py @@ -6,11 +6,16 @@ import os import re import copy + +from functools import cache + import jsonschema -import dtschema +from referencing import Registry +from referencing.exceptions import NoSuchResource, Unresolvable +from referencing.jsonschema import DRAFT201909 -from jsonschema.exceptions import RefResolutionError +import dtschema schema_base_url = "http://devicetree.org/" schema_basedir = os.path.dirname(os.path.abspath(__file__)) @@ -55,6 +60,26 @@ def _schema_allows_no_undefined_props(schema): return not additional_props or isinstance(additional_props, dict) or \ not uneval_props or isinstance(uneval_props, dict) + +@cache +def create_meta_schema_registry(): + import glob + import ruamel.yaml + + yaml = ruamel.yaml.YAML(typ='safe') + yaml.allow_duplicate_keys = False + + metaschemas = [] + for filename in glob.iglob(schema_basedir + "/meta-schemas/*.yaml"): + with open(filename, 'r', encoding='utf-8') as f: + metaschema = yaml.load(f.read()) + metaschemas += [( + metaschema['$id'], + DRAFT201909.create_resource(metaschema) + )] + return Registry().with_resources(metaschemas) + + class DTSchema(dict): DtValidator = jsonschema.validators.extend( jsonschema.Draft201909Validator, @@ -78,24 +103,13 @@ def __init__(self, schema_file, line_numbers=False): schema = yaml.load(f.read()) self.filename = os.path.abspath(schema_file) - self._validator = None - id = schema['$id'].rstrip('#') - match = re.search('(.*/schemas/)(.+)$', id) - self.base_path = os.path.abspath(schema_file)[:-len(match[2])] + self.DtValidator.META_SCHEMA['properties']['select'] = {'$recursiveRef': '#'} - super().__init__(schema) - - def validator(self): - if not self._validator: - resolver = jsonschema.RefResolver.from_schema(self, - handlers={'http': self.http_handler}) - meta_schema = resolver.resolve_from_url(self['$schema']) - self._validator = self.DtValidator(meta_schema, resolver=resolver) - return self._validator + super().__init__(schema) - def http_handler(self, uri): + def retrieve(self, uri): '''Custom handler for http://devicetree.org references''' uri = uri.rstrip('#') missing_files = '' @@ -108,30 +122,33 @@ def http_handler(self, uri): import ruamel.yaml yaml = ruamel.yaml.YAML(typ='safe') yaml.allow_duplicate_keys = False - return yaml.load(f.read()) - raise RefResolutionError(f'Error in referenced schema matching $id: {uri}\n\tTried these paths (check schema $id if path is wrong):\n{missing_files}') + return DRAFT201909.create_resource(yaml.load(f.read())) - def annotate_error(self, error, schema, path): + raise NoSuchResource(ref=uri) + + def annotate_error(self, error, uri, path): error.note = None - error.schema_file = None for e in error.context: - self.annotate_error(e, schema, path + e.schema_path) + self.annotate_error(e, uri, path + e.schema_path) - scope = self.validator().ID_OF(schema) - self.validator().resolver.push_scope(scope) - ref_depth = 1 + registry = create_meta_schema_registry() + schema = registry[uri].contents + error.schema_file = uri for p in path: while p not in schema and '$ref' in schema and isinstance(schema['$ref'], str): - ref = self.validator().resolver.resolve(schema['$ref']) - schema = ref[1] - self.validator().resolver.push_scope(ref[0]) - ref_depth += 1 - - if '$id' in schema and isinstance(schema['$id'], str): - error.schema_file = schema['$id'] + ref = schema['$ref'] + if ref.startswith('#'): + schema = registry.resolver(uri).lookup(ref).contents + else: + for rsrc in registry: + if ref.split('#')[0] in rsrc: + uri = rsrc + break + schema = registry.resolver(uri).lookup(ref).contents + error.schema_file = uri schema = schema[p] @@ -139,40 +156,38 @@ def annotate_error(self, error, schema, path): if 'description' in schema and isinstance(schema['description'], str): error.note = schema['description'] - while ref_depth > 0: - self.validator().resolver.pop_scope() - ref_depth -= 1 - if isinstance(error.schema, dict) and 'description' in error.schema: error.note = error.schema['description'] - def iter_errors(self): - meta_schema = self.validator().resolver.resolve_from_url(self['$schema']) + def iter_errors(self, strict=True): + if strict: + registry = create_meta_schema_registry() + + meta_schema_id = self['$schema'].rstrip('#') + self.validator = self.DtValidator(registry.contents(meta_schema_id), registry=registry) - for error in self.validator().iter_errors(self): + for error in self.validator.iter_errors(self): + scherr = jsonschema.exceptions.SchemaError.create_from(error) + self.annotate_error(scherr, meta_schema_id, scherr.schema_path) + scherr.linecol = get_line_col(self, scherr.path) + yield scherr + + for error in self.DtValidator(self.DtValidator.META_SCHEMA).iter_errors(self): scherr = jsonschema.exceptions.SchemaError.create_from(error) - self.annotate_error(scherr, meta_schema, scherr.schema_path) scherr.linecol = get_line_col(self, scherr.path) yield scherr def is_valid(self, strict=False): ''' Check if schema passes validation against json-schema.org schema ''' - if strict: - for error in self.iter_errors(): - raise error - else: - # Using the draft7 metaschema because 2019-09 with $recursiveRef seems broken - # Probably fixed with referencing library - for error in self.DtValidator(jsonschema.Draft7Validator.META_SCHEMA).iter_errors(self): - scherr = jsonschema.exceptions.SchemaError.create_from(error) - raise scherr + for error in self.iter_errors(strict): + raise error def fixup(self): processed_schema = copy.deepcopy(dict(self)) dtschema.fixups.fixup_schema(processed_schema) return processed_schema - def _check_schema_refs(self, schema, parent=None, is_common=False, has_constraint=False): + def _check_schema_refs(self, resolver, schema, parent=None, is_common=False, has_constraint=False): if not parent: is_common = not _schema_allows_no_undefined_props(schema) if isinstance(schema, dict): @@ -185,8 +200,7 @@ def _check_schema_refs(self, schema, parent=None, is_common=False, has_constrain ref_has_constraint = True if '$ref' in schema: - ref = schema['$ref'] - url, ref_sch = self.validator().resolver.resolve(ref) + ref_sch = resolver.lookup(schema['$ref']).contents ref_has_constraint = _schema_allows_no_undefined_props(ref_sch) if not (is_common or ref_has_constraint or has_constraint or @@ -195,17 +209,18 @@ def _check_schema_refs(self, schema, parent=None, is_common=False, has_constrain file=sys.stderr) for k, v in schema.items(): - self._check_schema_refs(v, parent=k, is_common=is_common, + self._check_schema_refs(resolver, v, parent=k, is_common=is_common, has_constraint=has_constraint) elif isinstance(schema, (list, tuple)): for i in range(len(schema)): - self._check_schema_refs(schema[i], parent=parent, is_common=is_common, + self._check_schema_refs(resolver, schema[i], parent=parent, is_common=is_common, has_constraint=has_constraint) def check_schema_refs(self): id = self['$id'].rstrip('#') base1 = re.search('schemas/(.+)$', id)[1] - base2 = self.filename.replace(self.filename[:-len(base1)], '') + base_path = self.filename[:-len(base1)] + base2 = self.filename.replace(base_path, '') if not base1 == base2: print(f"{self.filename}: $id: Cannot determine base path from $id, relative path/filename doesn't match actual path or filename\n", f"\t $id: {id}\n", @@ -213,16 +228,16 @@ def check_schema_refs(self): file=sys.stderr) return - scope = self.validator().ID_OF(self) - if scope: - self.validator().resolver.push_scope(scope) - self.paths = [ - (schema_base_url + 'schemas/', self.base_path), + (schema_base_url + 'schemas/', base_path), (schema_base_url + 'schemas/', schema_basedir + '/schemas/'), ] + rsrc = DRAFT201909.create_resource(self) + registry = Registry(retrieve=self.retrieve).with_resource(uri=id, resource=rsrc) + resolver = registry.resolver_with_root(rsrc) + try: - self._check_schema_refs(self) - except jsonschema.RefResolutionError as exc: - print(f"{self.filename}:\n\t{exc}", file=sys.stderr) + self._check_schema_refs(resolver, self) + except Unresolvable as exc: + print(f"{self.filename}: Unresolvable reference: {exc}", file=sys.stderr) diff --git a/dtschema/validator.py b/dtschema/validator.py index 2d1ccc65..50fe07dd 100644 --- a/dtschema/validator.py +++ b/dtschema/validator.py @@ -10,7 +10,8 @@ import json import jsonschema -from jsonschema.exceptions import RefResolutionError +from referencing import Registry +from referencing.jsonschema import DRAFT201909 import dtschema from dtschema.lib import _is_string_schema @@ -297,11 +298,12 @@ def _add_schema(schemas, filename): sch = process_schema(os.path.abspath(filename)) if not sch or '$id' not in sch: return False - if sch['$id'] in schemas: - print(f"{sch['$filename']}: warning: ignoring duplicate '$id' value '{sch['$id']}'", file=sys.stderr) + _id = sch['$id'].rstrip('#') + if _id in schemas: + print(f"{sch['$filename']}: warning: ignoring duplicate '$id' value '{_id}'", file=sys.stderr) return False else: - schemas[sch['$id']] = sch + schemas[_id] = sch return True @@ -342,6 +344,11 @@ def typeSize(validator, typeSize, instance, schema): yield jsonschema.ValidationError("size is %r, expected %r" % (size, typeSize)) +def _get_schema_by_path(schema, path): + if len(path)==1: return schema[path[0]] + return _get_schema_by_path(schema[path[0]],path[1:]) + + class DTValidator: '''Custom Validator for Devicetree Schemas @@ -352,9 +359,9 @@ class DTValidator: ''' DtValidator = jsonschema.validators.extend(jsonschema.Draft201909Validator, {'typeSize': typeSize}) - def __init__(self, schema_files, filter=None): + def __init__(self, schema_files, id_filter=""): self.schemas = {} - self.resolver = jsonschema.RefResolver('', None, handlers={'http': self.http_handler}) + self.registry = Registry(retrieve=self.retrieve) schema_cache = None if len(schema_files) == 1 and os.path.isfile(schema_files[0]): @@ -399,9 +406,10 @@ def __init__(self, schema_files, filter=None): self.always_schemas = [] self.compat_map = {} for sch in self.schemas.values(): + _id = sch['$id'].rstrip('#') if 'select' in sch: if sch['select'] is not False: - self.always_schemas += [sch['$id']] + self.always_schemas += [_id] elif 'properties' in sch and 'compatible' in sch['properties']: compatibles = dtschema.extract_node_compatibles(sch['properties']['compatible']) if len(compatibles) > 1: @@ -410,22 +418,12 @@ def __init__(self, schema_files, filter=None): if not schema_cache and c in self.compat_map: print(f'Warning: Duplicate compatible "{c}" found in schemas matching "$id":\n' f'\t{self.compat_map[c]}\n\t{sch["$id"]}', file=sys.stderr) - self.compat_map[c] = sch['$id'] + self.compat_map[c] = _id self.schemas['version'] = dtschema.__version__ - def http_handler(self, uri): - '''Custom handler for http://devicetree.org references''' - try: - uri += '#' - if uri in self.schemas: - return self.schemas[uri] - else: - # If we have a schema_cache, then the schema should have been there unless the schema had errors - if len(self.schemas): - return {'not': {'description': f"Can't find referenced schema: {uri}"}} - except: - raise RefResolutionError('Error in referenced schema matching $id: ' + uri) + def retrieve(self, uri): + return DRAFT201909.create_resource(self.schemas[uri]) def annotate_error(self, id, error): error.schema_file = id @@ -447,10 +445,8 @@ def iter_errors(self, instance, filter=None, compatible_match=False): schema_id = self.compat_map[inst_compat] if self._filter_match(schema_id, filter): schema = self.schemas[schema_id] - for error in self.DtValidator(schema, - resolver=self.resolver, - ).iter_errors(instance): - self.annotate_error(schema['$id'], error) + for error in self.DtValidator(schema, registry=self.registry).iter_errors(instance): + self.annotate_error(schema_id, error) yield error break @@ -462,9 +458,7 @@ def iter_errors(self, instance, filter=None, compatible_match=False): continue schema = {'if': self.schemas[schema_id]['select'], 'then': self.schemas[schema_id]} - for error in self.DtValidator(schema, - resolver=self.resolver, - ).iter_errors(instance): + for error in self.DtValidator(schema, registry=self.registry).iter_errors(instance): self.annotate_error(schema_id, error) yield error @@ -486,7 +480,7 @@ def check_missing_property_types(self): for p, val in self.props.items(): if val[0]['type'] is None: for id in val[0]['$id']: - print(f"{self.schemas[id]['$filename']}: {p}: missing type definition", file=sys.stderr) + print(f"{self.schemas[id.rstrip('#')]['$filename']}: {p}: missing type definition", file=sys.stderr) def check_duplicate_property_types(self): """ diff --git a/pyproject.toml b/pyproject.toml index 0f4b4d04..0a3c6ddf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dynamic = ["version"] dependencies = [ "ruamel.yaml>0.15.69", - "jsonschema>=4.1.2,<4.18", + "jsonschema>=4.18", "rfc3987", "pylibfdt", ] diff --git a/test/schemas/bad-example.yaml b/test/schemas/bad-example.yaml index a4131a1e..fcece00e 100644 --- a/test/schemas/bad-example.yaml +++ b/test/schemas/bad-example.yaml @@ -4,7 +4,7 @@ %YAML 1.2 --- $id: http://devicetree.org/schemas/bad-example.yaml -$schema: http://json-schema.org/draft-07/schema# +$schema: http://devicetree.org/meta-schemas/core.yaml# title: 0 diff --git a/test/test-dt-validate.py b/test/test-dt-validate.py index 5deaae6d..19a5eb15 100755 --- a/test/test-dt-validate.py +++ b/test/test-dt-validate.py @@ -97,7 +97,7 @@ def test_binding_schemas_id_is_unique(self): self.assertEqual(ids.count(schema['$id']), 0) ids.append(schema['$id']) - def test_binding_schemas_valid_draft7(self): + def test_binding_schemas_valid_draft201909(self): '''Test that all schema files under ./dtschema/schemas/ validate against the Draft7 metaschema The DT Metaschema is supposed to force all schemas to be valid against Draft7. This test makes absolutely sure that they are. @@ -105,7 +105,7 @@ def test_binding_schemas_valid_draft7(self): for filename in glob.iglob(os.path.join(dtschema_dir, 'schemas/**/*.yaml'), recursive=True): with self.subTest(schema=filename): schema = load(filename) - jsonschema.Draft7Validator.check_schema(schema) + jsonschema.Draft201909Validator.check_schema(schema) class TestDTValidate(unittest.TestCase):