Origin: upstream, 7b9c4e8412eac6fbda24092ad0d3577e9437dfe3
From: Rob Herring <robh@kernel.org>
Date: Fri, 14 Apr 2023 14:01:26 -0500
Subject: Switch to jsonschema 4.18+

We've been stuck on jsonschema 4.17 because 4.18 changed in
non-compatible ways with how RefResolver worked. Upstream was not
willing to fix the issues claiming our usage was incorrect.

The RefResolver interface is deprecated anyways so it isn't really worth
the effort to fix or find a work-around when we need to move to the
Registry interface anyways.

In 4.17, the Draft2019-09 meta-schema was (silently) broken due to
'$recursiveRef' not being handled, so we were stuck using Draft7
validator in some places. Now those are all updated to Draft2019-09.

Setting '$schema' to our meta-schema seems to not be how extending
the meta-schema is supposed to be done. '$schema' is supposed to be
which dialect of json-schema we are using (Draft 2019-09) rather than
our extended meta-schema. Going against this seems to cause problems
with referencing the core schema within our schema, so that has been
dropped in favor of validating against both meta-schemas separately.
Further reworking of how our meta-schema is structured is needed.

4.18+ has been out for some time now and is what distros are packaging.
Also, 4.17 seems to be broken with python 3.14 and a fix by upstream
seems unlikely. So now is the time to switch.

Signed-off-by: Rob Herring <robh@kernel.org>
---
 dtschema/meta-schemas/base.yaml |  12 ---
 dtschema/schema.py              | 139 ++++++++++++++++++--------------
 dtschema/validator.py           |  50 +++++-------
 pyproject.toml                  |   2 +-
 test/schemas/bad-example.yaml   |   2 +-
 test/test-dt-validate.py        |   4 +-
 6 files changed, 103 insertions(+), 106 deletions(-)

diff --git a/dtschema/meta-schemas/base.yaml b/dtschema/meta-schemas/base.yaml
index 44f869f4..6a5de383 100644
--- a/dtschema/meta-schemas/base.yaml
+++ b/dtschema/meta-schemas/base.yaml
@@ -8,7 +8,6 @@ $schema: https://json-schema.org/draft/2019-09/schema
 description: Metaschema for devicetree binding documentation
 
 allOf:
-  - $ref: http://json-schema.org/draft-07/schema#
   - $ref: http://devicetree.org/meta-schemas/keywords.yaml#
 
 properties:
@@ -43,17 +42,6 @@ properties:
       type: string
       format: email
 
-  select:
-    description: '"select" must contain a valid json-schema'
-    allOf:
-      - $ref: http://json-schema.org/draft-07/schema#
-      - oneOf:
-          - type: object
-            properties:
-              properties: true
-              required: true
-          - type: boolean
-
 propertyNames:
   enum: [ $id, $schema, title, description, examples, required, allOf, anyOf, oneOf,
           definitions, $defs, additionalProperties, dependencies, dependentRequired,
diff --git a/dtschema/schema.py b/dtschema/schema.py
index 85f63d57..22c045b8 100644
--- a/dtschema/schema.py
+++ b/dtschema/schema.py
@@ -6,11 +6,16 @@
 import os
 import re
 import copy
+
+from functools import cache
+
 import jsonschema
 
-import dtschema
+from referencing import Registry
+from referencing.exceptions import NoSuchResource, Unresolvable
+from referencing.jsonschema import DRAFT201909
 
-from jsonschema.exceptions import RefResolutionError
+import dtschema
 
 schema_base_url = "http://devicetree.org/"
 schema_basedir = os.path.dirname(os.path.abspath(__file__))
@@ -55,6 +60,26 @@ def _schema_allows_no_undefined_props(schema):
     return not additional_props or isinstance(additional_props, dict) or \
            not uneval_props or isinstance(uneval_props, dict)
 
+
+@cache
+def create_meta_schema_registry():
+    import glob
+    import ruamel.yaml
+
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.allow_duplicate_keys = False
+
+    metaschemas = []
+    for filename in glob.iglob(schema_basedir + "/meta-schemas/*.yaml"):
+        with open(filename, 'r', encoding='utf-8') as f:
+            metaschema = yaml.load(f.read())
+            metaschemas += [(
+                metaschema['$id'],
+                DRAFT201909.create_resource(metaschema)
+            )]
+    return Registry().with_resources(metaschemas)
+
+
 class DTSchema(dict):
     DtValidator = jsonschema.validators.extend(
         jsonschema.Draft201909Validator,
@@ -78,24 +103,13 @@ def __init__(self, schema_file, line_numbers=False):
                 schema = yaml.load(f.read())
 
         self.filename = os.path.abspath(schema_file)
-        self._validator = None
 
-        id = schema['$id'].rstrip('#')
-        match = re.search('(.*/schemas/)(.+)$', id)
-        self.base_path = os.path.abspath(schema_file)[:-len(match[2])]
+        self.DtValidator.META_SCHEMA['properties']['select'] = {'$recursiveRef': '#'}
 
-        super().__init__(schema)
-
-    def validator(self):
-        if not self._validator:
-            resolver = jsonschema.RefResolver.from_schema(self,
-                                handlers={'http': self.http_handler})
-            meta_schema = resolver.resolve_from_url(self['$schema'])
-            self._validator = self.DtValidator(meta_schema, resolver=resolver)
 
-        return self._validator
+        super().__init__(schema)
 
-    def http_handler(self, uri):
+    def retrieve(self, uri):
         '''Custom handler for http://devicetree.org references'''
         uri = uri.rstrip('#')
         missing_files = ''
@@ -108,30 +122,33 @@ def http_handler(self, uri):
                 import ruamel.yaml
                 yaml = ruamel.yaml.YAML(typ='safe')
                 yaml.allow_duplicate_keys = False
-                return yaml.load(f.read())
 
-        raise RefResolutionError(f'Error in referenced schema matching $id: {uri}\n\tTried these paths (check schema $id if path is wrong):\n{missing_files}')
+                return DRAFT201909.create_resource(yaml.load(f.read()))
 
-    def annotate_error(self, error, schema, path):
+        raise NoSuchResource(ref=uri)
+
+    def annotate_error(self, error, uri, path):
         error.note = None
-        error.schema_file = None
 
         for e in error.context:
-            self.annotate_error(e, schema, path + e.schema_path)
+            self.annotate_error(e, uri, path + e.schema_path)
 
-        scope = self.validator().ID_OF(schema)
-        self.validator().resolver.push_scope(scope)
-        ref_depth = 1
+        registry = create_meta_schema_registry()
+        schema = registry[uri].contents
+        error.schema_file = uri
 
         for p in path:
             while p not in schema and '$ref' in schema and isinstance(schema['$ref'], str):
-                ref = self.validator().resolver.resolve(schema['$ref'])
-                schema = ref[1]
-                self.validator().resolver.push_scope(ref[0])
-                ref_depth += 1
-
-            if '$id' in schema and isinstance(schema['$id'], str):
-                error.schema_file = schema['$id']
+                ref = schema['$ref']
+                if ref.startswith('#'):
+                    schema = registry.resolver(uri).lookup(ref).contents
+                else:
+                    for rsrc in registry:
+                        if ref.split('#')[0] in rsrc:
+                            uri = rsrc
+                            break
+                    schema = registry.resolver(uri).lookup(ref).contents
+                    error.schema_file = uri
 
             schema = schema[p]
 
@@ -139,40 +156,38 @@ def annotate_error(self, error, schema, path):
                 if 'description' in schema and isinstance(schema['description'], str):
                     error.note = schema['description']
 
-        while ref_depth > 0:
-            self.validator().resolver.pop_scope()
-            ref_depth -= 1
-
         if isinstance(error.schema, dict) and 'description' in error.schema:
             error.note = error.schema['description']
 
-    def iter_errors(self):
-        meta_schema = self.validator().resolver.resolve_from_url(self['$schema'])
+    def iter_errors(self, strict=True):
+        if strict:
+            registry = create_meta_schema_registry()
+
+            meta_schema_id = self['$schema'].rstrip('#')
+            self.validator = self.DtValidator(registry.contents(meta_schema_id), registry=registry)
 
-        for error in self.validator().iter_errors(self):
+            for error in self.validator.iter_errors(self):
+                scherr = jsonschema.exceptions.SchemaError.create_from(error)
+                self.annotate_error(scherr, meta_schema_id, scherr.schema_path)
+                scherr.linecol = get_line_col(self, scherr.path)
+                yield scherr
+
+        for error in self.DtValidator(self.DtValidator.META_SCHEMA).iter_errors(self):
             scherr = jsonschema.exceptions.SchemaError.create_from(error)
-            self.annotate_error(scherr, meta_schema, scherr.schema_path)
             scherr.linecol = get_line_col(self, scherr.path)
             yield scherr
 
     def is_valid(self, strict=False):
         ''' Check if schema passes validation against json-schema.org schema '''
-        if strict:
-            for error in self.iter_errors():
-                raise error
-        else:
-            # Using the draft7 metaschema because 2019-09 with $recursiveRef seems broken
-            # Probably fixed with referencing library
-            for error in self.DtValidator(jsonschema.Draft7Validator.META_SCHEMA).iter_errors(self):
-                scherr = jsonschema.exceptions.SchemaError.create_from(error)
-                raise scherr
+        for error in self.iter_errors(strict):
+            raise error
 
     def fixup(self):
         processed_schema = copy.deepcopy(dict(self))
         dtschema.fixups.fixup_schema(processed_schema)
         return processed_schema
 
-    def _check_schema_refs(self, schema, parent=None, is_common=False, has_constraint=False):
+    def _check_schema_refs(self, resolver, schema, parent=None, is_common=False, has_constraint=False):
         if not parent:
             is_common = not _schema_allows_no_undefined_props(schema)
         if isinstance(schema, dict):
@@ -185,8 +200,7 @@ def _check_schema_refs(self, schema, parent=None, is_common=False, has_constrain
 
             ref_has_constraint = True
             if '$ref' in schema:
-                ref = schema['$ref']
-                url, ref_sch = self.validator().resolver.resolve(ref)
+                ref_sch = resolver.lookup(schema['$ref']).contents
                 ref_has_constraint = _schema_allows_no_undefined_props(ref_sch)
 
             if not (is_common or ref_has_constraint or has_constraint or
@@ -195,17 +209,18 @@ def _check_schema_refs(self, schema, parent=None, is_common=False, has_constrain
                       file=sys.stderr)
 
             for k, v in schema.items():
-                self._check_schema_refs(v, parent=k, is_common=is_common,
+                self._check_schema_refs(resolver, v, parent=k, is_common=is_common,
                                         has_constraint=has_constraint)
         elif isinstance(schema, (list, tuple)):
             for i in range(len(schema)):
-                self._check_schema_refs(schema[i], parent=parent, is_common=is_common,
+                self._check_schema_refs(resolver, schema[i], parent=parent, is_common=is_common,
                                         has_constraint=has_constraint)
 
     def check_schema_refs(self):
         id = self['$id'].rstrip('#')
         base1 = re.search('schemas/(.+)$', id)[1]
-        base2 = self.filename.replace(self.filename[:-len(base1)], '')
+        base_path = self.filename[:-len(base1)]
+        base2 = self.filename.replace(base_path, '')
         if not base1 == base2:
             print(f"{self.filename}: $id: Cannot determine base path from $id, relative path/filename doesn't match actual path or filename\n",
                   f"\t $id: {id}\n",
@@ -213,16 +228,16 @@ def check_schema_refs(self):
                   file=sys.stderr)
             return
 
-        scope = self.validator().ID_OF(self)
-        if scope:
-            self.validator().resolver.push_scope(scope)
-
         self.paths = [
-            (schema_base_url + 'schemas/', self.base_path),
+            (schema_base_url + 'schemas/', base_path),
             (schema_base_url + 'schemas/', schema_basedir + '/schemas/'),
         ]
 
+        rsrc = DRAFT201909.create_resource(self)
+        registry = Registry(retrieve=self.retrieve).with_resource(uri=id, resource=rsrc)
+        resolver = registry.resolver_with_root(rsrc)
+
         try:
-            self._check_schema_refs(self)
-        except jsonschema.RefResolutionError as exc:
-            print(f"{self.filename}:\n\t{exc}", file=sys.stderr)
+            self._check_schema_refs(resolver, self)
+        except Unresolvable as exc:
+            print(f"{self.filename}: Unresolvable reference: {exc}", file=sys.stderr)
diff --git a/dtschema/validator.py b/dtschema/validator.py
index 2d1ccc65..50fe07dd 100644
--- a/dtschema/validator.py
+++ b/dtschema/validator.py
@@ -10,7 +10,8 @@
 import json
 import jsonschema
 
-from jsonschema.exceptions import RefResolutionError
+from referencing import Registry
+from referencing.jsonschema import DRAFT201909
 
 import dtschema
 from dtschema.lib import _is_string_schema
@@ -297,11 +298,12 @@ def _add_schema(schemas, filename):
     sch = process_schema(os.path.abspath(filename))
     if not sch or '$id' not in sch:
         return False
-    if sch['$id'] in schemas:
-        print(f"{sch['$filename']}: warning: ignoring duplicate '$id' value '{sch['$id']}'", file=sys.stderr)
+    _id = sch['$id'].rstrip('#')
+    if _id in schemas:
+        print(f"{sch['$filename']}: warning: ignoring duplicate '$id' value '{_id}'", file=sys.stderr)
         return False
     else:
-        schemas[sch['$id']] = sch
+        schemas[_id] = sch
 
     return True
 
@@ -342,6 +344,11 @@ def typeSize(validator, typeSize, instance, schema):
         yield jsonschema.ValidationError("size is %r, expected %r" % (size, typeSize))
 
 
+def _get_schema_by_path(schema, path):
+    if len(path)==1: return schema[path[0]]
+    return _get_schema_by_path(schema[path[0]],path[1:])
+
+
 class DTValidator:
     '''Custom Validator for Devicetree Schemas
 
@@ -352,9 +359,9 @@ class DTValidator:
     '''
     DtValidator = jsonschema.validators.extend(jsonschema.Draft201909Validator, {'typeSize': typeSize})
 
-    def __init__(self, schema_files, filter=None):
+    def __init__(self, schema_files, id_filter=""):
         self.schemas = {}
-        self.resolver = jsonschema.RefResolver('', None, handlers={'http': self.http_handler})
+        self.registry = Registry(retrieve=self.retrieve)
         schema_cache = None
 
         if len(schema_files) == 1 and os.path.isfile(schema_files[0]):
@@ -399,9 +406,10 @@ def __init__(self, schema_files, filter=None):
         self.always_schemas = []
         self.compat_map = {}
         for sch in self.schemas.values():
+            _id = sch['$id'].rstrip('#')
             if 'select' in sch:
                 if sch['select'] is not False:
-                    self.always_schemas += [sch['$id']]
+                    self.always_schemas += [_id]
             elif 'properties' in sch and 'compatible' in sch['properties']:
                 compatibles = dtschema.extract_node_compatibles(sch['properties']['compatible'])
                 if len(compatibles) > 1:
@@ -410,22 +418,12 @@ def __init__(self, schema_files, filter=None):
                     if not schema_cache and c in self.compat_map:
                         print(f'Warning: Duplicate compatible "{c}" found in schemas matching "$id":\n'
                               f'\t{self.compat_map[c]}\n\t{sch["$id"]}', file=sys.stderr)
-                    self.compat_map[c] = sch['$id']
+                    self.compat_map[c] = _id
 
         self.schemas['version'] = dtschema.__version__
 
-    def http_handler(self, uri):
-        '''Custom handler for http://devicetree.org references'''
-        try:
-            uri += '#'
-            if uri in self.schemas:
-                return self.schemas[uri]
-            else:
-                # If we have a schema_cache, then the schema should have been there unless the schema had errors
-                if len(self.schemas):
-                    return {'not': {'description': f"Can't find referenced schema: {uri}"}}
-        except:
-            raise RefResolutionError('Error in referenced schema matching $id: ' + uri)
+    def retrieve(self, uri):
+        return DRAFT201909.create_resource(self.schemas[uri])
 
     def annotate_error(self, id, error):
         error.schema_file = id
@@ -447,10 +445,8 @@ def iter_errors(self, instance, filter=None, compatible_match=False):
                     schema_id = self.compat_map[inst_compat]
                     if self._filter_match(schema_id, filter):
                         schema = self.schemas[schema_id]
-                        for error in self.DtValidator(schema,
-                                                    resolver=self.resolver,
-                                                    ).iter_errors(instance):
-                            self.annotate_error(schema['$id'], error)
+                        for error in self.DtValidator(schema, registry=self.registry).iter_errors(instance):
+                            self.annotate_error(schema_id, error)
                             yield error
                     break
 
@@ -462,9 +458,7 @@ def iter_errors(self, instance, filter=None, compatible_match=False):
                 continue
             schema = {'if': self.schemas[schema_id]['select'],
                       'then': self.schemas[schema_id]}
-            for error in self.DtValidator(schema,
-                                          resolver=self.resolver,
-                                          ).iter_errors(instance):
+            for error in self.DtValidator(schema, registry=self.registry).iter_errors(instance):
                 self.annotate_error(schema_id, error)
                 yield error
 
@@ -486,7 +480,7 @@ def check_missing_property_types(self):
         for p, val in self.props.items():
             if val[0]['type'] is None:
                 for id in val[0]['$id']:
-                    print(f"{self.schemas[id]['$filename']}: {p}: missing type definition", file=sys.stderr)
+                    print(f"{self.schemas[id.rstrip('#')]['$filename']}: {p}: missing type definition", file=sys.stderr)
 
     def check_duplicate_property_types(self):
         """
diff --git a/pyproject.toml b/pyproject.toml
index 0f4b4d04..0a3c6ddf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ dynamic = ["version"]
 
 dependencies = [
     "ruamel.yaml>0.15.69",
-    "jsonschema>=4.1.2,<4.18",
+    "jsonschema>=4.18",
     "rfc3987",
     "pylibfdt",
 ]
diff --git a/test/schemas/bad-example.yaml b/test/schemas/bad-example.yaml
index a4131a1e..fcece00e 100644
--- a/test/schemas/bad-example.yaml
+++ b/test/schemas/bad-example.yaml
@@ -4,7 +4,7 @@
 %YAML 1.2
 ---
 $id: http://devicetree.org/schemas/bad-example.yaml
-$schema: http://json-schema.org/draft-07/schema#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
 
 title: 0
 
diff --git a/test/test-dt-validate.py b/test/test-dt-validate.py
index 5deaae6d..19a5eb15 100755
--- a/test/test-dt-validate.py
+++ b/test/test-dt-validate.py
@@ -97,7 +97,7 @@ def test_binding_schemas_id_is_unique(self):
                 self.assertEqual(ids.count(schema['$id']), 0)
                 ids.append(schema['$id'])
 
-    def test_binding_schemas_valid_draft7(self):
+    def test_binding_schemas_valid_draft201909(self):
         '''Test that all schema files under ./dtschema/schemas/ validate against the Draft7 metaschema
         The DT Metaschema is supposed to force all schemas to be valid against
         Draft7. This test makes absolutely sure that they are.
@@ -105,7 +105,7 @@ def test_binding_schemas_valid_draft7(self):
         for filename in glob.iglob(os.path.join(dtschema_dir, 'schemas/**/*.yaml'), recursive=True):
             with self.subTest(schema=filename):
                 schema = load(filename)
-                jsonschema.Draft7Validator.check_schema(schema)
+                jsonschema.Draft201909Validator.check_schema(schema)
 
 
 class TestDTValidate(unittest.TestCase):