File: lazy_nodes.py

package info (click to toggle)
python-asdf 4.3.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 7,032 kB
  • sloc: python: 24,068; makefile: 123
file content (314 lines) | stat: -rw-r--r-- 10,353 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
"""
Objects that act like dict, list, OrderedDict but allow
lazy conversion of tagged ASDF tree nodes to custom objects.
"""

import collections
import inspect
import warnings
import weakref

from . import tagged, treeutil, yamlutil
from .exceptions import AsdfConversionWarning, AsdfLazyReferenceError
from .extension._serialization_context import BlockAccess

__all__ = ["AsdfDictNode", "AsdfListNode", "AsdfOrderedDictNode"]


class _TaggedObjectCacheItem:
    """
    A tagged node and a (weakref) to the converted custom object
    """

    def __init__(self, tagged_node, custom_object):
        self.tagged_node = tagged_node
        try:
            self._custom_object_ref = weakref.ref(custom_object)
        except TypeError:
            # if a weakref is not possible, store the object
            self._custom_object_ref = lambda obj=custom_object: obj

    @property
    def custom_object(self):
        return self._custom_object_ref()


class _TaggedObjectCache:
    """
    A cache of tagged nodes and their corresponding custom objects.

    This is critical for trees that contain references/pointers to the
    same object at multiple locations in the tree.

    Only weakrefs are key to the custom objects to allow large items
    deleted from the tree to be garbage collected. This means that an
    item added to the cache may later fail to retrieve (if the weakref-ed
    custom object was deleted).
    """

    def __init__(self):
        # start with a clear cache
        self.clear()

    def clear(self):
        self._cache = {}

    def retrieve(self, tagged_node):
        """
        Check the cache for a previously converted object.

        Parameters
        ----------
        tagged_node : Tagged
            The tagged representation of the custom object

        Returns
        -------
        custom_object : None or the converted object
            The custom object previously converted from the tagged_node or
            ``None`` if the object hasn't been converted (or was previously
            deleted from the tree).
        """
        key = id(tagged_node)
        if key not in self._cache:
            return None
        item = self._cache[key]
        custom_object = item.custom_object
        if custom_object is None:
            del self._cache[key]
        return custom_object

    def store(self, tagged_node, custom_object):
        """
        Store a converted custom object in the cache.

        Parameters
        ----------
        tagged_node : Tagged
            The tagged representation of the custom object

        custom_object : converted object
            The custom object (a weakref to this object will be kept in the cache).
        """
        self._cache[id(tagged_node)] = _TaggedObjectCacheItem(tagged_node, custom_object)


def _resolve_af_ref(af_ref):
    msg = "Failed to resolve AsdfFile reference"
    if af_ref is None:
        raise AsdfLazyReferenceError(msg)
    af = af_ref()
    if af is None:
        raise AsdfLazyReferenceError(msg)
    return af


def _to_lazy_node(node, af_ref):
    """
    Convert an object to a _AsdfNode subclass.
    If the object does not have a corresponding subclass
    it will be returned unchanged.
    """
    if isinstance(node, list):
        return AsdfListNode(node, af_ref)
    elif isinstance(node, collections.OrderedDict):
        return AsdfOrderedDictNode(node, af_ref)
    elif isinstance(node, dict):
        return AsdfDictNode(node, af_ref)
    return node


class _AsdfNode:
    """
    The "lazy node" base class that handles object
    conversion and wrapping and contains a weak reference
    to the `asdf.AsdfFile` that triggered the creation of this
    node (when the "lazy tree" was loaded).
    """

    def __init__(self, data=None, af_ref=None):
        self._af_ref = af_ref
        self.data = data

    @property
    def tagged(self):
        """
        Return the tagged tree backing this node
        """
        return self.data

    def __deepcopy__(self, memo):
        return treeutil.walk_and_modify(self, lambda n: n)

    def _convert_and_cache(self, value, key):
        """
        Convert ``value`` to either:

            - a custom object if ``value`` is `asdf.tagged.Tagged`
            - an ``asdf.lazy_nodes.AsdfListNode` if ``value`` is
              a ``list``
            - an ``asdf.lazy_nodes.AsdfDictNode` if ``value`` is
              a ``dict``
            - an ``asdf.lazy_nodes.AsdfOrderedDictNode` if ``value`` is
              a ``OrderedDict``
            - otherwise return ``value`` unmodified

        After conversion the result (``obj``) will be stored in this
        `asdf.lazy_nodes._AsdfNode` using the provided key and cached
        in the corresponding `asdf.AsdfFile` instance (so other
        references to ``value`` in the tree will return the same
        ``obj``).

        Parameters
        ----------
        value :
            The object to convert from a Tagged to custom object
            or wrap with an _AsdfNode or return unmodified.

        key :
            The key under which the converted/wrapped object will
            be stored.


        Returns
        -------
        obj :
            The converted or wrapped (or the value if no conversion
            or wrapping is required).
        """
        # if the value has already been wrapped, return it
        if isinstance(value, _AsdfNode):
            return value
        if not isinstance(value, tagged.Tagged) and type(value) not in _base_type_to_node_map:
            return value
        af = _resolve_af_ref(self._af_ref)
        # if the obj that will be returned from this value
        # is already cached, use the cached obj
        if (obj := af._tagged_object_cache.retrieve(value)) is not None:
            self[key] = obj
            return obj
        # for Tagged instances, convert them to their custom obj
        if isinstance(value, tagged.Tagged):
            extension_manager = af.extension_manager
            tag = value._tag
            if not extension_manager.handles_tag(tag):
                if not af._ignore_unrecognized_tag:
                    warnings.warn(
                        f"{tag} is not recognized, converting to raw Python data structure",
                        AsdfConversionWarning,
                    )
                obj = _to_lazy_node(value, self._af_ref)
            else:
                converter = extension_manager.get_converter_for_tag(tag)
                if not getattr(converter, "lazy", False) or inspect.isgeneratorfunction(
                    converter._delegate.from_yaml_tree
                ):
                    obj = yamlutil.tagged_tree_to_custom_tree(value, af)
                else:
                    data = _to_lazy_node(value.data, self._af_ref)
                    sctx = af._create_serialization_context(BlockAccess.READ)
                    obj = converter.from_yaml_tree(data, tag, sctx)
                    sctx.assign_object(obj)
                    sctx.assign_blocks()
                    sctx._mark_extension_used(converter.extension)
        else:
            # for non-tagged objects, wrap in an _AsdfNode
            node_type = _base_type_to_node_map[type(value)]
            obj = node_type(value, self._af_ref)
        # cache the converted/wrapped obj with the AsdfFile so other
        # references to the same Tagged value will result in the
        # same obj
        af._tagged_object_cache.store(value, obj)
        self[key] = obj
        return obj


class AsdfListNode(_AsdfNode, collections.UserList):
    """
    An class that acts like a ``list``. The items in this ``list``
    will start out as tagged nodes which will only be converted to
    custom objects the first time they are indexed (the custom object
    will then be cached for later reuse).

    If sliced, this will return a new instance of `AsdfListNode` for
    the sliced portion of the list.
    """

    def __init__(self, data=None, af_ref=None):
        if data is None:
            data = []
        _AsdfNode.__init__(self, data, af_ref)
        collections.UserList.__init__(self, data)

    def __copy__(self):
        return AsdfListNode(self.data.copy(), self._af_ref)

    def __eq__(self, other):
        if self is other:
            return True
        return list(self) == list(other)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __getitem__(self, key):
        # key might be an int or slice
        value = super().__getitem__(key)
        if isinstance(key, slice):
            return AsdfListNode(value, self._af_ref)
        return self._convert_and_cache(value, key)


class AsdfDictNode(_AsdfNode, collections.UserDict):
    """
    An class that acts like a ``dict``. The values for this
    ``dict`` will start out as tagged nodes which will only
    be converted to custom objects the first time the corresponding
    key is used (the custom object will then be cached for later
    reuse).
    """

    def __init__(self, data=None, af_ref=None):
        if data is None:
            data = {}
        _AsdfNode.__init__(self, data, af_ref)
        collections.UserDict.__init__(self, data)

    def __copy__(self):
        return AsdfDictNode(self.data.copy(), self._af_ref)

    def __eq__(self, other):
        if self is other:
            return True
        return dict(self) == dict(other)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __getitem__(self, key):
        return self._convert_and_cache(super().__getitem__(key), key)


class AsdfOrderedDictNode(AsdfDictNode, collections.OrderedDict):
    """
    An class that acts like a ``collections.OrderedDict``. The values
    for this ``OrderedDict`` will start out as tagged nodes which will only
    be converted to custom objects the first time the corresponding
    key is used (the custom object will then be cached for later
    reuse).
    """

    def __init__(self, data=None, af_ref=None):
        if data is None:
            data = collections.OrderedDict()
        AsdfDictNode.__init__(self, data, af_ref)

    def __copy__(self):
        return AsdfOrderedDictNode(self.data.copy(), self._af_ref)


_base_type_to_node_map = {
    dict: AsdfDictNode,
    list: AsdfListNode,
    collections.OrderedDict: AsdfOrderedDictNode,
}