File: test_compute_features.py

package info (click to toggle)
python-vector 1.6.3-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 2,984 kB
  • sloc: python: 40,200; makefile: 13
file content (376 lines) | stat: -rw-r--r-- 11,401 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
# Copyright (c) 2019-2025, Saransh Chopra, Henry Schreiner, Eduardo Rodrigues, Jonas Eschle, and Jim Pivarski.
#
# Distributed under the 3-clause BSD license, see accompanying file LICENSE
# or https://github.com/scikit-hep/vector for details.

"""
Ensures that new or modified vector.compute.* functions don't break any existing
or future backends by using unsupportable Python language features.

Compute functions are highly restricted, a least common denominator for all the
backends we *ever* want to support. The functions themselves are duck-typed:
arguments could be numbers, NumPy arrays, Awkward Arrays, and potentially
TensorFlow/Torch/JAX/etc. An ``if`` statement on individual numbers would have
to be ``np.where` or a masked assignment in NumPy, so ``if`` is not allowed.
JAX traces a function for JIT-compilation and autodifferentiation by passing a
"tracer" object through it, and that object can only follow one code path, another
reason to exclude ``if`` statements. Loops are even more problematic.

This suite of tests statically analyzes all of the compute functions by decompiling
their bytecode with uncompyle6 (on Python 3.8; will have to be modified slightly
every few years). Some compute functions are dynamically generated, so they don't
all have an AST to inspect.

The sieve has been defined narrowly: compute functions can use more functions,
binary operators, and possibly more language features than are allowed here.
Expanding this set of rules is therefore allowed and encouraged. The test failure
and requirement to expand the rules is intended to force you to think about
new features, to ask yourself if they can be supported by all current and hoped-for
backends, and whether a (formally) simpler implementation is possible.
"""

from __future__ import annotations

import collections
import contextlib
import inspect
import sys

import pytest

import vector._compute.lorentz
import vector._compute.planar
import vector._compute.spatial

uncompyle6 = pytest.importorskip("uncompyle6")
spark_parser = pytest.importorskip("spark_parser")
pytestmark = pytest.mark.dis

Context = collections.namedtuple("Context", ["name", "closure"])


functions = dict(
    [
        (
            f"{y.__name__}({', '.join(repr(v) if isinstance(v, str) else v.__name__ for v in w)})",
            z[0],
        )
        for x, y in inspect.getmembers(
            vector._compute.planar, predicate=inspect.ismodule
        )
        if hasattr(y, "dispatch_map")
        for w, z in y.dispatch_map.items()
    ]
    + [
        (
            f"{y.__name__}({', '.join(repr(v) if isinstance(v, str) else v.__name__ for v in w)})",
            z[0],
        )
        for x, y in inspect.getmembers(
            vector._compute.spatial, predicate=inspect.ismodule
        )
        if hasattr(y, "dispatch_map")
        for w, z in y.dispatch_map.items()
    ]
    + [
        (
            f"{y.__name__}({', '.join(repr(v) if isinstance(v, str) else v.__name__ for v in w)})",
            z[0],
        )
        for x, y in inspect.getmembers(
            vector._compute.lorentz, predicate=inspect.ismodule
        )
        if hasattr(y, "dispatch_map")
        for w, z in y.dispatch_map.items()
    ]
)


python_version = f"{sys.version_info[0]}.{sys.version_info[1]}"
is_pypy = "__pypy__" in sys.builtin_module_names
try:
    uncompyle6.scanner.get_scanner(python_version, is_pypy=is_pypy)
except RuntimeError as err:
    is_unsupported = True
    unsupported_message = str(err)
else:
    is_unsupported = False
    unsupported_message = ""


@pytest.mark.skipif(is_unsupported, reason=unsupported_message)
@pytest.mark.slow
@pytest.mark.parametrize("signature", functions.keys())
def test(signature):
    analyze_function(functions[signature])


def analyze_function(function):
    if function not in analyze_function.done:
        closure = dict(function.__globals__)
        if function.__closure__ is not None:
            for var, cell in zip(function.__code__.co_freevars, function.__closure__):
                # the cell has not been filled yet, so ignore it
                with contextlib.suppress(ValueError):
                    closure[var] = cell.cell_contents

        analyze_code(function.__code__, Context(function.__name__, closure))
        analyze_function.done.add(function)


analyze_function.done = set()


def analyze_code(code, context):
    # this block is all uncompyle6
    parser = uncompyle6.parser.get_python_parser(
        python_version,
        debug_parser=dict(spark_parser.DEFAULT_DEBUG),
        compile_mode="exec",
        is_pypy=is_pypy,
    )
    scanner = uncompyle6.scanner.get_scanner(python_version, is_pypy=is_pypy)
    tokens, customize = scanner.ingest(code, code_objects={}, show_asm=False)
    parsed = uncompyle6.parser.parse(parser, tokens, customize, code)

    # now the disassembled bytecodes have been parsed into a tree for us to walk
    analyze_body(parsed, context)


def analyze_body(node, context):
    assert node.kind == "stmts"
    assert len(node) >= 1

    for statement in node[:-1]:
        analyze_assignment(statement, context)
    analyze_return(node[-1], context)


def analyze_assignment(node, context):
    assert node.kind == "sstmt"
    assert len(node) == 1

    assert node[0].kind == "assign", (
        "only assignments and a final 'return' are allowed (and not tuple-assignment)"
    )
    assert len(node[0]) == 2
    assert node[0][1].kind == "store"

    if node[0][1][0].kind == "STORE_FAST":
        analyze_expression(expr(node[0][0]), context)

    elif node[0][1][0].kind == "unpack":
        assert len(node[0][1][0]) >= 2
        assert node[0][1][0][0].kind.startswith("UNPACK_SEQUENCE")
        for item in node[0][1][0][1:]:
            assert item.kind == "store"
            assert len(item) == 1
            assert item[0].kind == "STORE_FAST"

    else:
        print(node[0][1][0])
        raise AssertionError("what is this?")


def expr(node):
    assert node.kind == "expr"
    assert len(node) == 1
    return node[0]


def is_pi(node):
    return (
        node.kind == "attribute"
        and len(node) == 2
        and expr(node[0]).kind == "LOAD_FAST"
        and expr(node[0]).attr == "lib"
        and node[1].kind == "LOAD_ATTR"
        and node[1].attr == "pi"
    )


def is_nan_to_num(node):
    if node.kind != "call_kw36" or len(node) < 3:
        return False

    function = expr(node[0])
    return (
        function.kind == "attribute"
        and expr(function[0]).attr == "lib"
        and function[1].attr == "nan_to_num"
    )


def analyze_return(node, context):
    assert node.kind == "sstmt"
    assert len(node) == 1

    assert node[0].kind == "return", "compute function must end with a 'return'"
    assert len(node[0]) == 2
    assert node[0][0].kind in ("ret_expr", "return_expr")
    assert len(node[0][0]) == 1
    expr(node[0][0][0])
    assert node[0][1].kind == "RETURN_VALUE"

    if node[0][0][0][0].kind == "tuple":
        assert len(node[0][0][0][0]) >= 2, "returning an empty tuple?"
        assert node[0][0][0][0][-1].kind.startswith("BUILD_TUPLE")
        for item in node[0][0][0][0][:-1]:
            analyze_expression(expr(item), context)

    else:
        analyze_expression(node[0][0][0][0], context)


def analyze_expression(node, context):
    if node.kind == "LOAD_FAST":
        # Don't bother checking to see if this variable has been defined.
        # Unit checks test that if the coverage is complete.
        pass

    elif node.kind == "LOAD_CONST":
        assert isinstance(node.attr, (int, float))

    elif is_pi(node):
        pass

    elif node.kind == "unary_op":
        assert len(node) == 2
        analyze_expression(expr(node[0]), context)
        assert node[1].kind == "unary_operator"
        assert len(node[1]) == 1
        analyze_unary_operator(node[1][0], context)

    elif node.kind == "bin_op":
        assert len(node) == 3
        analyze_expression(expr(node[0]), context)
        analyze_expression(expr(node[1]), context)
        assert node[2].kind == "binary_operator"
        assert len(node[2]) == 1
        analyze_binary_operator(node[2][0], context)

    elif node.kind == "compare":
        assert len(node) == 1
        assert node[0].kind == "compare_single", "only do single comparisons"
        assert len(node[0]) == 3
        analyze_expression(expr(node[0][0]), context)
        analyze_expression(expr(node[0][1]), context)
        assert node[0][2].kind == "COMPARE_OP"
        assert node[0][2].attr in allowed_comparisons, (
            f"add {node[0][2].attr!r} to allowed_comparisons"
        )

    elif node.kind == "call":
        assert len(node) >= 2

        assert node[-1].kind.startswith("CALL_METHOD") or node[-1].kind.startswith(
            "CALL_FUNCTION"
        )
        analyze_callable(expr(node[0]), context)

        for argument in node[1:-1]:
            expr_arg = argument[0] if argument.kind == "pos_arg" else argument
            assert expr_arg.kind == "expr", "only positional arguments"
            analyze_expression(expr(expr_arg), context)

    elif is_nan_to_num(node):
        analyze_expression(expr(node[1]), context)

    else:
        print(node)
        raise AssertionError("what is this?")


def analyze_unary_operator(node, context):
    assert node.kind in allowed_unary_operators, (
        f"add {node.kind!r} to allowed_unary_operators"
    )


def analyze_binary_operator(node, context):
    assert node.kind in allowed_binary_operators, (
        f"add {node.kind!r} to allowed_binary_operators"
    )


def analyze_callable(node, context):
    if node.kind == "attribute37":
        assert len(node) == 2
        module = expr(node[0])
        assert module.kind in {"LOAD_FAST", "LOAD_GLOBAL"}
        assert node[1].kind == "LOAD_METHOD"

        if module.attr == "lib":
            assert node[1].attr in allowed_lib_functions, (
                f"add {node[1].attr!r} to allowed_lib_functions"
            )

        else:
            module_name = ".".join(
                context.closure.get(module.attr).__name__.split(".")[:-1]
            )
            assert module_name in (
                "vector._compute.planar",
                "vector._compute.spatial",
                "vector._compute.lorentz",
            )

    elif node.kind in {"LOAD_GLOBAL", "LOAD_DEREF"}:
        function = context.closure.get(node.attr)
        assert function is not None, f"unrecognized function in scope: {node.attr!r}"
        analyze_function(function)

    else:
        print(node)
        raise AssertionError("what is this?")


allowed_unary_operators = [
    "UNARY_NEGATIVE",
]

allowed_binary_operators = [
    "BINARY_ADD",
    "BINARY_SUBTRACT",
    "BINARY_MULTIPLY",
    "BINARY_TRUE_DIVIDE",
    "BINARY_MODULO",
    "BINARY_POWER",
    "BINARY_AND",
]

allowed_comparisons = [
    "==",
    "!=",
    "<",
    ">",
]

allowed_lib_functions = [
    "absolute",
    "sign",
    "copysign",
    "maximum",
    "minimum",
    "sqrt",
    "exp",
    "log",
    "sin",
    "cos",
    "tan",
    "arcsin",
    "arccos",
    "arctan",
    "arctan2",
    "sinh",
    "cosh",
    "tanh",
    "arcsinh",
    "arccosh",
    "arctanh",
    "isclose",
    # TODO: https://github.com/scikit-hep/vector/issues/615
    # remove once https://github.com/cupy/cupy/issues/9143
    # is fixed.
    "where",
]