1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
#ifndef MYPYC_UTIL_H
#define MYPYC_UTIL_H
#include <Python.h>
#include <frameobject.h>
#include <assert.h>
#if defined(__clang__) || defined(__GNUC__)
#define likely(x) __builtin_expect((x),1)
#define unlikely(x) __builtin_expect((x),0)
#define CPy_Unreachable() __builtin_unreachable()
#else
#define likely(x) (x)
#define unlikely(x) (x)
#define CPy_Unreachable() abort()
#endif
#if defined(__clang__) || defined(__GNUC__)
#define CPy_NOINLINE __attribute__((noinline))
#elif defined(_MSC_VER)
#define CPy_NOINLINE __declspec(noinline)
#else
#define CPy_NOINLINE
#endif
#ifndef Py_GIL_DISABLED
// Everything is running in the same thread, so no need for thread locals
#define CPyThreadLocal
#else
// 1. Use C11 standard thread_local storage, if available
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__)
#define CPyThreadLocal _Thread_local
// 2. Microsoft Visual Studio fallback
#elif defined(_MSC_VER)
#define CPyThreadLocal __declspec(thread)
// 3. GNU thread local storage for GCC/Clang targets that still need it
#elif defined(__GNUC__) || defined(__clang__)
#define CPyThreadLocal __thread
#else
#error "Can't define CPyThreadLocal for this compiler/target (consider using a non-free-threaded Python build)"
#endif
#endif // Py_GIL_DISABLED
// INCREF and DECREF that assert the pointer is not NULL.
// asserts are disabled in release builds so there shouldn't be a perf hit.
// I'm honestly kind of surprised that this isn't done by default.
#define CPy_INCREF(p) do { assert(p); Py_INCREF(p); } while (0)
#define CPy_DECREF(p) do { assert(p); Py_DECREF(p); } while (0)
// Here just for consistency
#define CPy_XDECREF(p) Py_XDECREF(p)
#ifndef Py_GIL_DISABLED
// The *_NO_IMM operations below perform refcount manipulation for
// non-immortal objects (Python 3.12 and later).
//
// Py_INCREF and other CPython operations check for immortality. This
// can be expensive when we know that an object cannot be immortal.
//
// This optimization cannot be performed in free-threaded mode so we
// fall back to just calling the normal incref/decref operations.
static inline void CPy_INCREF_NO_IMM(PyObject *op)
{
op->ob_refcnt++;
}
static inline void CPy_DECREF_NO_IMM(PyObject *op)
{
if (--op->ob_refcnt == 0) {
_Py_Dealloc(op);
}
}
static inline void CPy_XDECREF_NO_IMM(PyObject *op)
{
if (op != NULL && --op->ob_refcnt == 0) {
_Py_Dealloc(op);
}
}
#define CPy_INCREF_NO_IMM(op) CPy_INCREF_NO_IMM((PyObject *)(op))
#define CPy_DECREF_NO_IMM(op) CPy_DECREF_NO_IMM((PyObject *)(op))
#define CPy_XDECREF_NO_IMM(op) CPy_XDECREF_NO_IMM((PyObject *)(op))
#else
#define CPy_INCREF_NO_IMM(op) CPy_INCREF(op)
#define CPy_DECREF_NO_IMM(op) CPy_DECREF(op)
#define CPy_XDECREF_NO_IMM(op) CPy_XDECREF(op)
#endif
// Tagged integer -- our representation of Python 'int' objects.
// Small enough integers are represented as unboxed integers (shifted
// left by 1); larger integers (larger than 63 bits on a 64-bit
// platform) are stored as a tagged pointer (PyObject *)
// representing a Python int object, with the lowest bit set.
// Tagged integers are always normalized. A small integer *must not*
// have the tag bit set.
typedef size_t CPyTagged;
typedef size_t CPyPtr;
#define CPY_INT_BITS (CHAR_BIT * sizeof(CPyTagged))
#define CPY_TAGGED_MAX (((Py_ssize_t)1 << (CPY_INT_BITS - 2)) - 1)
#define CPY_TAGGED_MIN (-((Py_ssize_t)1 << (CPY_INT_BITS - 2)))
#define CPY_TAGGED_ABS_MIN (0-(size_t)CPY_TAGGED_MIN)
typedef PyObject CPyModule;
// Tag bit used for long integers
#define CPY_INT_TAG 1
// Error value for signed fixed-width (low-level) integers
#define CPY_LL_INT_ERROR -113
// Error value for unsigned fixed-width (low-level) integers
#define CPY_LL_UINT_ERROR 239
// Error value for floats
#define CPY_FLOAT_ERROR -113.0
typedef void (*CPyVTableItem)(void);
static inline CPyTagged CPyTagged_ShortFromInt(int x) {
return x << 1;
}
static inline CPyTagged CPyTagged_ShortFromSsize_t(Py_ssize_t x) {
return x << 1;
}
// Are we targeting Python 3.X or newer?
#define CPY_3_11_FEATURES (PY_VERSION_HEX >= 0x030b0000)
#define CPY_3_12_FEATURES (PY_VERSION_HEX >= 0x030c0000)
#define CPY_3_14_FEATURES (PY_VERSION_HEX >= 0x030e0000)
#if CPY_3_12_FEATURES
// Same as macros in CPython internal/pycore_long.h, but with a CPY_ prefix
#define CPY_NON_SIZE_BITS 3
#define CPY_SIGN_ZERO 1
#define CPY_SIGN_NEGATIVE 2
#define CPY_SIGN_MASK 3
#define CPY_LONG_DIGIT(o, n) ((o)->long_value.ob_digit[n])
// Only available on Python 3.12 and later
#define CPY_LONG_TAG(o) ((o)->long_value.lv_tag)
#define CPY_LONG_IS_NEGATIVE(o) (((o)->long_value.lv_tag & CPY_SIGN_MASK) == CPY_SIGN_NEGATIVE)
// Only available on Python 3.12 and later
#define CPY_LONG_SIZE(o) ((o)->long_value.lv_tag >> CPY_NON_SIZE_BITS)
// Number of digits; negative for negative ints
#define CPY_LONG_SIZE_SIGNED(o) (CPY_LONG_IS_NEGATIVE(o) ? -CPY_LONG_SIZE(o) : CPY_LONG_SIZE(o))
// Number of digits, assuming int is non-negative
#define CPY_LONG_SIZE_UNSIGNED(o) CPY_LONG_SIZE(o)
#else
#define CPY_LONG_DIGIT(o, n) ((o)->ob_digit[n])
#define CPY_LONG_IS_NEGATIVE(o) (((o)->ob_base.ob_size < 0)
#define CPY_LONG_SIZE_SIGNED(o) ((o)->ob_base.ob_size)
#define CPY_LONG_SIZE_UNSIGNED(o) ((o)->ob_base.ob_size)
#endif
// Are we targeting Python 3.13 or newer?
#define CPY_3_13_FEATURES (PY_VERSION_HEX >= 0x030d0000)
// Are we targeting Python 3.14 or newer?
#define CPY_3_14_FEATURES (PY_VERSION_HEX >= 0x030e0000)
#endif
|
