1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
|
"""JSONPath tokenization."""
from __future__ import annotations
import re
from functools import partial
from typing import TYPE_CHECKING
from typing import Iterator
from typing import Pattern
from .exceptions import JSONPathSyntaxError
from .token import TOKEN_AND
from .token import TOKEN_COLON
from .token import TOKEN_COMMA
from .token import TOKEN_CONTAINS
from .token import TOKEN_DDOT
from .token import TOKEN_DOT
from .token import TOKEN_DOT_KEY_PROPERTY
from .token import TOKEN_DOT_PROPERTY
from .token import TOKEN_DOUBLE_QUOTE_STRING
from .token import TOKEN_EQ
from .token import TOKEN_ERROR
from .token import TOKEN_FALSE
from .token import TOKEN_FILTER
from .token import TOKEN_FILTER_CONTEXT
from .token import TOKEN_FLOAT
from .token import TOKEN_FUNCTION
from .token import TOKEN_GE
from .token import TOKEN_GT
from .token import TOKEN_IN
from .token import TOKEN_INT
from .token import TOKEN_INTERSECTION
from .token import TOKEN_KEY
from .token import TOKEN_KEY_NAME
from .token import TOKEN_KEYS
from .token import TOKEN_KEYS_FILTER
from .token import TOKEN_LBRACKET
from .token import TOKEN_LE
from .token import TOKEN_LG
from .token import TOKEN_LPAREN
from .token import TOKEN_LT
from .token import TOKEN_MISSING
from .token import TOKEN_NAME
from .token import TOKEN_NE
from .token import TOKEN_NIL
from .token import TOKEN_NONE
from .token import TOKEN_NOT
from .token import TOKEN_NULL
from .token import TOKEN_OR
from .token import TOKEN_PSEUDO_ROOT
from .token import TOKEN_RBRACKET
from .token import TOKEN_RE
from .token import TOKEN_RE_FLAGS
from .token import TOKEN_RE_PATTERN
from .token import TOKEN_ROOT
from .token import TOKEN_RPAREN
from .token import TOKEN_SELF
from .token import TOKEN_SINGLE_QUOTE_STRING
from .token import TOKEN_TRUE
from .token import TOKEN_UNDEFINED
from .token import TOKEN_UNION
from .token import TOKEN_WHITESPACE
from .token import TOKEN_WILD
from .token import Token
if TYPE_CHECKING:
from . import JSONPathEnvironment
class Lexer:
"""Tokenize a JSONPath string.
Some customization can be achieved by subclassing _Lexer_ and setting
class attributes. Then setting `lexer_class` on a `JSONPathEnvironment`.
Attributes:
key_pattern: The regular expression pattern used to match mapping
keys/properties.
logical_not_pattern: The regular expression pattern used to match
logical negation tokens. By default, `not` and `!` are
equivalent.
logical_and_pattern: The regular expression pattern used to match
logical _and_ tokens. By default, `and` and `&&` are equivalent.
logical_or_pattern: The regular expression pattern used to match
logical _or_ tokens. By default, `or` and `||` are equivalent.
"""
key_pattern = r"[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*"
# ! or `not`
logical_not_pattern = r"(?:not\b)|!"
# && or `and`
logical_and_pattern = r"&&|(?:and\b)"
# || or `or`
logical_or_pattern = r"\|\||(?:or\b)"
def __init__(self, *, env: JSONPathEnvironment) -> None:
self.env = env
self.double_quote_pattern = r'"(?P<G_DQUOTE>(?:(?!(?<!\\)").)*)"'
self.single_quote_pattern = r"'(?P<G_SQUOTE>(?:(?!(?<!\\)').)*)'"
# .thing
self.dot_property_pattern = rf"(?P<G_DOT>\.)(?P<G_PROP>{self.key_pattern})"
# .~thing
self.dot_key_pattern = (
r"(?P<G_DOT_KEY>\.)"
rf"(?P<G_KEY>{re.escape(env.keys_selector_token)})"
rf"(?P<G_PROP_KEY>{self.key_pattern})"
)
# /pattern/ or /pattern/flags
self.re_pattern = r"/(?P<G_RE>(?:(?!(?<!\\)/).)*)/(?P<G_RE_FLAGS>[aims]*)"
# func(
self.function_pattern = r"(?P<G_FUNC>[a-z][a-z_0-9]+)(?P<G_FUNC_PAREN>\()"
self.rules = self.compile_strict_rules() if env.strict else self.compile_rules()
def compile_rules(self) -> Pattern[str]:
"""Prepare regular expression rules."""
env_tokens = [
(TOKEN_ROOT, self.env.root_token),
(TOKEN_PSEUDO_ROOT, self.env.pseudo_root_token),
(TOKEN_SELF, self.env.self_token),
(TOKEN_KEY, self.env.key_token),
(TOKEN_UNION, self.env.union_token),
(TOKEN_INTERSECTION, self.env.intersection_token),
(TOKEN_FILTER_CONTEXT, self.env.filter_context_token),
(TOKEN_KEYS, self.env.keys_selector_token),
(TOKEN_KEYS_FILTER, self.env.keys_filter_token),
]
rules = [
(TOKEN_DOUBLE_QUOTE_STRING, self.double_quote_pattern),
(TOKEN_SINGLE_QUOTE_STRING, self.single_quote_pattern),
(TOKEN_RE_PATTERN, self.re_pattern),
(TOKEN_DOT_KEY_PROPERTY, self.dot_key_pattern),
(TOKEN_DOT_PROPERTY, self.dot_property_pattern),
(
TOKEN_FLOAT,
r"(:?-?[0-9]+\.[0-9]+(?:[eE][+-]?[0-9]+)?)|(-?[0-9]+[eE]-[0-9]+)",
),
(TOKEN_INT, r"-?[0-9]+(?:[eE]\+?[0-9]+)?"),
(TOKEN_DDOT, r"\.\."),
(TOKEN_DOT, r"\."),
(TOKEN_AND, self.logical_and_pattern),
(TOKEN_OR, self.logical_or_pattern),
*[
(token, re.escape(pattern))
for token, pattern in sorted(
env_tokens, key=lambda x: len(x[1]), reverse=True
)
if pattern
],
(TOKEN_WILD, r"\*"),
(TOKEN_FILTER, r"\?"),
(TOKEN_IN, r"in\b"),
(TOKEN_TRUE, r"[Tt]rue\b"),
(TOKEN_FALSE, r"[Ff]alse\b"),
(TOKEN_NIL, r"[Nn]il\b"),
(TOKEN_NULL, r"[Nn]ull\b"),
(TOKEN_NONE, r"[Nn]one\b"),
(TOKEN_CONTAINS, r"contains\b"),
(TOKEN_UNDEFINED, r"undefined\b"),
(TOKEN_MISSING, r"missing\b"),
(TOKEN_LBRACKET, r"\["),
(TOKEN_RBRACKET, r"]"),
(TOKEN_COMMA, r","),
(TOKEN_COLON, r":"),
(TOKEN_EQ, r"=="),
(TOKEN_NE, r"!="),
(TOKEN_LG, r"<>"),
(TOKEN_LE, r"<="),
(TOKEN_GE, r">="),
(TOKEN_RE, r"=~"),
(TOKEN_LT, r"<"),
(TOKEN_GT, r">"),
(TOKEN_NOT, self.logical_not_pattern), # Must go after "!="
(TOKEN_FUNCTION, self.function_pattern),
(TOKEN_NAME, self.key_pattern), # Must go after reserved words
(TOKEN_LPAREN, r"\("),
(TOKEN_RPAREN, r"\)"),
(TOKEN_WHITESPACE, r"[ \n\t\r]+"),
(TOKEN_ERROR, r"."),
]
return re.compile(
"|".join(f"(?P<{token}>{pattern})" for token, pattern in rules),
re.DOTALL,
)
def compile_strict_rules(self) -> Pattern[str]:
"""Prepare regular expression rules in strict mode."""
env_tokens = [
(TOKEN_ROOT, self.env.root_token),
(TOKEN_SELF, self.env.self_token),
]
rules = [
(TOKEN_DOUBLE_QUOTE_STRING, self.double_quote_pattern),
(TOKEN_SINGLE_QUOTE_STRING, self.single_quote_pattern),
(TOKEN_DOT_PROPERTY, self.dot_property_pattern),
(
TOKEN_FLOAT,
r"(:?-?[0-9]+\.[0-9]+(?:[eE][+-]?[0-9]+)?)|(-?[0-9]+[eE]-[0-9]+)",
),
(TOKEN_INT, r"-?[0-9]+(?:[eE]\+?[0-9]+)?"),
(TOKEN_DDOT, r"\.\."),
(TOKEN_DOT, r"\."),
(TOKEN_AND, r"&&"),
(TOKEN_OR, r"\|\|"),
*[
(token, re.escape(pattern))
for token, pattern in sorted(
env_tokens, key=lambda x: len(x[1]), reverse=True
)
if pattern
],
(TOKEN_WILD, r"\*"),
(TOKEN_FILTER, r"\?"),
(TOKEN_TRUE, r"true\b"),
(TOKEN_FALSE, r"false\b"),
(TOKEN_NULL, r"null\b"),
(TOKEN_LBRACKET, r"\["),
(TOKEN_RBRACKET, r"]"),
(TOKEN_COMMA, r","),
(TOKEN_COLON, r":"),
(TOKEN_EQ, r"=="),
(TOKEN_NE, r"!="),
(TOKEN_LG, r"<>"),
(TOKEN_LE, r"<="),
(TOKEN_GE, r">="),
(TOKEN_LT, r"<"),
(TOKEN_GT, r">"),
(TOKEN_NOT, r"!"), # Must go after "!="
(TOKEN_FUNCTION, self.function_pattern),
(TOKEN_NAME, self.key_pattern), # Must go after reserved words
(TOKEN_LPAREN, r"\("),
(TOKEN_RPAREN, r"\)"),
(TOKEN_WHITESPACE, r"[ \n\t\r]+"),
(TOKEN_ERROR, r"."),
]
return re.compile(
"|".join(f"(?P<{token}>{pattern})" for token, pattern in rules),
re.DOTALL,
)
def tokenize(self, path: str) -> Iterator[Token]: # noqa PLR0912
"""Generate a sequence of tokens from a JSONPath string."""
_token = partial(Token, path=path)
for match in self.rules.finditer(path):
kind = match.lastgroup
assert kind is not None
if kind == TOKEN_DOT_PROPERTY:
yield _token(
kind=TOKEN_DOT,
value=match.group("G_DOT"),
index=match.start("G_DOT"),
)
yield _token(
kind=TOKEN_NAME,
value=match.group("G_PROP"),
index=match.start("G_PROP"),
)
elif kind == TOKEN_DOT_KEY_PROPERTY:
yield _token(
kind=TOKEN_DOT,
value=match.group("G_DOT_KEY"),
index=match.start("G_DOT_KEY"),
)
yield _token(
kind=TOKEN_KEY_NAME,
value=match.group("G_PROP_KEY"),
index=match.start("G_PROP_KEY"),
)
elif kind == TOKEN_DOUBLE_QUOTE_STRING:
yield _token(
kind=TOKEN_DOUBLE_QUOTE_STRING,
value=match.group("G_DQUOTE"),
index=match.start("G_DQUOTE"),
)
elif kind == TOKEN_SINGLE_QUOTE_STRING:
yield _token(
kind=TOKEN_SINGLE_QUOTE_STRING,
value=match.group("G_SQUOTE"),
index=match.start("G_SQUOTE"),
)
elif kind == TOKEN_RE_PATTERN:
yield _token(
kind=TOKEN_RE_PATTERN,
value=match.group("G_RE"),
index=match.start("G_RE"),
)
yield _token(
TOKEN_RE_FLAGS,
value=match.group("G_RE_FLAGS"),
index=match.start("G_RE_FLAGS"),
)
elif kind in (TOKEN_NONE, TOKEN_NULL):
yield _token(
kind=TOKEN_NIL,
value=match.group(),
index=match.start(),
)
elif kind == TOKEN_FUNCTION:
yield _token(
kind=TOKEN_FUNCTION,
value=match.group("G_FUNC"),
index=match.start("G_FUNC"),
)
yield _token(
kind=TOKEN_LPAREN,
value=match.group("G_FUNC_PAREN"),
index=match.start("G_FUNC_PAREN"),
)
elif kind == TOKEN_ERROR:
raise JSONPathSyntaxError(
f"unexpected token {match.group()!r}",
token=_token(
TOKEN_ERROR,
value=match.group(),
index=match.start(),
),
)
else:
yield _token(
kind=kind,
value=match.group(),
index=match.start(),
)
|
