1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
from __future__ import absolute_import
import logging
from unittest import TestCase, main, skipIf
from lark import Lark, Tree, Transformer, UnexpectedInput
from lark.lexer import Lexer, Token
import lark.lark as lark_module
from io import BytesIO
try:
import regex
except ImportError:
regex = None
class MockFile(BytesIO):
def close(self):
pass
def __enter__(self):
return self
def __exit__(self, *args):
pass
class MockFS:
def __init__(self):
self.files = {}
def open(self, name, mode="r", **kwargs):
if name not in self.files:
if "r" in mode:
# If we are reading from a file, it should already exist
raise FileNotFoundError(name)
f = self.files[name] = MockFile()
else:
f = self.files[name]
f.seek(0)
return f
def exists(self, name):
return name in self.files
class CustomLexer(Lexer):
def __init__(self, lexer_conf):
pass
def lex(self, data):
for obj in data:
yield Token('A', obj)
class InlineTestT(Transformer):
def add(self, children):
return sum(children if isinstance(children, list) else children.children)
def NUM(self, token):
return int(token)
def __reduce__(self):
raise TypeError("This Transformer should not be pickled.")
def append_zero(t):
return t.update(value=t.value + '0')
class TestCache(TestCase):
g = '''start: "a"'''
def setUp(self):
self.fs = lark_module.FS
self.mock_fs = MockFS()
lark_module.FS = self.mock_fs
def tearDown(self):
self.mock_fs.files = {}
lark_module.FS = self.fs
def test_simple(self):
fn = "bla"
Lark(self.g, parser='lalr', cache=fn)
assert fn in self.mock_fs.files
parser = Lark(self.g, parser='lalr', cache=fn)
assert parser.parse('a') == Tree('start', [])
def test_automatic_naming(self):
assert len(self.mock_fs.files) == 0
Lark(self.g, parser='lalr', cache=True)
assert len(self.mock_fs.files) == 1
parser = Lark(self.g, parser='lalr', cache=True)
assert parser.parse('a') == Tree('start', [])
parser = Lark(self.g + ' "b"', parser='lalr', cache=True)
assert len(self.mock_fs.files) == 2
assert parser.parse('ab') == Tree('start', [])
parser = Lark(self.g, parser='lalr', cache=True)
assert parser.parse('a') == Tree('start', [])
def test_custom_lexer(self):
parser = Lark(self.g, parser='lalr', lexer=CustomLexer, cache=True)
parser = Lark(self.g, parser='lalr', lexer=CustomLexer, cache=True)
assert len(self.mock_fs.files) == 1
assert parser.parse('a') == Tree('start', [])
def test_options(self):
# Test options persistence
Lark(self.g, parser="lalr", debug=True, cache=True)
parser = Lark(self.g, parser="lalr", debug=True, cache=True)
assert parser.options.options['debug']
def test_inline(self):
# Test inline transformer (tree-less) & lexer_callbacks
# Note: the Transformer should not be saved to the file,
# and is made unpickable to check for that
g = r"""
start: add+
add: NUM "+" NUM
NUM: /\d+/
%ignore " "
"""
text = "1+2 3+4"
expected = Tree('start', [30, 70])
parser = Lark(g, parser='lalr', transformer=InlineTestT(), cache=True, lexer_callbacks={'NUM': append_zero})
res0 = parser.parse(text)
parser = Lark(g, parser='lalr', transformer=InlineTestT(), cache=True, lexer_callbacks={'NUM': append_zero})
assert len(self.mock_fs.files) == 1
res1 = parser.parse(text)
res2 = InlineTestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text))
assert res0 == res1 == res2 == expected
def test_imports(self):
g = """
%import .grammars.ab (startab, expr)
"""
parser = Lark(g, parser='lalr', start='startab', cache=True, source_path=__file__)
assert len(self.mock_fs.files) == 1
parser = Lark(g, parser='lalr', start='startab', cache=True, source_path=__file__)
assert len(self.mock_fs.files) == 1
res = parser.parse("ab")
self.assertEqual(res, Tree('startab', [Tree('expr', ['a', 'b'])]))
@skipIf(regex is None, "'regex' lib not installed")
def test_recursive_pattern(self):
g = """
start: recursive+
recursive: /\w{3}\d{3}(?R)?/
"""
assert len(self.mock_fs.files) == 0
Lark(g, parser="lalr", regex=True, cache=True)
assert len(self.mock_fs.files) == 1
with self.assertLogs("lark", level="ERROR") as cm:
Lark(g, parser='lalr', regex=True, cache=True)
assert len(self.mock_fs.files) == 1
# need to add an error log, because 'self.assertNoLogs' was added in Python 3.10
logging.getLogger('lark').error("dummy message")
# should only have the dummy log
self.assertCountEqual(cm.output, ["ERROR:lark:dummy message"])
def test_error_message(self):
# Checks that error message generation works
# This is especially important since sometimes the `str` method fails with
# the mysterious "<unprintable UnexpectedCharacters object>" or similar
g = r"""
start: add+
add: /\d+/ "+" /\d+/
%ignore " "
"""
texts = ("1+", "+1", "", "1 1+1")
parser1 = Lark(g, parser='lalr', cache=True)
parser2 = Lark(g, parser='lalr', cache=True)
assert len(self.mock_fs.files) == 1
for text in texts:
with self.assertRaises((UnexpectedInput)) as cm1:
parser1.parse(text)
with self.assertRaises((UnexpectedInput)) as cm2:
parser2.parse(text)
self.assertEqual(str(cm1.exception), str(cm2.exception))
if __name__ == '__main__':
main()
|
