1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
|
# -*- coding: utf-8 -*-
import py
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.pyparser.pygram import syms, tokens
from pypy.interpreter.pyparser.error import SyntaxError, IndentationError, TabError
from pypy.interpreter.astcompiler import consts
class TestPythonParser:
spaceconfig = {}
def setup_class(self):
self.parser = pyparse.PythonParser(self.space)
def parse(self, source, mode="exec", info=None):
if info is None:
info = pyparse.CompileInfo("<test>", mode)
return self.parser.parse_source(source, info)
def test_with_and_as(self):
py.test.raises(SyntaxError, self.parse, "with = 23")
py.test.raises(SyntaxError, self.parse, "as = 2")
def test_dont_imply_dedent(self):
info = pyparse.CompileInfo("<test>", "single",
consts.PyCF_DONT_IMPLY_DEDENT)
self.parse('if 1:\n x\n', info=info)
self.parse('x = 5 ', info=info)
def test_clear_state(self):
assert self.parser.root is None
tree = self.parse("name = 32")
assert self.parser.root is None
def test_encoding_pep3120(self):
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parse("""foo = '日本'""", info=info)
assert info.encoding == 'utf-8'
def test_encoding(self):
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parse("""# coding: latin-1
stuff = "nothing"
""", info=info)
assert tree.type == syms.file_input
assert info.encoding == "iso-8859-1"
sentence = u"u'Die Männer ärgern sich!'"
input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7")
tree = self.parse(input, info=info)
assert info.encoding == "utf-7"
input = "# coding: iso-8859-15\nx"
self.parse(input, info=info)
assert info.encoding == "iso-8859-15"
input = "\xEF\xBB\xBF# coding: utf-8\nx"
self.parse(input, info=info)
assert info.encoding == "utf-8"
input = "\xEF\xBB\xBF# coding: latin-1\nx"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "UTF-8 BOM with latin-1 coding cookie"
input = "# coding: not-here"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "Unknown encoding: not-here"
input = u"# coding: ascii\n\xe2".encode('utf-8')
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == ("'ascii' codec can't decode byte 0xc3 "
"in position 16: ordinal not in range(128)")
def test_unicode_identifier(self):
tree = self.parse("a日本 = 32")
tree = self.parse("日本 = 32")
def test_syntax_error(self):
parse = self.parse
exc = py.test.raises(SyntaxError, parse, "name another for").value
assert exc.msg == "invalid syntax"
assert exc.lineno == 1
assert exc.offset == 6
assert exc.text.startswith("name another for")
exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value
assert exc.msg == "end of line (EOL) while scanning string literal"
assert exc.lineno == 1
assert exc.offset == 5
exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value
assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal"
assert exc.lineno == 1
assert exc.offset == 5
assert exc.lastlineno == 3
for input in ("())", "(()", "((", "))"):
py.test.raises(SyntaxError, parse, input)
exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value
assert exc.msg == "parenthesis is never closed"
assert exc.lineno == 1
assert exc.offset == 5
assert exc.lastlineno == 5
exc = py.test.raises(SyntaxError, parse, "abc)").value
assert exc.msg == "unmatched ')'"
assert exc.lineno == 1
assert exc.offset == 4
def test_is(self):
self.parse("x is y")
self.parse("x is not y")
def test_indentation_error(self):
parse = self.parse
input = """
def f():
pass"""
exc = py.test.raises(IndentationError, parse, input).value
assert exc.msg == "expected an indented block"
assert exc.lineno == 3
assert exc.text.startswith("pass")
assert exc.offset == 1
input = "hi\n indented"
exc = py.test.raises(IndentationError, parse, input).value
assert exc.msg == "unexpected indent"
input = "def f():\n pass\n next_stmt"
exc = py.test.raises(IndentationError, parse, input).value
assert exc.msg == "unindent does not match any outer indentation level"
assert exc.lineno == 3
assert exc.offset == 3
def test_taberror(self):
src = """
if 1:
pass
\tpass
"""
exc = py.test.raises(TabError, "self.parse(src)").value
assert exc.msg == "inconsistent use of tabs and spaces in indentation"
assert exc.lineno == 4
assert exc.offset == 5
assert exc.text == " \tpass\n"
def test_mac_newline(self):
self.parse("this_is\ra_mac\rfile")
def test_mode(self):
assert self.parse("x = 43*54").type == syms.file_input
tree = self.parse("43**54", "eval")
assert tree.type == syms.eval_input
py.test.raises(SyntaxError, self.parse, "x = 54", "eval")
tree = self.parse("x = 43", "single")
assert tree.type == syms.single_input
def test_multiline_string(self):
self.parse("''' \n '''")
self.parse("r''' \n '''")
def test_bytes_literal(self):
self.parse('b" "')
self.parse('br" "')
self.parse('b""" """')
self.parse("b''' '''")
self.parse("br'\\\n'")
py.test.raises(SyntaxError, self.parse, "b'a\\n")
def test_new_octal_literal(self):
self.parse('0o777')
py.test.raises(SyntaxError, self.parse, '0o777L')
py.test.raises(SyntaxError, self.parse, "0o778")
def test_new_binary_literal(self):
self.parse('0b1101')
py.test.raises(SyntaxError, self.parse, '0b0l')
py.test.raises(SyntaxError, self.parse, "0b112")
def test_print_function(self):
self.parse("from __future__ import print_function\nx = print\n")
def test_universal_newlines(self):
fmt = 'stuff = """hello%sworld"""'
expected_tree = self.parse(fmt % '\n')
for linefeed in ["\r\n","\r"]:
tree = self.parse(fmt % linefeed)
assert expected_tree == tree
def test_revdb_dollar_num(self):
assert not self.space.config.translation.reverse_debugger
py.test.raises(SyntaxError, self.parse, '$0')
py.test.raises(SyntaxError, self.parse, '$0 + 5')
py.test.raises(SyntaxError, self.parse,
"from __future__ import print_function\nx = ($0, print)")
def test_py3k_reject_old_binary_literal(self):
py.test.raises(SyntaxError, self.parse, '0777')
def test_py3k_extended_unpacking(self):
self.parse('a, *rest, b = 1, 2, 3, 4, 5')
self.parse('(a, *rest, b) = 1, 2, 3, 4, 5')
def test_u_triple_quote(self):
self.parse('u""""""')
self.parse('U""""""')
self.parse("u''''''")
self.parse("U''''''")
def test_bad_single_statement(self):
py.test.raises(SyntaxError, self.parse, '1\n2', "single")
py.test.raises(SyntaxError, self.parse, 'a = 13\nb = 187', "single")
py.test.raises(SyntaxError, self.parse, 'del x\ndel y', "single")
py.test.raises(SyntaxError, self.parse, 'f()\ng()', "single")
py.test.raises(SyntaxError, self.parse, 'f()\n# blah\nblah()', "single")
py.test.raises(SyntaxError, self.parse, 'f()\nxy # blah\nblah()', "single")
py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single")
def test_unpack(self):
self.parse('[*{2}, 3, *[4]]')
self.parse('{*{2}, 3, *[4]}')
self.parse('{**{}, 3:4, **{5:6, 7:8}}')
self.parse('f(2, *a, *b, **b, **c, **d)')
def test_async_await(self):
self.parse("async def coro(): await func")
py.test.raises(SyntaxError, self.parse, 'await x')
#Test as var and func name
self.parse("async = 1")
self.parse("await = 1")
self.parse("def async(): pass")
#async for
self.parse("""async def foo():
async for a in b:
pass""")
py.test.raises(SyntaxError, self.parse, 'def foo(): async for a in b: pass')
#async with
self.parse("""async def foo():
async with a:
pass""")
py.test.raises(SyntaxError, self.parse, 'def foo(): async with a: pass')
class TestPythonParserWithSpace:
def setup_class(self):
self.parser = pyparse.PythonParser(self.space)
def parse(self, source, mode="exec", info=None):
if info is None:
info = pyparse.CompileInfo("<test>", mode)
return self.parser.parse_source(source, info)
def test_encoding(self):
info = pyparse.CompileInfo("<test>", "exec")
tree = self.parse("""# coding: latin-1
stuff = "nothing"
""", info=info)
assert tree.type == syms.file_input
assert info.encoding == "iso-8859-1"
sentence = u"'Die Männer ärgen sich!'"
input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7")
tree = self.parse(input, info=info)
assert info.encoding == "utf-7"
input = "# coding: iso-8859-15\nx"
self.parse(input, info=info)
assert info.encoding == "iso-8859-15"
input = "\xEF\xBB\xBF# coding: utf-8\nx"
self.parse(input, info=info)
assert info.encoding == "utf-8"
#
info.flags |= consts.PyCF_SOURCE_IS_UTF8
input = "#\nx"
info.encoding = None
self.parse(input, info=info)
assert info.encoding == "utf-8"
input = "# coding: latin1\nquux"
self.parse(input, info=info)
assert info.encoding == "latin1"
info.flags |= consts.PyCF_IGNORE_COOKIE
self.parse(input, info=info)
assert info.encoding == "utf-8"
info.flags &= ~(consts.PyCF_SOURCE_IS_UTF8 | consts.PyCF_IGNORE_COOKIE)
#
input = "\xEF\xBB\xBF# coding: latin-1\nx"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "UTF-8 BOM with latin-1 coding cookie"
input = "# coding: not-here"
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == "Unknown encoding: not-here"
input = u"# coding: ascii\n\xe2".encode('utf-8')
exc = py.test.raises(SyntaxError, self.parse, input).value
assert exc.msg == ("'ascii' codec can't decode byte 0xc3 "
"in position 16: ordinal not in range(128)")
def test_error_forgotten_chars(self):
info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4")
assert "(expected ':')" in info.value.msg
info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i")
assert "(expected ':')" in info.value.msg
info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1")
assert "(expected '(')" in info.value.msg
class TestPythonParserRevDB(TestPythonParser):
spaceconfig = {"translation.reverse_debugger": True}
def test_revdb_dollar_num(self):
self.parse('$0')
self.parse('$5')
self.parse('$42')
self.parse('2+$42.attrname')
self.parse("from __future__ import print_function\nx = ($0, print)")
py.test.raises(SyntaxError, self.parse, '$')
py.test.raises(SyntaxError, self.parse, '$a')
py.test.raises(SyntaxError, self.parse, '$.5')
|
