1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
from io import StringIO
from token import NAME, NUMBER, NEWLINE, ENDMARKER
from tokenize import generate_tokens
from story5.tokenizer import Tokenizer
from story5.parser import Parser
from story5.grammar import Alt, GrammarParser, Rule
def test_grammar():
program = ("stmt: asmt | expr\n"
"asmt: NAME '=' expr\n"
"expr: NAME\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [Rule('stmt', [Alt(['asmt']), Alt(['expr'])]),
Rule('asmt', [Alt(['NAME', "'='", 'expr'])]),
Rule('expr', [Alt(['NAME'])])]
def test_failure():
program = ("stmt: asmt | expr\n"
"asmt: NAME '=' expr 42\n"
"expr: NAME\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules is None
def test_action():
program = "start: NAME { foo + bar } | NUMBER { -baz }\n"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [Rule("start", [Alt(["NAME"], "foo + bar"),
Alt(["NUMBER"], "- baz")])]
assert rules != [Rule("start", [Alt(["NAME"], "foo + bar"),
Alt(["NUMBER"], "baz")])]
def test_action_repr_str():
alt = Alt(["one", "two"])
assert repr(alt) == "Alt(['one', 'two'])"
assert str(alt) == "one two"
alt = Alt(["one", "two"], "foo + bar")
assert repr(alt) == "Alt(['one', 'two'], 'foo + bar')"
assert str(alt) == "one two { foo + bar }"
def test_indents():
program = ("stmt: foo | bar\n"
" | baz\n"
" | booh | bah\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [Rule('stmt',
[Alt(['foo']), Alt(['bar']),
Alt(['baz']),
Alt(['booh']), Alt(['bah'])])]
def test_indents2():
program = ("stmt:\n"
" | foo | bar\n"
" | baz\n"
" | booh | bah\n"
"foo: bar\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [Rule('stmt',
[Alt(['foo']), Alt(['bar']),
Alt(['baz']),
Alt(['booh']), Alt(['bah'])]),
Rule('foo', [Alt(['bar'])])]
|
