1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
# Generated by re2py
# re2py $INPUT -o $OUTPUT --input-encoding utf8
# This example supports multiple input encodings: UTF-8 and UTF-32.
# Both lexers are generated from the same rules block, and the use
# blocks add only encoding-specific configurations.
def lex_utf8(yyinput):
yycursor = 0
yystate = 0
while True:
match yystate:
case 0:
yych = yyinput[yycursor]
yycursor += 1
if yych == 0xE2:
yystate = 2
continue
yystate = 1
continue
case 1:
return None
case 2:
yymarker = yycursor
yych = yyinput[yycursor]
if yych != 0x88:
yystate = 1
continue
yycursor += 1
yych = yyinput[yycursor]
if yych == 0x80:
yycursor += 1
yystate = 4
continue
yystate = 3
continue
case 3:
yycursor = yymarker
yystate = 1
continue
case 4:
yych = yyinput[yycursor]
if yych != 0x78:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0x20:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0xE2:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0x88:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0x83:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0x79:
yystate = 3
continue
yycursor += 1
return yycursor
case _:
raise "internal lexer error"
def lex_utf32(yyinput):
yycursor = 0
yystate = 0
while True:
match yystate:
case 0:
yych = yyinput[yycursor]
yycursor += 1
if yych == 0x00002200:
yystate = 2
continue
yystate = 1
continue
case 1:
return None
case 2:
yymarker = yycursor
yych = yyinput[yycursor]
if yych != 0x00000078:
yystate = 1
continue
yycursor += 1
yych = yyinput[yycursor]
if yych == 0x00000020:
yycursor += 1
yystate = 4
continue
yystate = 3
continue
case 3:
yycursor = yymarker
yystate = 1
continue
case 4:
yych = yyinput[yycursor]
if yych != 0x00002203:
yystate = 3
continue
yycursor += 1
yych = yyinput[yycursor]
if yych != 0x00000079:
yystate = 3
continue
yycursor += 1
return yycursor
case _:
raise "internal lexer error"
s8 = [0xe2, 0x88, 0x80, 0x78, 0x20, 0xe2, 0x88, 0x83, 0x79]
assert lex_utf8(s8) == len(s8)
s32 = [0x2200, 0x78, 0x20, 0x2203, 0x79]
assert lex_utf32(s32) == len(s32)
|
