1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
local asserteq = require('pl.test').asserteq
local lexer = require 'pl.lexer'
local seq = require 'pl.seq'
local List = require('pl.List')
local open = require('pl.stringio').open
local copy2 = seq.copy2
local function test_scan(str, filter, options, expected_tokens, lang)
local matches
if lang then
matches, filter = filter, options
else
lang = 'scan'
end
asserteq(copy2(lexer[lang](str, matches, filter, options)), expected_tokens)
if lang == 'scan' then
asserteq(copy2(lexer[lang](open(str), matches, filter, options)), expected_tokens)
end
end
local s = '20 = hello'
test_scan(s, {space=false}, {number=false}, {
{'number', '20'}, {'space', ' '}, {'=', '='}, {'space', ' '}, {'iden', 'hello'}
})
test_scan(s, {space=true}, {number=true}, {
{'number', 20}, {'=', '='}, {'iden', 'hello'}
})
s = [[ 'help' "help" "dolly you're fine" "a \"quote\" here"]]
test_scan(s, nil, nil, {
{'string', 'help'}, {'string', 'help'},
{'string', "dolly you're fine"}, {'string', 'a \\\"quote\\\" here'} -- Escapes are preserved literally.
})
test_scan([[\abc\]], nil, nil, {
{'\\', '\\'}, {'iden', 'abc'}, {'\\', '\\'}
})
test_scan([["" ""]], nil, nil, {
{'string', ''}, {'string', ''}
})
test_scan([["abc" "def\\"]], nil, nil, {
{'string', 'abc'}, {'string', 'def\\\\'}
})
test_scan([["abc\\" "def"]], nil, nil, {
{'string', 'abc\\\\'}, {'string', 'def'}
})
test_scan([["abc\\\" "]], nil, nil, {
{'string', 'abc\\\\\\" '}
})
local function test_roundtrip(str)
test_scan(str, {}, {string=false}, {{'string', str}})
end
test_roundtrip [["hello\\"]]
test_roundtrip [["hello\"dolly"]]
test_roundtrip [['hello\'dolly']]
test_roundtrip [['']]
test_roundtrip [[""]]
test_scan('test(20 and a > b)', nil, nil, {
{'iden', 'test'}, {'(', '('}, {'number', 20}, {'keyword', 'and'},
{'iden', 'a'}, {'>', '>'}, {'iden', 'b'}, {')', ')'}
}, 'lua')
test_scan('10+2.3', nil, nil, {
{'number', 10}, {'+', '+'}, {'number', 2.3}
}, 'lua')
local txt = [==[
-- comment
--[[
block
comment
]][[
hello dammit
]][[hello]]
]==]
test_scan(txt, {}, nil, {
{'comment', '-- comment\n'},
{'comment', '--[[\nblock\ncomment\n]]'},
{'string', 'hello dammit\n'},
{'string', 'hello'},
{'space', '\n'}
}, 'lua')
local lines = [[
for k,v in pairs(t) do
if type(k) == 'number' then
print(v) -- array-like case
else
print(k,v)
end -- if
end
]]
local ls = List()
for tp,val in lexer.lua(lines,{space=true,comments=true}) do
assert(tp ~= 'space' and tp ~= 'comment')
if tp == 'keyword' then ls:append(val) end
end
asserteq(ls,List{'for','in','do','if','then','else','end','end'})
txt = [[
// comment
/* a long
set of words */ // more
]]
test_scan(txt, {}, nil, {
{'comment', '// comment\n'},
{'comment', '/* a long\nset of words */'},
{'space', ' '},
{'comment', '// more\n'}
}, 'cpp')
test_scan([['' "" " \\" '\'' "'"]], nil, nil, {
{'char', ''}, -- Char literals with no or more than one characters are not a lexing error.
{'string', ''},
{'string', ' \\\\'},
{'char', "\\'"},
{'string', "'"}
}, 'cpp')
local iter = lexer.lua([[
foo
bar
]])
asserteq(lexer.lineno(iter), 0)
iter()
asserteq(lexer.lineno(iter), 1)
asserteq(lexer.lineno(iter), 1)
iter()
asserteq(lexer.lineno(iter), 2)
iter()
asserteq(lexer.lineno(iter), 3)
iter()
iter()
asserteq(lexer.lineno(iter), 3)
do -- numbers without leading zero; ".123"
local s = 'hello = +.234'
test_scan(s, {space=true}, {number=true}, {
{'iden', 'hello'}, {'=', '='}, {'number', .234}
})
end
|
