1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
asserteq = require('pl.test').asserteq
T = require 'pl.test' . tuple
lexer = require 'pl.lexer'
seq = require 'pl.seq'
List = require ('pl.List')
copy2 = seq.copy2
s = '20 = hello'
asserteq(copy2(lexer.scan (s,nil,{space=false},{number=false})),
{{'number','20'},{'space',' '},{'=','='},{'space',' '},{'iden','hello'}})
asserteq(copy2(lexer.scan (s,nil,{space=true},{number=true})),
{{'number',20},{'=','='},{'iden','hello'}})
asserteq(copy2(lexer.lua('test(20 and a > b)',{space=true})),
{{'iden','test'},{'(','('},{'number',20},{'keyword','and'},{'iden','a'},
{'>','>'},{'iden','b'},{')',')'}} )
lines = [[
for k,v in pairs(t) do
if type(k) == 'number' then
print(v) -- array-like case
else
print(k,v)
end -- if
end
]]
ls = List()
for tp,val in lexer.lua(lines,{space=true,comments=true}) do
assert(tp ~= 'space' and tp ~= 'comment')
if tp == 'keyword' then ls:append(val) end
end
asserteq(ls,List{'for','in','do','if','then','else','end','end'})
tok = lexer.scan([[
'help' "help" "dolly you're fine" "a \"quote\" here"
]],nil,{space=true,string=true})
function t2() local t,v = tok(); return v end
asserteq(t2(),'help')
asserteq(t2(),'help')
asserteq(t2(),"dolly you're fine")
asserteq(t2(),"a \\\"quote\\\" here") --> NOT convinced this is correct!
tok = lexer.lua('10+2.3') ---> '+' is no longer considered part of the number!
asserteq(T(tok()),T('number',10))
asserteq(T(tok()),T('+','+'))
asserteq(T(tok()),T('number',2.3))
local txt = [==[
-- comment
--[[
block
comment
]][[
hello dammit
]][[hello]]
]==]
tok = lexer.lua(txt,{})
asserteq(tok(),'comment')
asserteq(tok(),'comment')
asserteq(tok(),'string')
asserteq(tok(),'string')
asserteq(tok(),'space')
txt = [[
// comment
/* a long
set of words */ // more
]]
tok = lexer.cpp(txt,{})
asserteq(tok(),'comment')
asserteq(tok(),'comment')
asserteq(tok(),'space')
asserteq(tok(),'comment')
local function teststring (s)
local tok = lexer.lua(s,{},{string=false})
local t,v = tok()
asserteq(t,"string")
asserteq(v,s)
end
teststring [["hello\\"]]
teststring [["hello\"dolly"]]
teststring [['hello\'dolly']]
teststring [['']]
teststring [[""]]
|
