1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
{
-- alex scanner for use with uulib
-- compile with alex -o Scanner.hs -g Scanner.x
module Scanner(tokenize) where
import UU.Scanner
import Data.Word (Word8)
}
$litChar = [^[\" \\]]
$identChar = [a-zA-Z0-9\'_]
tokens :-
$white+ ; -- whitespace
"--".* ; -- comment
\" ($litChar | \\ \\ | \\ \" )* \" { valueToken TkString } -- string
[0-9]+ { valueToken TkInteger16 } -- int
( let | in ) { reserved } -- reserved keywords
[\(\)\=] { reserved } -- reserved symbols
[\+\*] { reserved } -- operators
[a-zA-Z] $identChar* { valueToken TkVarid } -- identifier
{
-- boilerplate code needed for Alex
type AlexInput = (Pos, String)
alexInputPrevChar :: AlexInput -> Char
alexInputPrevChar = error "alexInputPrevChar: there is no need to go back in the input."
-- In Alex3 alexGetByte must be defined.
alexGetByte :: AlexInput -> Maybe (Word8, AlexInput)
alexGetByte (_, []) = Nothing
alexGetByte (p, (c:cs))
= let p' = adv p c
in Just ((fromIntegral $ ord c), (p', cs))
alexGetChar :: AlexInput -> Maybe (Char, AlexInput)
alexGetChar (_, []) = Nothing
alexGetChar (p, (c:cs))
= let p' = adv p c
in Just (c, (p', cs))
-- use the Alex scanner to generate a list of tokens for the uulib token parsers
tokenize :: String -> String -> [Token]
tokenize filename str
= go (initpos, str)
where
initpos = Pos 1 1 filename
go inp@(pos, cs)
= case alexScan inp 0 of
AlexEOF -> []
AlexError inp' -> valueToken TkError [head cs] pos : go inp'
AlexSkip inp' _ -> go inp'
AlexToken inp' len act -> act (take len cs) pos : go inp'
}
|
