1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
structure Tokens = Tokens
type pos = int
type svalue = Tokens.svalue
type ('a,'b) token = ('a,'b) Tokens.token
type lexresult = (svalue,pos) token
open Tokens
val lineNum = ref 0
val eof = fn () => EOF(!lineNum,!lineNum)
structure KeyWord : sig
val find : string ->
(int * int -> (svalue,int) token) option
end =
struct
val TableSize = 211
val HashFactor = 5
val hash = fn s =>
foldl (fn (c,v)=>(v*HashFactor+(ord c)) mod TableSize) 0 (explode s)
val HashTable = Array.array(TableSize,nil) :
(string * (int * int -> (svalue,int) token)) list Array.array
val add = fn (s,v) =>
let val i = hash s
in Array.update(HashTable,i,(s,v) :: (Array.sub(HashTable, i)))
end
val find = fn s =>
let val i = hash s
fun f ((key,v)::r) = if s=key then SOME v else f r
| f nil = NONE
in f (Array.sub(HashTable, i))
end
val _ =
(List.app add
[("and",YAND),
("array",YARRAY),
("begin",YBEGIN),
("case",YCASE),
("const",YCONST),
("div",YDIV),
("do",YDO),
("downto",YDOWNTO),
("else",YELSE),
("end",YEND),
("extern",YEXTERN),
("file",YFILE),
("for",YFOR),
("forward",YFORWARD),
("function",YFUNCTION),
("goto",YGOTO),
("hex",YHEX),
("if",YIF),
("in",YIN),
("label",YLABEL),
("mod",YMOD),
("nil",YNIL),
("not",YNOT),
("oct",YOCT),
("of",YOF),
("or",YOR),
("packed",YPACKED),
("procedure",YPROCEDURE),
("program",YPROG),
("record",YRECORD),
("repeat",YREPEAT),
("set",YSET),
("then",YTHEN),
("to",YTO),
("type",YTYPE),
("until",YUNTIL),
("var",YVAR),
("while",YWHILE),
("with",YWITH)
])
end
open KeyWord
%%
%header (functor PascalLexFun(structure Tokens : Pascal_TOKENS));
%s C B;
alpha=[A-Za-z];
digit=[0-9];
optsign=("+"|"-")?;
integer={digit}+;
frac="."{digit}+;
exp=(e|E){optsign}{digit}+;
octdigit=[0-7];
ws = [\ \t];
%%
<INITIAL>{ws}+ => (lex());
<INITIAL>\n+ => (lineNum := (!lineNum) + (String.size yytext); lex());
<INITIAL>{alpha}+ => (case find yytext of SOME v => v(!lineNum,!lineNum)
| _ => YID(!lineNum,!lineNum));
<INITIAL>{alpha}({alpha}|{digit})* => (YID(!lineNum,!lineNum));
<INITIAL>{optsign}{integer}({frac}{exp}?|{frac}?{exp}) => (YNUMB(!lineNum,!lineNum));
<INITIAL>{optsign}{integer} => (YINT(!lineNum,!lineNum));
<INITIAL>{octdigit}+(b|B) => (YBINT(!lineNum,!lineNum));
<INITIAL>"'"([^']|"''")*"'" => (YSTRING(!lineNum,!lineNum));
<INITIAL>"(*" => (YYBEGIN C; lex());
<INITIAL>".." => (YDOTDOT(!lineNum,!lineNum));
<INITIAL>"." => (YDOT(!lineNum,!lineNum));
<INITIAL>"(" => (YLPAR(!lineNum,!lineNum));
<INITIAL>")" => (YRPAR(!lineNum,!lineNum));
<INITIAL>";" => (YSEMI(!lineNum,!lineNum));
<INITIAL>"," => (YCOMMA(!lineNum,!lineNum));
<INITIAL>":" => (YCOLON(!lineNum,!lineNum));
<INITIAL>"^" => (YCARET(!lineNum,!lineNum));
<INITIAL>"[" => (YLBRA(!lineNum,!lineNum));
<INITIAL>"]" => (YRBRA(!lineNum,!lineNum));
<INITIAL>"~" => (YTILDE(!lineNum,!lineNum));
<INITIAL>"<" => (YLESS(!lineNum,!lineNum));
<INITIAL>"=" => (YEQUAL(!lineNum,!lineNum));
<INITIAL>">" => (YGREATER(!lineNum,!lineNum));
<INITIAL>"+" => (YPLUS(!lineNum,!lineNum));
<INITIAL>"-" => (YMINUS(!lineNum,!lineNum));
<INITIAL>"|" => (YBAR(!lineNum,!lineNum));
<INITIAL>"*" => (YSTAR(!lineNum,!lineNum));
<INITIAL>"/" => (YSLASH(!lineNum,!lineNum));
<INITIAL>"{" => (YYBEGIN B; lex());
<INITIAL>. => (YILLCH(!lineNum,!lineNum));
<C>\n+ => (lineNum := (!lineNum) + (String.size yytext); lex());
<C>[^()*\n]+ => (lex());
<C>"(*" => (lex());
<C>"*)" => (YYBEGIN INITIAL; lex());
<C>[*()] => (lex());
<B>\n+ => (lineNum := (!lineNum) + (String.size yytext); lex());
<B>[^{}\n]+ => (lex());
<B>"{" => (lex());
<B>"}" => (YYBEGIN INITIAL; lex());
|
