1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
header {
#include <iostream>
#include "UnicodeCharBuffer.hpp"
#include "UnicodeCharScanner.hpp"
}
options {
language="Cpp";
// genHashLines = false;
}
class L extends Lexer("UnicodeCharScanner");
options
{
// Allow any char but \uFFFF (16 bit -1)
// hmmm antlr does not allow \u10FFFE
charVocabulary='\u0000'..'\uFFFE';
noConstructors = true;
}
{
public:
bool done;
L( std::istream& in )
: UnicodeCharScanner(new UnicodeCharBuffer(in),true)
{
initLiterals();
}
L( UnicodeCharBuffer& ib )
: UnicodeCharScanner(ib,true)
{
initLiterals();
}
void uponEOF()
{
done = true;
}
}
WORD : ( ~(' '|'\n'|'\r'|'\t') )+
;
WS :
(' '
|'\n' { newline(); } ('\r')?
|'\r' { newline(); }
|'\t' { tab(); }
)
{$setType(ANTLR_USE_NAMESPACE(antlr)Token::SKIP);}
;
protected
ID_START_LETTER
: '$'
| '_'
| 'a'..'z'
| '\u0080'..'\ufffe'
;
protected
ID_LETTER
: ID_START_LETTER
| '0'..'9'
;
|
