1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
//#include <iostream>
#include "LexTokenStream.hpp"
#include "JavaTokenTypes.hpp"
#include "antlr/CommonToken.hpp"
extern "C" {
extern char* yytext;
extern char string_buf[]; // used for strings
extern int yylex();
extern int yyrestart(std::FILE*);
}
static LexTokenStream* currentLexer=0;
extern "C"
void newLine()
{
currentLexer->newLine();
}
LexTokenStream::LexTokenStream(std::FILE* fp)
{
yyrestart(fp);
reachedEOF = false;
line = 1;
}
void LexTokenStream::newLine()
{
++line;
}
ANTLR_USE_NAMESPACE(antlr)RefToken LexTokenStream::nextToken()
{
ANTLR_USE_NAMESPACE(antlr)RefToken ret;
if (!reachedEOF)
{
int type;
currentLexer = this;
while ((type = yylex()) == -1) // Token::SKIP
;
if (type != 0)
{
if( type != JavaTokenTypes::STRING_LITERAL )
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,yytext));
else
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(type,string_buf));
// ANTLR_USE_NAMESPACE(std)cout << "Read token " << type << " [" << ret->getText() << "]" << ANTLR_USE_NAMESPACE(std)endl;
}
else
reachedEOF = true;
}
if (reachedEOF)
{
ret = ANTLR_USE_NAMESPACE(antlr)RefToken(new ANTLR_USE_NAMESPACE(antlr)CommonToken(1,"EOF"));
// cout << "EOF reached" << endl;
}
ret->setLine(line);
return ret;
}
|
