1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
/*--------------------------------------------------------------------*//*:Ignore this sentence.
Copyright (C) 1999, 2001 SIL International. All rights reserved.
Distributable under the terms of either the Common Public License or the
GNU Lesser General Public License, as specified in the LICENSING.txt file.
File: GrpTokenStreamFilter.hpp
Responsibility: Sharon Correll
Last reviewed: Not yet.
Description:
Implementions of methods for classes that are extensions to the ANTLR classes.
-------------------------------------------------------------------------------*//*:End Ignore*/
#include "Grp.h"
#ifdef _MSC_VER
#pragma hdrstop
#endif
#undef THIS_FILE
DEFINE_THIS_FILE
/*----------------------------------------------------------------------------------------------
Get the next token from the lexer. If it is a C-preprocessor line marker, do something
special, otherwise pass the token on through to the parser.
----------------------------------------------------------------------------------------------*/
RefToken GrpTokenStreamFilter::nextToken()
{
RefToken tok;
if (m_tokPeek)
{
tok = m_tokPeek;
m_tokPeek = RefToken(NULL);
}
else
tok = m_lexer->nextToken();
while (tok && tok->getType() == OP_LINEMARKER )
{
// Handle the information from the line-and-file marker.
RefToken tokLineNumber = m_lexer->nextToken();
Assert(tokLineNumber && tokLineNumber->getType() == LIT_INT);
RefToken tokFileName = m_lexer->nextToken();
//Assert(tokFileName && tokFileName->getType() == LIT_STRING);
int nLineInMarker = atoi(tokLineNumber->getText().c_str());
int nLinePre = tokLineNumber->getLine();
m_staPrevFile = m_staFile;
m_nPrevLineOffset = m_nLineOffset;
m_nLastLineMarker = nLinePre;
if (tokFileName && tokFileName->getType() == LIT_STRING)
{
m_staFile = tokFileName->getText().c_str();
tok = m_lexer->nextToken();
}
else
{
// m_staFile stays the same
tok = tokFileName;
}
m_nLineOffset = nLineInMarker - nLinePre - 1; // -1, because line marker gives
// the number of the NEXT line
}
if (tok && tok->getType() == LITERAL_else)
{
// "else" immediately followed by "if" on the same line is equivalent to "elseif",
// which does not need a separate "endif".
m_tokPeek = m_lexer->nextToken();
if (m_tokPeek && m_tokPeek->getType() == LITERAL_if &&
m_tokPeek->getLine() == tok->getLine())
{
tok->setType(Zelseif);
tok->setText("else if");
m_tokPeek = RefToken(NULL); // throw away if the if
}
// otherwise, keep the peeked-at token for the next nextToken() call.
}
else if (tok && tok->getType() == AT_IDENT)
{
// Break the token of the form "@abc" or "@:abc" into two tokens: OP_AT and Qalias.
std::string s = tok->getText();
Assert(s[0] == '@');
if (s.length() > 1)
{
unsigned int ichMin = (s[1] == ':') ? 2 : 1; // ignore colon
if (s.length() > ichMin)
{
std::string sIdent = s.substr(ichMin, s.length());
RefToken tokNext = m_lexer->publicMakeToken(Qalias);
if (s[ichMin] >= '0' && s[ichMin] <= '9')
tokNext->setType(LIT_INT);
tokNext->setText(sIdent);
tokNext->setLine(tok->getLine());
m_tokPeek = tokNext;
}
}
tok->setType(OP_AT);
tok->setText("@");
}
// Adjust the line and file information in the token.
if (tok)
{
Token * bareToken = tok.get();
GrpToken * wrToken = dynamic_cast<GrpToken *>(bareToken);
Assert(wrToken);
int nLinePre = tok->getLine();
wrToken->SetOrigLineAndFile(nLinePre + m_nLineOffset, m_staFile);
}
return tok;
}
/*----------------------------------------------------------------------------------------------
Initialize a tree node with the line-and-file information from the lexer token.
----------------------------------------------------------------------------------------------*/
void GrpASTNode::initialize(RefToken t)
{
CommonASTNode::initialize(t);
Token * bareToken = t.get();
GrpToken * wrToken = dynamic_cast<GrpToken *>(bareToken);
Assert(wrToken);
m_lnf = wrToken->LineAndFile();
}
/*----------------------------------------------------------------------------------------------
Intercept a lexer error and add the original line and file information.
----------------------------------------------------------------------------------------------*/
void GrpTokenStreamFilter::ReportLexerError(const ScannerException & ex)
{
int nLinePre = ex.getLine();
int nLineOrig = nLinePre + m_nLineOffset;
AddGlobalError(true, 101, ex.getErrorMessage(), GrpLineAndFile(nLinePre, nLineOrig, m_staFile));
}
/*----------------------------------------------------------------------------------------------
Intercept a parser error and add the original line and file information.
----------------------------------------------------------------------------------------------*/
void GrpTokenStreamFilter::ReportParserError(const ParserException & ex)
{
int nLinePre = ex.getLine();
if (nLinePre <= m_nLastLineMarker)
// Problematic token was before the last line marker in the pre-processed file.
AddGlobalError(true, 102, ex.getErrorMessage(),
GrpLineAndFile(nLinePre, nLinePre + m_nPrevLineOffset, m_staPrevFile));
else
AddGlobalError(true, 103, ex.getErrorMessage(),
GrpLineAndFile(nLinePre, nLinePre + m_nLineOffset, m_staFile));
}
|
