1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
|
/**
* <b>SOFTWARE RIGHTS</b>
* <p>
* ANTLR 2.6.0 MageLang Insitute, 1998
* <p>
* We reserve no legal rights to the ANTLR--it is fully in the
* public domain. An individual or company may do whatever
* they wish with source code distributed with ANTLR or the
* code generated by ANTLR, including the incorporation of
* ANTLR, or its output, into commerical software.
* <p>
* We encourage users to develop software with ANTLR. However,
* we do ask that credit is given to us for developing
* ANTLR. By "credit", we mean that if you use ANTLR or
* incorporate any source code into one of your programs
* (commercial product, research project, or otherwise) that
* you acknowledge this fact somewhere in the documentation,
* research report, etc... If you like ANTLR and have
* developed a nice tool with the output, please mention that
* you developed it using ANTLR. In addition, we ask that the
* headers remain intact in our source code. As long as these
* guidelines are kept, we expect to continue enhancing this
* system and expect to make other tools available as they are
* completed.
* <p>
* The ANTLR gang:
* @version ANTLR 2.6.0 MageLang Insitute, 1998
* @author Terence Parr, <a href=http://www.MageLang.com>MageLang Institute</a>
* @author <br>John Lilley, <a href=http://www.Empathy.com>Empathy Software</a>
* @author <br><a href="mailto:pete@yamuna.demon.co.uk">Pete Wells</a>
*/
#include "Antlr/CharScanner.hpp"
#include "Antlr/CommonToken.hpp"
#include "Antlr/TokenStream.hpp"
#include "Antlr/ScannerException.hpp"
#include <map>
#include <cctype>
#include <iostream>
#include <cstring>
#include <cstdlib>
#ifdef _MSC_VER
#pragma warning(disable: 4355) // used in base member initializer list
#endif
CharScannerLiteralsLess::CharScannerLiteralsLess(const CharScanner* theScanner)
: scanner(theScanner)
{}
bool CharScannerLiteralsLess::operator() (const std::string& x,const std::string& y) const
{
if (scanner->getCaseSensitiveLiterals()) {
return std::less<std::string>()(x,y);
} else {
#ifdef NO_STRCASECMP
return (_stricmp(x.c_str(),y.c_str())<0);
#else
return (strcasecmp(x.c_str(),y.c_str())<0);
#endif
}
}
CharScanner::CharScanner(InputBuffer& cb)
: saveConsumedInput(true) //, caseSensitiveLiterals(true)
, literals(CharScannerLiteralsLess(this))
, inputState(new LexerInputState(cb))
, commitToPath(false)
{
setTokenObjectFactory(&CommonToken::factory);
}
CharScanner::CharScanner(InputBuffer* cb)
: saveConsumedInput(true) //, caseSensitiveLiterals(true)
, literals(CharScannerLiteralsLess(this))
, inputState(new LexerInputState(cb))
, commitToPath(false)
{
setTokenObjectFactory(&CommonToken::factory);
}
CharScanner::CharScanner(const LexerSharedInputState& state)
: saveConsumedInput(true) //, caseSensitiveLiterals(true)
, literals(CharScannerLiteralsLess(this))
, inputState(state)
, commitToPath(false)
{
setTokenObjectFactory(&CommonToken::factory);
}
CharScanner::~CharScanner()
{
}
void CharScanner::append(char c)
{
if (saveConsumedInput)
text+=c;
}
void CharScanner::append(const std::string& s)
{
if (saveConsumedInput)
text+=s;
}
void CharScanner::commit()
{
inputState->getInput().commit();
}
void CharScanner::consume()
{
if (inputState->guessing == 0) {
if (caseSensitive) {
append(LA(1));
} else {
// use input.LA(), not LA(), to get original case
// CharScanner.LA() would toLower it.
append(inputState->getInput().LA(1));
}
}
inputState->getInput().consume();
}
/** Consume chars until one matches the given char */
void CharScanner::consumeUntil(int c)
{
while (LA(1) != EOF_CHAR && LA(1) != c)
{
consume();
}
}
/** Consume chars until one matches the given set */
void CharScanner::consumeUntil(const BitSet& set)
{
while (LA(1) != EOF_CHAR && !set.member(LA(1))) {
consume();
}
}
bool CharScanner::getCaseSensitive() const
{ return caseSensitive; }
//bool CharScanner::getCaseSensitiveLiterals() const
//{ return caseSensitiveLiterals; }
bool CharScanner::getCommitToPath() const
{ return commitToPath; }
const std::string& CharScanner::getFilename() const
{ return inputState->filename; }
InputBuffer& CharScanner::getInputBuffer()
{ return inputState->getInput(); }
LexerSharedInputState CharScanner::getInputState()
{ return inputState; }
int CharScanner::getLine() const
{ return inputState->line; }
// return a copy of the current text buffer
const std::string& CharScanner::getText() const
{ return text; }
RefToken CharScanner::getTokenObject() const
{ return _returnToken; }
int CharScanner::LA(int i)
{
if ( caseSensitive ) {
return inputState->getInput().LA(i);
} else {
return toLower(inputState->getInput().LA(i));
}
}
RefToken CharScanner::makeToken(int t)
{
RefToken tok=tokenFactory();
tok->setType(t);
tok->setLine(inputState->line);
return tok;
}
int CharScanner::mark()
{
return inputState->getInput().mark();
}
void CharScanner::match(int c)
{
if ( LA(1) != c ) {
throw ScannerException(std::string("mismatched char: '") + charName(LA(1)) + "' expected '"+charName(c)+"'", inputState->line);
}
consume();
}
void CharScanner::match(const BitSet& b)
{
if (!b.member(LA(1))) {
throw ScannerException(std::string("mismatched char: '") + charName(LA(1)) + "'", inputState->line);
}
consume();
}
void CharScanner::match(const std::string& s)
{
int len = s.length();
for (int i=0; i<len; i++) {
if ( LA(1) != s[i] ) {
throw ScannerException(std::string("mismatched char: '") + charName(LA(1)) + "'", inputState->line);
}
consume();
}
}
void CharScanner::matchNot(int c)
{
if ( LA(1) == c ) {
throw ScannerException(std::string("mismatched char: '") + charName(LA(1)) + "'", inputState->line);
}
consume();
}
void CharScanner::matchRange(int c1, int c2)
{
if (LA(1)<c1 || LA(1)>c2) {
throw ScannerException(std::string("char out of range: '") + charName(LA(1)) + "'", inputState->line);
}
consume();
}
void CharScanner::newline()
{ ++inputState->line; }
void CharScanner::panic()
{
std::cerr << "CharScanner: panic" << std::endl;
exit(1);
}
void CharScanner::panic(const std::string& s)
{
std::cerr << "CharScanner: panic: " << s << std::endl;
exit(1);
}
/** Report exception errors caught in nextToken() */
void CharScanner::reportError(const ScannerException& ex)
{
if (getFilename() == "")
std::cerr << "Error: " << ex.toString() << std::endl;
else
std::cerr << "Error in " << getFilename() << ": " << ex.toString() << std::endl;
}
/** Parser error-reporting function can be overridden in subclass */
void CharScanner::reportError(const std::string& s)
{
if (getFilename() == "")
std::cerr << "Error: " << s << std::endl;
else
std::cerr << "Error in " << getFilename() << ": " << s << std::endl;
}
/** Parser warning-reporting function can be overridden in subclass */
void CharScanner::reportWarning(const std::string& s)
{
if (getFilename() == "")
std::cerr << "Warning: " << s << std::endl;
else
std::cerr << "Warning in " << getFilename() << ": " << s << std::endl;
}
void CharScanner::resetText()
{ text=""; }
void CharScanner::rewind(int pos)
{
inputState->getInput().rewind(pos);
}
void CharScanner::setCaseSensitive(bool t)
{
caseSensitive = t;
}
void CharScanner::setCommitToPath(bool commit)
{
commitToPath = commit;
}
void CharScanner::setFilename(const std::string& f)
{ inputState->filename=f; }
void CharScanner::setLine(int l)
{ inputState->line=l; }
void CharScanner::setText(const std::string& s)
{ text=s; }
void CharScanner::setTokenObjectFactory(factory_type factory)
{ tokenFactory=factory; }
// Test the token text against the literals table
// Override this method to perform a different literals test
int CharScanner::testLiteralsTable(int ttype) const
{
std::map<std::string,int,CharScannerLiteralsLess>::const_iterator i = literals.find(text);
if (i != literals.end())
ttype = (*i).second;
return ttype;
}
// Override this method to get more specific case handling
int CharScanner::toLower(int c) const
{
return tolower(c);
}
void CharScanner::traceIn(const std::string& rname)
{
std::cout << "enter lexer " << rname << "; c==" << LA(1) << std::endl;
}
void CharScanner::traceOut(const std::string& rname)
{
std::cout << "exit lexer " << rname << "; c==" << LA(1) << std::endl;
}
const char* CharScanner::charName(int ch)
{
if (ch == EOF)
return "EOF";
else {
static char buf[2];
buf[0] = static_cast<char>(ch);
buf[1] = '\0';
return buf;
}
}
#ifndef NO_STATIC_CONSTS
const char CharScanner::NO_CHAR;
const int CharScanner::EOF_CHAR;
#endif
|