1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef _INIT_PARSER_TOKENIZER_H
#define _INIT_PARSER_TOKENIZER_H
#include <string>
namespace init {
// Used to tokenize a std::string.
// Call Next() to advance through each token until it returns false,
// indicating there are no more tokens left in the string.
// The current token can be accessed with current(), which returns
// a Token.
// Supported tokens are:
// TOK_START - Next() has yet to be called
// TOK_END - At the end of string
// TOK_NEWLINE - The end of a line denoted by \n.
// TOK_TEXT - A word.
// Comments are denoted with '#' and the tokenizer will ignore
// the rest of the line.
// Double quotes can be used to insert whitespace into words.
// A backslash at the end of a line denotes continuation and
// a TOK_NEWLINE will not be generated for that line.
class Tokenizer {
public:
explicit Tokenizer(const std::string& data);
~Tokenizer();
enum TokenType { TOK_START, TOK_END, TOK_NEWLINE, TOK_TEXT };
struct Token {
TokenType type;
std::string text;
};
// Returns the curret token.
const Token& current();
// Move to the next token, returns false at the end of input.
bool Next();
private:
void GetData();
void AdvChar();
void AdvText();
void AdvUntil(char x);
void AdvWhiteSpace();
void StartText();
void EndText();
const std::string& data_;
Token current_;
bool eof_;
size_t pos_;
char cur_char_;
size_t tok_start_;
};
} // namespace init
#endif
|
