1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
#ifndef __TOKENIZER_H__
#define __TOKENIZER_H__
#include <string>
#include <list>
class Tokenizer;
class Token
{
friend class Tokenizer;
public:
enum Type {
Word,
Para,
Eof
};
private:
Type type;
std::wstring content;
private:
Token(Type type) { this->type = type; };
Token(Type type, const std::wstring &content): content(content) {
this->type = type;
}
public:
Type getType() const { return type; };
const std::wstring& getContent() const { return content; };
std::wstring toString() const;
};
class Tokenizer
{
private:
std::wstring text;
int currentPos;
std::list<Token> stack;
public:
Tokenizer(const std::wstring &s): text(s) { currentPos = 0; };
public:
Token getNextToken();
void unget(const Token &token);
bool isFinished();
private:
bool skipSpaces(bool notSearch);
};
#endif
|
