1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
// Copyright (C) 2005 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_TOKENIZER_KERNEl_1_
#define DLIB_TOKENIZER_KERNEl_1_
#include <string>
#include <iosfwd>
#include <climits>
#include "../algs.h"
#include "tokenizer_kernel_abstract.h"
namespace dlib
{
class tokenizer_kernel_1
{
/*!
INITIAL VALUE
- in == 0
- streambuf == 0
- have_peeked == false
- head == "_" + lowercase_letters() + uppercase_letters()
- body == "_" + lowercase_letters() + uppercase_letters() + numbers()
- headset == pointer to an array of UCHAR_MAX bools and set according
to the CONVENTION.
- bodyset == pointer to an array of UCHAR_MAX bools and set according
to the CONVENTION.
CONVENTION
- if (stream_is_set()) then
- get_stream() == *in
- streambuf == in->rdbuf()
- else
- in == 0
- streambuf == 0
- body == get_identifier_body()
- head == get_identifier_head()
- if (the char x appears in head) then
- headset[static_cast<unsigned char>(x)] == true
- else
- headset[static_cast<unsigned char>(x)] == false
- if (the char x appears in body) then
- bodyset[static_cast<unsigned char>(x)] == true
- else
- bodyset[static_cast<unsigned char>(x)] == false
- if (have_peeked) then
- next_token == the next token to be returned from get_token()
- next_type == the type of token in peek_token
!*/
public:
// The name of this enum is irrelevant but on some compilers (gcc on MAC OS X) not having it named
// causes an error for whatever reason
enum some_random_name
{
END_OF_LINE,
END_OF_FILE,
IDENTIFIER,
CHAR,
NUMBER,
WHITE_SPACE
};
tokenizer_kernel_1 (
);
virtual ~tokenizer_kernel_1 (
);
void clear(
);
void set_stream (
std::istream& in
);
bool stream_is_set (
) const;
std::istream& get_stream (
) const;
void get_token (
int& type,
std::string& token
);
void swap (
tokenizer_kernel_1& item
);
void set_identifier_token (
const std::string& head,
const std::string& body
);
int peek_type (
) const;
const std::string& peek_token (
) const;
const std::string get_identifier_head (
) const;
const std::string get_identifier_body (
) const;
const std::string lowercase_letters (
) const;
const std::string uppercase_letters (
) const;
const std::string numbers (
) const;
private:
// restricted functions
tokenizer_kernel_1(const tokenizer_kernel_1&); // copy constructor
tokenizer_kernel_1& operator=(const tokenizer_kernel_1&); // assignment operator
// data members
std::istream* in;
std::streambuf* streambuf;
std::string head;
std::string body;
bool* headset;
bool* bodyset;
mutable std::string next_token;
mutable int next_type;
mutable bool have_peeked;
};
inline void swap (
tokenizer_kernel_1& a,
tokenizer_kernel_1& b
) { a.swap(b); }
}
#ifdef NO_MAKEFILE
#include "tokenizer_kernel_1.cpp"
#endif
#endif // DLIB_TOKENIZER_KERNEl_1
|
