1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
#ifndef TOKEN_H
#define TOKEN_H
/*
* Basic tokenization structures. NOTE! Those tokens had better
* be pretty small, since we're going to keep them all in memory
* indefinitely.
*
* Copyright (C) 2003 Transmeta Corp.
* 2003 Linus Torvalds
*
* Licensed under the Open Software License version 1.1
*/
#include <sys/types.h>
#include "lib.h"
/*
* This describes the pure lexical elements (tokens), with
* no semantic meaning. In other words, an identifier doesn't
* have a type or meaning, it is only a specific string in
* the input stream.
*
* Semantic meaning is handled elsewhere.
*/
enum constantfile {
CONSTANT_FILE_MAYBE, // To be determined, not inside any #ifs in this file
CONSTANT_FILE_IFNDEF, // To be determined, currently inside #ifndef
CONSTANT_FILE_NOPE, // No
CONSTANT_FILE_YES // Yes
};
extern const char *includepath[];
struct stream {
int fd;
const char *name;
const char *path; // input-file path - see set_stream_include_path()
const char **next_path;
/* Use these to check for "already parsed" */
enum constantfile constant;
int dirty;
struct ident *protect;
struct token *ifndef;
struct token *top_if;
};
extern int input_stream_nr;
extern struct stream *input_streams;
struct ident {
struct ident *next; /* Hash chain of identifiers */
struct symbol *symbols; /* Pointer to semantic meaning list */
unsigned char len; /* Length of identifier name */
unsigned char tainted:1,
reserved:1,
keyword:1;
char name[]; /* Actual identifier */
};
enum token_type {
TOKEN_EOF,
TOKEN_ERROR,
TOKEN_IDENT,
TOKEN_ZERO_IDENT,
TOKEN_NUMBER,
TOKEN_CHAR,
TOKEN_STRING,
TOKEN_SPECIAL,
TOKEN_STREAMBEGIN,
TOKEN_STREAMEND,
TOKEN_MACRO_ARGUMENT,
TOKEN_STR_ARGUMENT,
TOKEN_QUOTED_ARGUMENT,
TOKEN_CONCAT,
TOKEN_GNU_KLUDGE,
TOKEN_UNTAINT,
TOKEN_ARG_COUNT,
TOKEN_IF,
TOKEN_SKIP_GROUPS,
TOKEN_ELSE,
};
/* Combination tokens */
#define COMBINATION_STRINGS { \
"+=", "++", \
"-=", "--", "->", \
"*=", \
"/=", \
"%=", \
"<=", ">=", \
"==", "!=", \
"&&", "&=", \
"||", "|=", \
"^=", "##", \
"<<", ">>", "..", \
"<<=", ">>=", "...", \
"", \
"<", ">", "<=", ">=" \
}
extern unsigned char combinations[][4];
enum special_token {
SPECIAL_BASE = 256,
SPECIAL_ADD_ASSIGN = SPECIAL_BASE,
SPECIAL_INCREMENT,
SPECIAL_SUB_ASSIGN,
SPECIAL_DECREMENT,
SPECIAL_DEREFERENCE,
SPECIAL_MUL_ASSIGN,
SPECIAL_DIV_ASSIGN,
SPECIAL_MOD_ASSIGN,
SPECIAL_LTE,
SPECIAL_GTE,
SPECIAL_EQUAL,
SPECIAL_NOTEQUAL,
SPECIAL_LOGICAL_AND,
SPECIAL_AND_ASSIGN,
SPECIAL_LOGICAL_OR,
SPECIAL_OR_ASSIGN,
SPECIAL_XOR_ASSIGN,
SPECIAL_HASHHASH,
SPECIAL_LEFTSHIFT,
SPECIAL_RIGHTSHIFT,
SPECIAL_DOTDOT,
SPECIAL_SHL_ASSIGN,
SPECIAL_SHR_ASSIGN,
SPECIAL_ELLIPSIS,
SPECIAL_ARG_SEPARATOR,
SPECIAL_UNSIGNED_LT,
SPECIAL_UNSIGNED_GT,
SPECIAL_UNSIGNED_LTE,
SPECIAL_UNSIGNED_GTE,
};
struct string {
unsigned int length;
char data[];
};
/* will fit into 32 bits */
struct argcount {
unsigned normal:10;
unsigned quoted:10;
unsigned str:10;
unsigned vararg:1;
};
/*
* This is a very common data structure, it should be kept
* as small as humanly possible. Big (rare) types go as
* pointers.
*/
struct token {
struct position pos;
struct token *next;
union {
const char *number;
struct ident *ident;
unsigned int special;
struct string *string;
int character;
int argnum;
struct argcount count;
};
};
#define MAX_STRING 4095
static inline struct token *containing_token(struct token **p)
{
void *addr = (char *)p - ((char *)&((struct token *)0)->next - (char *)0);
return addr;
}
#define token_type(x) ((x)->pos.type)
/*
* Last token in the stream - points to itself.
* This allows us to not test for NULL pointers
* when following the token->next chain..
*/
extern struct token eof_token_entry;
#define eof_token(x) ((x) == &eof_token_entry)
extern int init_stream(const char *, int fd, const char **next_path);
extern const char *stream_name(int stream);
extern struct ident *hash_ident(struct ident *);
extern struct ident *built_in_ident(const char *);
extern struct token *built_in_token(int, const char *);
extern const char *show_special(int);
extern const char *show_ident(const struct ident *);
extern const char *show_string(const struct string *string);
extern const char *show_token(const struct token *);
extern struct token * tokenize(const char *, int, struct token *, const char **next_path);
extern struct token * tokenize_buffer(void *, unsigned long, struct token *);
extern void show_identifier_stats(void);
extern struct token *preprocess(struct token *);
static inline int match_op(struct token *token, int op)
{
return token->pos.type == TOKEN_SPECIAL && token->special == op;
}
static inline int match_ident(struct token *token, struct ident *id)
{
return token->pos.type == TOKEN_IDENT && token->ident == id;
}
#endif
|
