1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
################################################################################
#
# PROGRAM: parser.pl
#
################################################################################
#
# DESCRIPTION: Generate tokenizer code for C parser
#
################################################################################
#
# $Project: /Convert-Binary-C $
# $Author: mhx $
# $Date: 2009/03/15 04:10:45 +0100 $
# $Revision: 22 $
# $Source: /token/parser.pl $
#
################################################################################
#
# Copyright (c) 2002-2009 Marcus Holland-Moritz. All rights reserved.
# This program is free software; you can redistribute it and/or modify
# it under the same terms as Perl itself.
#
################################################################################
use Devel::Tokenizer::C;
# keywords that cannot be disabled
@no_disable = qw(
break
case char continue
default do
else
for
goto
if int
return
sizeof struct switch
typedef
union
while
);
# keywords that can be disabled
@disable = qw(
asm auto
const
double
enum extern
float
inline
long
register restrict
short signed static
unsigned
void volatile
);
@basic = qw(
char
double
float
int
long
short signed
unsigned
);
# put them in a hash
@NDIS{@no_disable} = (1) x @no_disable;
$file = shift;
if( $file =~ /parser/ ) {
$t = Devel::Tokenizer::C->new( TokenFunc => \&t_parser )
->add_tokens( @disable, @no_disable );
}
elsif( $file =~ /basic/ ) {
$t = Devel::Tokenizer::C->new( TokenFunc => \&t_basic,
TokenString => 'c',
TokenEnd => '*name',
)
->add_tokens( @basic );
}
elsif( $file =~ /keywords/ ) {
$t = Devel::Tokenizer::C->new( TokenFunc => \&t_keywords, TokenString => 'str' )
->add_tokens( @disable );
}
elsif( $file =~ /ckeytok/ ) {
$t = Devel::Tokenizer::C->new( TokenFunc => \&t_ckeytok, TokenString => 'name' )
->add_tokens( @disable, @no_disable );
}
else { die "invalid file: $file\n" }
open OUT, ">$file" or die "$file: $!";
print OUT $t->generate;
close OUT;
sub t_parser {
my $token = shift;
if( exists $NDIS{$token} ) {
return "return \U$token\E_TOK;\n";
}
else {
return "if( pState->pCPC->keywords & HAS_KEYWORD_\U$token\E )\n"
. " return \U$token\E_TOK;\n";
}
};
sub t_basic {
my $token = shift;
if( $token eq 'long' ) {
return <<END
tflags |= tflags & T_LONG ? T_LONGLONG : T_LONG;
goto success;
END
}
return <<END
tflags |= T_\U$token\E;
goto success;
END
};
sub t_keywords {
my $token = shift;
return "keywords &= ~HAS_KEYWORD_\U$token\E;\n"
."goto success;\n";
};
sub t_ckeytok {
my $token = shift;
return <<END
static const CKeywordToken ckt = { \U$token\E_TOK, "$token" };
return &ckt;
END
};
|
