1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
|
#!/usr/bin/perl
use strict;
use warnings;
use YAML::XS qw/Dump/;
my (@info, @token_enum, @kind_enum, @syntax_enum, @type_to_info);
foreach (<DATA>) {
my ($kind, $type, $data) = split /\s+/;
my $info = { type => "$type", kind => "$kind", data => "$data" };
push @info, $info;
unless (grep { $_ eq $type } @token_enum) {
push @token_enum, $type;
$type_to_info[scalar @token_enum - 1] = $info;
}
unless (grep { $_ eq $kind } @kind_enum) {
push @kind_enum, $kind if ($kind);
}
}
@syntax_enum = qw/Value Term Expr Stmt BlockStmt/;
my $token_type = join ",\n", map { "\t$_" } @token_enum;
my $token_kind = join ",\n", map { "\t$_" } @kind_enum;
my $token_info = join ",\n", map {
sprintf(qq|\t{Enum::Token::Type::%s, Enum::Token::Kind::%s, "%s", "%s"}|,
$_->{type}, $_->{kind}, $_->{type}, $_->{data});
} @info;
my $type_to_info = join ",\n", map {
sprintf(qq|\t{Enum::Token::Type::%s, Enum::Token::Kind::%s, "%s", "%s"}|,
$_->{type}, $_->{kind}, $_->{type}, $_->{data});
} @type_to_info;
my %keyword_map;
$keyword_map{$_->{data}}++ foreach @info;
my $keywords = join "\n", map {
sprintf(qq|"%s", {Enum::Token::Type::%s, Enum::Token::Kind::%s, "%s", "%s"}|,
$_->{data}, $_->{type}, $_->{kind}, $_->{type}, $_->{data});
} grep {
$keyword_map{$_->{data}} == 1
} grep {
$_->{data}
} @info;
my $count = 0;
my $token_type_enums = join ",\n", map {
' ' x 4 . "T_$_ => " . $count++;
} @token_enum;
$count = 0;
my $syntax_type_enums = join ",\n", map {
' ' x 4 . "T_$_ => " . $count++;
} @syntax_enum;
$count = 0;
my $token_kind_enums = join ",\n", map {
' ' x 4 . "T_$_ => " . $count++;
} @kind_enum;
my %token_type_constants_map;
my %token_kind_constants_map;
my %syntax_type_constants_map;
$count = 0;
$token_type_constants_map{"T_$_"} = $count++ foreach @token_enum;
$count = 0;
$token_kind_constants_map{"T_$_"} = $count++ foreach @kind_enum;
$count = 0;
$syntax_type_constants_map{"T_$_"} = $count++ foreach @syntax_enum;
my $constants = +{
token_type => \%token_type_constants_map,
syntax_type => \%syntax_type_constants_map,
token_kind => \%token_kind_constants_map
};
open(my $fh, '>', 'include/gen_token.hpp');
print $fh <<"CODE";
namespace Enum {
namespace Token {
namespace Type {
typedef enum {
$token_type
} Type;
}
namespace Kind {
typedef enum {
$token_kind
} Kind;
}
}
}
CODE
open($fh, '>', 'src/compiler/util/Compiler_gen_token_decl.cpp');
print $fh <<"CODE";
#include <common.hpp>
TokenInfo decl_tokens[] = {
$token_info
};
TokenInfo type_to_info[] = {
$type_to_info
};
CODE
open($fh, '>', 'lib/Compiler/Lexer/Constants.pm');
print $fh <<CODE;
use strict;
use warnings;
package Compiler::Lexer::TokenType;
use constant {
$token_type_enums
};
package Compiler::Lexer::SyntaxType;
use constant {
$syntax_type_enums
};
package Compiler::Lexer::Kind;
use constant {
$token_kind_enums
};
1;
CODE
open($fh, '>', 'gen/gen_constants.yaml');
print $fh Dump $constants;
open($fh, '>', 'gen/reserved_keywords.gperf');
print $fh <<CODE;
%{
#include <lexer.hpp>
typedef struct _ReservedKeyword {
const char *name;
TokenInfo info;
} ReservedKeyword;
%}
ReservedKeyword;
%%
$keywords
%%
CODE
close($fh);
__DATA__
Return Return return
Operator Add +
Operator Sub -
Operator Mul *
Operator Div /
Operator Mod %
Operator ThreeTermOperator ?
Operator Greater >
Operator Less <
Operator StringAdd .
Operator Ref \\
Operator Glob *
Operator BitNot ~
Operator BitOr |
Operator AlphabetOr or
Operator BitAnd &
Operator AlphabetAnd and
Operator BitXOr ^
Operator AlphabetXOr xor
Operator StringMul x
Assign AddEqual +=
Assign SubEqual -=
Assign MulEqual *=
Assign DivEqual /=
Assign ModEqual %=
Assign StringAddEqual .=
Assign LeftShiftEqual <<=
Assign RightShiftEqual >>=
Assign StringMulEqual x=
Operator GreaterEqual >=
Operator LessEqual <=
Operator EqualEqual ==
Operator Diamond <>
Operator Compare <=>
Operator PolymorphicCompare ~~
Operator RegOK =~
Operator RegNot !~
Operator NotEqual !=
Operator StringLess lt
Operator StringLessEqual le
Operator StringGreater gt
Operator StringGreaterEqual ge
Operator StringEqual eq
Operator StringNotEqual ne
Operator StringCompare cmp
Operator Inc ++
Operator Dec --
Operator Exp **
Assign PowerEqual **=
Assign DefaultEqual //=
Operator LeftShift <<
Operator RightShift >>
Operator And &&
Operator Or ||
Assign AndBitEqual &=
Assign OrBitEqual |=
Assign NotBitEqual ^=
Assign OrEqual ||=
Assign AndEqual &&=
Operator Slice ..
Operator DefaultOperator //
Operator ToDo ...
Decl VarDecl my
Decl FunctionDecl sub
Function Method
Assign Assign =
SingleTerm ArraySize $#
SingleTerm Is
SingleTerm Not !
SingleTerm AlphabetNot not
Function BuiltinFunc chomp
Function BuiltinFunc chop
Function BuiltinFunc chr
Function BuiltinFunc crypt
Function BuiltinFunc index
Function BuiltinFunc lc
Function BuiltinFunc lcfirst
Function BuiltinFunc length
Function BuiltinFunc ord
Function BuiltinFunc pack
Function BuiltinFunc unpack
Function BuiltinFunc sort
Function BuiltinFunc reverse
Function BuiltinFunc rindex
Function BuiltinFunc sprintf
Function BuiltinFunc substr
Function BuiltinFunc uc
Function BuiltinFunc ucfirst
Function BuiltinFunc pos
Function BuiltinFunc quotemeta
Function BuiltinFunc split
Function BuiltinFunc study
Function BuiltinFunc pop
Function BuiltinFunc push
Function BuiltinFunc splice
Function BuiltinFunc shift
Function BuiltinFunc unshift
Function BuiltinFunc grep
Function BuiltinFunc join
Function BuiltinFunc map
Function BuiltinFunc delete
Function BuiltinFunc each
Function BuiltinFunc exists
Function BuiltinFunc keys
Function BuiltinFunc values
Function BuiltinFunc binmode
Function BuiltinFunc close
Function BuiltinFunc closedir
Function BuiltinFunc dbmclose
Function BuiltinFunc dbmopen
Function BuiltinFunc die
Function BuiltinFunc eof
Function BuiltinFunc fileno
Function BuiltinFunc flock
Function BuiltinFunc format
Function BuiltinFunc getc
Function BuiltinFunc print
Function BuiltinFunc say
Function BuiltinFunc printf
Function BuiltinFunc read
Function BuiltinFunc readdir
Function BuiltinFunc rewinddir
Function BuiltinFunc seek
Function BuiltinFunc seekdir
Function BuiltinFunc select
Function BuiltinFunc syscall
Function BuiltinFunc sysread
Function BuiltinFunc sysseek
Function BuiltinFunc syswrite
Function BuiltinFunc tell
Function BuiltinFunc telldir
Function BuiltinFunc truncate
Function BuiltinFunc warn
Function BuiltinFunc write
Function BuiltinFunc vec
Function BuiltinFunc chdir
Function BuiltinFunc chmod
Function BuiltinFunc chown
Function BuiltinFunc chroot
Function BuiltinFunc fcntl
Function BuiltinFunc glob
Function BuiltinFunc ioctl
Function BuiltinFunc link
Function BuiltinFunc lstat
Function BuiltinFunc mkdir
Function BuiltinFunc open
Function BuiltinFunc opendir
Function BuiltinFunc readlink
Function BuiltinFunc rename
Function BuiltinFunc rmdir
Function BuiltinFunc stat
Function BuiltinFunc symlink
Function BuiltinFunc umask
Function BuiltinFunc unlink
Function BuiltinFunc utime
Function BuiltinFunc caller
Function BuiltinFunc dump
Function BuiltinFunc eval
Function BuiltinFunc exit
Function BuiltinFunc wantarray
Function BuiltinFunc formline
Function BuiltinFunc reset
Function BuiltinFunc scalar
Function BuiltinFunc alarm
Function BuiltinFunc exec
Function BuiltinFunc fork
Function BuiltinFunc getpgrp
Function BuiltinFunc getppid
Function BuiltinFunc getpriority
Function BuiltinFunc kill
Function BuiltinFunc pipe
Function BuiltinFunc setpgrp
Function BuiltinFunc setpriority
Function BuiltinFunc sleep
Function BuiltinFunc system
Function BuiltinFunc times
Function BuiltinFunc wait
Function BuiltinFunc waitpid
Function BuiltinFunc no
Function BuiltinFunc tie
Function BuiltinFunc tied
Function BuiltinFunc untie
Function BuiltinFunc accept
Function BuiltinFunc bind
Function BuiltinFunc connect
Function BuiltinFunc getpeername
Function BuiltinFunc getsockname
Function BuiltinFunc getsockopt
Function BuiltinFunc listen
Function BuiltinFunc recv
Function BuiltinFunc send
Function BuiltinFunc setsockopt
Function BuiltinFunc shutdown
Function BuiltinFunc socket
Function BuiltinFunc socketpair
Function BuiltinFunc msgctl
Function BuiltinFunc msgget
Function BuiltinFunc msgrcv
Function BuiltinFunc msgsnd
Function BuiltinFunc semctl
Function BuiltinFunc semget
Function BuiltinFunc semop
Function BuiltinFunc shmctl
Function BuiltinFunc shmget
Function BuiltinFunc shmread
Function BuiltinFunc shmwrite
Function BuiltinFunc endgrent
Function BuiltinFunc endhostent
Function BuiltinFunc endnetent
Function BuiltinFunc endpwent
Function BuiltinFunc getgrent
Function BuiltinFunc getgrgid
Function BuiltinFunc getgrnam
Function BuiltinFunc getlogin
Function BuiltinFunc getpwent
Function BuiltinFunc getpwnam
Function BuiltinFunc getpwuid
Function BuiltinFunc setgrent
Function BuiltinFunc setpwent
Function BuiltinFunc endprotoent
Function BuiltinFunc endservent
Function BuiltinFunc gethostbyaddr
Function BuiltinFunc gethostbyname
Function BuiltinFunc gethostent
Function BuiltinFunc getnetbyaddr
Function BuiltinFunc getnetbyname
Function BuiltinFunc getnetent
Function BuiltinFunc getprotobyname
Function BuiltinFunc getprotobynumber
Function BuiltinFunc getprotoent
Function BuiltinFunc getservbyname
Function BuiltinFunc getservbyport
Function BuiltinFunc getservent
Function BuiltinFunc sethostent
Function BuiltinFunc setnetent
Function BuiltinFunc setprotoent
Function BuiltinFunc setservent
Function BuiltinFunc gmtime
Function BuiltinFunc localtime
Function BuiltinFunc time
Function BuiltinFunc ref
Function BuiltinFunc bless
Function BuiltinFunc defined
Function BuiltinFunc abs
Function BuiltinFunc atan2
Function BuiltinFunc cos
Function BuiltinFunc exp
Function BuiltinFunc hex
Function BuiltinFunc int
Function BuiltinFunc log
Function BuiltinFunc oct
Function BuiltinFunc rand
Function BuiltinFunc sin
Function BuiltinFunc sqrt
Function BuiltinFunc srand
Decl RequireDecl require
Import Import import
SpecificKeyword SpecificKeyword __PACKAGE__
SpecificKeyword SpecificKeyword __FILE__
SpecificKeyword SpecificKeyword __LINE__
SpecificKeyword SpecificKeyword __SUB__
DataWord DataWord __DATA__
DataWord DataWord __END__
ModWord ModWord BEGIN
ModWord ModWord CHECK
ModWord ModWord INIT
ModWord ModWord END
ModWord ModWord UNITCHECK
AUTOLOAD AUTOLOAD AUTOLOAD
CORE CORE CORE
DESTROY DESTROY DESTROY
Handle STDIN STDIN
Handle STDOUT STDOUT
Handle STDERR STDERR
Control Redo redo
Control Next next
Control Last last
Control Goto goto
Control Continue continue
Do Do do
Control Break break
Handle Handle -b
Handle Handle -c
Handle Handle -d
Handle Handle -e
Handle Handle -f
Handle Handle -g
Handle Handle -k
Handle Handle -l
Handle Handle -o
Handle Handle -p
Handle Handle -r
Handle Handle -s
Handle Handle -t
Handle Handle -u
Handle Handle -w
Handle Handle -x
Handle Handle -z
Handle Handle -A
Handle Handle -B
Handle Handle -C
Handle Handle -M
Handle Handle -O
Handle Handle -R
Handle Handle -S
Handle Handle -T
Handle Handle -W
Handle Handle -X
Decl LocalDecl local
Decl OurDecl our
Decl StateDecl state
Decl UseDecl use
Module UsedName
Module RequiredName
Stmt IfStmt if
Stmt ElseStmt else
Stmt ElsifStmt elsif
Stmt UnlessStmt unless
Stmt UntilStmt until
Stmt WhenStmt when
Stmt GivenStmt given
DefaultStmt DefaultStmt default
Comma Comma ,
Colon Colon :
StmtEnd SemiColon ;
Symbol LeftParenthesis (
Symbol RightParenthesis )
Symbol LeftBrace {
Symbol RightBrace }
Symbol LeftBracket [
Symbol RightBracket ]
Modifier ArrayDereference @{
Modifier HashDereference %{
Modifier ScalarDereference ${
Modifier CodeDereference &{
Modifier ShortScalarDereference
Modifier ShortArrayDereference @$
Modifier ShortHashDereference %$
Modifier ShortCodeDereference &$
Modifier ArraySizeDereference $#{
Term Key
Term BareWord
Operator Arrow =>
Operator Pointer ->
Operator NamespaceResolver ::
Namespace Namespace
Package Package package
Class Class
Decl CallDecl &
SingleTerm CodeRef \\&
Stmt WhileStmt while
Stmt ForStmt for
Stmt ForeachStmt foreach
Annotation Annotation #@
Term ArgumentArray @_
Term SpecificValue $_
Term SpecificValue $0
Term SpecificValue $1
Term SpecificValue $2
Term SpecificValue $3
Term SpecificValue $4
Term SpecificValue $5
Term SpecificValue $6
Term SpecificValue $7
Term SpecificValue $8
Term SpecificValue $9
Term SpecificValue $&
Term SpecificValue $`
Term SpecificValue $'
Term SpecificValue $+
Term SpecificValue $.
Term SpecificValue $/
Term SpecificValue $|
Term SpecificValue $*
Term SpecificValue $,
Term SpecificValue $\\
Term SpecificValue $\"
Term SpecificValue $%
Term SpecificValue $=
Term SpecificValue $-
Term SpecificValue $~
Term SpecificValue $^
Term SpecificValue $:
Term SpecificValue $?
Term SpecificValue $!
Term SpecificValue $@
Term SpecificValue $$
Term SpecificValue $<
Term SpecificValue $>
Term SpecificValue $(
Term SpecificValue $)
Term SpecificValue $[
Term SpecificValue $]
Term SpecificValue $;
Term SpecificValue $^A
Term SpecificValue $^D
Term SpecificValue $^E
Term SpecificValue $^F
Term SpecificValue $^G
Term SpecificValue $^H
Term SpecificValue $^I
Term SpecificValue $^L
Term SpecificValue $^M
Term SpecificValue $^O
Term SpecificValue $^P
Term SpecificValue $^R
Term SpecificValue $^T
Term SpecificValue $^W
Term SpecificValue $^X
Term ConstValue
Term ProgramArgument @ARGV
Term LibraryDirectories @INC
Term Environment %ENV
Term Include %INC
Term Signal %SIG
RegOpt RegOpt
RegPrefix RegQuote q
RegPrefix RegDoubleQuote qq
RegPrefix RegList qw
RegPrefix RegExec qx
RegPrefix RegDecl qr
RegPrefix RegMatch m
Term RegDelim
Term HandleDelim
Term RegMiddleDelim
RegReplacePrefix RegAllReplace tr
RegReplacePrefix RegAllReplace y
RegReplacePrefix RegReplace s
Term RegReplaceFrom
Term RegReplaceTo
Decl FieldDecl
Ref TypeRef
Ref LabelRef
Decl LocalVarDecl
Decl GlobalVarDecl
Decl MultiLocalVarDecl
Decl MultiGlobalVarDecl
Term Prototype
Term Var
Term CodeVar
Term ArrayVar
Term HashVar
Term Int
Term Double
Term String
Term RawString
Term ExecString
Term VersionString
Term HereDocumentTag
Term HereDocumentRawTag
Term HereDocumentExecTag
Term HereDocumentBareTag
Term RawHereDocument
Term HereDocument
Term HereDocumentEnd
Decl FormatDecl
Term Format
Term FormatEnd
Term Object
Term RegExp
Term Array
Term Hash
Operator Operator
Term LocalVar
Term LocalArrayVar
Term LocalHashVar
Term GlobalVar
Term GlobalArrayVar
Term GlobalHashVar
Ref ArrayRef
Ref HashRef
Get ArrayAt
Get HashAt
Set ArraySet
Set HashSet
Decl Function
Function Call
Term Argument
Term List
Term Default undef
Verbose Pod
Verbose Comment
Verbose WhiteSpace
Symbol PostDeref
Symbol PostDerefStar
Symbol PostDerefArraySliceOpenBracket
Symbol PostDerefArraySliceCloseBracket
Symbol PostDerefHashSliceOpenBrace
Symbol PostDerefHashSliceCloseBrace
Symbol PostDerefCodeOpenParen
Symbol PostDerefCodeCloseParen
Undefined Undefined
|
