1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
#! /usr/local/bin/perl -sw
# IT'S A PARSER *AND* A LEXER...
use Parse::RecDescent;
use Data::Dumper;
$lexer = new Parse::RecDescent q
{
lex: token(s)
token: 'I\b' <token:POSS>
| 'see\b' <token:VERB>
| 'on\b' <token:PREP>
| 'by\b' <token:PREP>
| /the\b|a\b/i <token:ARTICLE>
| /\w+/ <token:WORD,PUNCT,OTHER>
};
$data = join '', <DATA>;
print_tokens($lexer->lex(\$data));
print "left: [$data]\n";
sub print_tokens
{
foreach $token ( @{$_[0]} )
{
print Dumper($token), "\n";
}
}
__DATA__
I see a cat on the windowsill by the door!!!!!
|
