File: tokenizer.pl

package info (click to toggle)
libpegex-perl 0.75-2
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 908 kB
  • sloc: perl: 3,288; makefile: 43; sh: 2
file content (35 lines) | stat: -rw-r--r-- 652 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
use strict;
use Pegex;
use XXX;

{
    package TokenReceiver;
    use base 'Pegex::Tree';
    use XXX;

    my %tokens = (
        true => 1,
        false => 0,
        love => '',
        hate => '',
        bliss => '',
    );

    sub gotrule {
        my ($self, $got) = @_;
        return $got if ref $got;
        my $token = lc($got);
        if (exists $tokens{$token}) {
            my $value = $tokens{$token};
            return(length($value) ? $value : $token);
        }
        return $got->[0];
    }
}

my $grammar = <<'...';
expr: token+
token: /~(<WORD>+)~/
...

XXX pegex($grammar, 'TokenReceiver')->parse("LOVE true hate FALSE");