1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
# Before `make install' is performed this script should be runnable with
# `make test'. After `make install' it should work as `perl test.pl'
#########################
# change 'tests => 1' to 'tests => last_test_to_print';
use Test::More tests => 12;
use Lingua::EN::Tagger;
ok('Lingua::EN::Tagger', 'module compiled'); # If we made it this far, we're ok.
#########################
# Insert your test code below, the Test module is use()ed here so read
# its man page ( perldoc Test ) for help writing this test script.
######################################
# Start by creating the parser object
######################################
ok( $parser = Lingua::EN::Tagger->new( stem => 1), 'creating parser object' );
########################
# Check the stemmer
########################
is( $parser->stem( 'realize' ), 'realiz', 'stemming word' );
########################
# Does the parser work
########################
# Make sure that it doesn't die when parsing a non-text sample
ok( $parser->add_tags( "#!/usr/bin/perl -w\nuse strict;\nmy \$var = 'hello world';
print \$var | 'no value'; "), "non-text sample");
is( $parser->add_tags(), undef, 'NULL string' );
is( $parser->add_tags( '' ), undef, 'Empty string');
# How about real text?
ok( $parser->add_tags( penn() ), 'add_tags() method' );
ok( $tagged = $parser->get_readable( &penn() ), 'get_readable() method' );
########################################
# Check the accuracy of the parser
# with the get_readable() method
########################################
ok( $accuracy = compute_accuracy( $tagged, &penn_benchmark() ), 'computing accuracy' );
cmp_ok( $accuracy, '>=', 95, "overall accuracy ($accuracy%)" );
##########################
# Check hyphenated words
##########################
$tagged = $parser->get_readable( hyphen() );
$accuracy = compute_accuracy( $tagged, &hyphen_benchmark() );
cmp_ok( $accuracy, "==", 100, "hyphenated words ($accuracy%)" );
###################################
# Check a string of words that are
# largly unknown to the lexicon
###################################
$tagged = $parser->get_readable( jibberish() );
$accuracy = compute_accuracy( $tagged, &jibberish_benchmark() );
cmp_ok( $accuracy, '>=', 80, "unknown word accuracy ($accuracy%)" );
###############################################
# Words that mostly don't occur in the lexicon
###############################################
sub jibberish {
return "Дмитри occludes the 5 corybantic sciolists from fressing upon the
northeast-oriented perambulations of the yabbering doyenne";
}
sub jibberish_benchmark {
return "Дмитри/NNP occludes/VBZ the/DET 5/CD corybantic/JJ sciolists/NNS from/IN
fressing/VBG upon/IN the/DET northeast-oriented/JJ perambulations/NNS of/IN
the/DET yabbering/VBG doyenne/NN";
}
##########################################################
# Hyphenated words that mostly don't occur in the lexicon
##########################################################
sub hyphen {
# brother-in-law not in lexicon, sister-in-law is
return "The brother-in-law. The sister-in-law. A strategy of tit-for-tat among
middle-eastern states.";
}
sub hyphen_benchmark {
return "The/DET brother-in-law/NN ./PP The/DET sister-in-law/NN ./PP A/DET
strategy/NN of/IN tit-for-tat/NN among/IN middle-eastern/JJ states/NNS ./PP";
}
####################################################
# Test the tagger against an actual tagged corpus
####################################################
sub penn {
return 'Élenor Rainés, a lawyer and director of government relations for the Industrial Biotechnical Association, contends that a judge well-versed in patent law and the concerns of research-based industries would have ruled otherwise. And Judge Newman, a former patent lawyer, wrote in her dissent when the court denied a motion for a rehearing of the case by the full court, "The panel\'s judicial legislation has affected an important high-technological industry, without regard to the consequences for research and innovation or the public interest." Says Ms. Raines, "[The judgement] confirms our concern that the absence of patent lawyers on the court could prove troublesome."';
}
sub penn_cleaned {
qr|Élenor Rainés , a lawyer and director of government relations for the Industrial Biotechnical Association , contends that a judge well-versed in patent law and the concerns of research-based industries would have ruled otherwise \. And Judge Newman , a former patent lawyer , wrote in her dissent when the court denied a motion for a rehearing of the case by the full court , `` The panel 's judicial legislation has affected an important high-technological industry , without regard to the consequences for research and innovation or the public interest \. '' Says Ms\. Raines , `` \[ The judgement \] confirms our concern that the absence of patent lawyers on the court could prove troublesome \. ''|i
}
sub penn_benchmark {
return "Élenor/NNP Rainés/NNP ,/PPC a/DET lawyer/NN and/CC director/NN of/IN government/NN relations/NNS for/IN the/DET Industrial/NNP Biotechnical/NNP Association/NNP ,/PPC contends/VBZ that/IN a/DET judge/NN well-versed/JJ in/IN patent/NN law/NN and/CC the/DET concerns/NNS of/IN research-based/JJ industries/NNS would/MD have/VB ruled/VBN otherwise/RB ./PP And/CC Judge/NNP Newman/NNP ,/PPC a/DET former/JJ patent/NN lawyer/NN ,/PPC wrote/VBD in/IN her/PRPS dissent/NN when/WRB the/DET court/NN denied/VBD a/DET motion/NN for/IN a/DET rehearing/NN of/IN the/DET case/NN by/IN the/DET full/JJ court/NN ,/PPC ``/PPL The/DET panel/NN 's/POS judicial/JJ legislation/NN has/VBZ affected/VBN an/DET important/JJ high-technological/JJ industry/NN ,/PPC without/IN regard/NN to/TO the/DET consequences/NNS for/IN research/NN and/CC innovation/NN or/CC the/DET public/JJ interest/NN ./PP ''/PPR Says/VBZ Ms./NNP Raines/NNP ,/PPC ``/PPL [/LRB The/DET judgement/NN ]/RRB confirms/VBZ our/PRPS concern/NN that/IN the/DET absence/NN of/IN patent/NN lawyers/NNS on/IN the/DET court/NN could/MD prove/VB troublesome/JJ ./PP ''/PPR";
}
sub compute_accuracy {
my ( $test, $benchmark ) = @_;
utf8::decode( $benchmark );
my @tokenized = split /\s+/, $test;
my @model = split /\s+/, $benchmark;
my $accurate = 0;
my $num_items = scalar @tokenized;
for( my $i=0; $i <= $#tokenized; $i++ ){
my ( $my_word, $my_tag, $word, $tag );
( $my_word, $my_tag ) = $tokenized[$i] =~ /^(.*)\/([A-Z]+)$/
if $tokenized[$i];
( $word, $tag ) = $model[$i] =~ /^(.*)\/([A-Z]+)$/
if $model[$i];
if( $my_word ne $word ){
shift @model;
next;
}
$accurate++ if $my_tag eq $tag;
}
return sprintf( "%.1f", 100 * $accurate / $num_items);
}
|
