1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
print "1..13\n";
use WWW::RobotRules::AnyDBM_File;
$file = "test-$$";
$r = new WWW::RobotRules::AnyDBM_File "myrobot/2.0", $file;
$r->parse("http://www.aas.no/robots.txt", "");
$r->visit("www.aas.no");
print "not " if $r->no_visits("www.aas.no") != 1;
print "ok 1\n";
$r->push_rules("www.sn.no", "/aas", "/per");
$r->push_rules("www.sn.no", "/god", "/old");
@r = $r->rules("www.sn.no");
print "Rules: @r\n";
print "not " if "@r" ne "/aas /per /god /old";
print "ok 2\n";
$r->clear_rules("per");
$r->clear_rules("www.sn.no");
@r = $r->rules("www.sn.no");
print "Rules: @r\n";
print "not " if "@r" ne "";
print "ok 3\n";
$r->visit("www.aas.no", time+10);
$r->visit("www.sn.no");
print "No visits: ", $r->no_visits("www.aas.no"), "\n";
print "Last visit: ", $r->last_visit("www.aas.no"), "\n";
print "Fresh until: ", $r->fresh_until("www.aas.no"), "\n";
print "not " if $r->no_visits("www.aas.no") != 2;
print "ok 4\n";
print "not " if abs($r->last_visit("www.sn.no") - time) > 2;
print "ok 5\n";
$r = undef;
# Try to reopen the database without a name specified
$r = new WWW::RobotRules::AnyDBM_File undef, $file;
$r->visit("www.aas.no");
print "not " if $r->no_visits("www.aas.no") != 3;
print "ok 6\n";
print "Agent-Name: ", $r->agent, "\n";
print "not " if $r->agent ne "myrobot";
print "ok 7\n";
$r = undef;
print "*** Dump of database ***\n";
tie(%cat, AnyDBM_File, $file, 0, 0644) or die "Can't tie: $!";
while (($key,$val) = each(%cat)) {
print "$key\t$val\n";
}
print "******\n";
untie %cat;
# Try to open database with a different agent name
$r = new WWW::RobotRules::AnyDBM_File "MOMSpider/2.0", $file;
print "not " if $r->no_visits("www.sn.no");
print "ok 8\n";
# Try parsing
$r->parse("http://www.sn.no:8080/robots.txt", <<EOT, (time + 3));
User-Agent: *
Disallow: /
User-Agent: Momspider
Disallow: /foo
Disallow: /bar
EOT
@r = $r->rules("www.sn.no:8080");
print "not " if "@r" ne "/foo /bar";
print "ok 9\n";
print "not " if $r->allowed("http://www.sn.no") >= 0;
print "ok 10\n";
print "not " if $r->allowed("http://www.sn.no:8080/foo/gisle");
print "ok 11\n";
sleep(5); # wait until file has expired
print "not " if $r->allowed("http://www.sn.no:8080/foo/gisle") >= 0;
print "ok 12\n";
$r = undef;
print "*** Dump of database ***\n";
tie(%cat, AnyDBM_File, $file, 0, 0644) or die "Can't tie: $!";
while (($key,$val) = each(%cat)) {
print "$key\t$val\n";
}
print "******\n";
untie %cat; # Otherwise the next line fails on DOSish
unlink "$file", "$file.pag", "$file.dir", "$file.db";
# Try open a an emty database without specifying a name
eval {
$r = new WWW::RobotRules::AnyDBM_File undef, $file;
};
print $@;
print "not " unless $@; # should fail
print "ok 13\n";
unlink "$file", "$file.pag", "$file.dir", "$file.db";
|
