1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
# robots.txt for Slashdot.org
# $Id$
# "Any empty [Disallow] value, indicates that all URLs can be retrieved.
# At least one Disallow field needs to be present in a record."
User-agent: Mediapartners-Google
Disallow:
User-agent: Googlebot
Disallow: /firehose.pl
Disallow: /submit.pl
Disallow: /comments.pl
Disallow: /users.pl
Disallow: /zoo.pl
Disallow: firehose.pl
Disallow: submit.pl
Disallow: comments.pl
Disallow: users.pl
Disallow: zoo.pl
Disallow: /~
Disallow: ~
User-agent: Slurp
Disallow:
User-agent: Yahoo-NewsCrawler
Disallow:
User-Agent: msnbot
Disallow:
User-agent: *
Disallow: /authors.pl
Disallow: /index.pl
Disallow: /comments.pl
Disallow: /firehose.pl
Disallow: /journal.pl
Disallow: /messages.pl
Disallow: /metamod.pl
Disallow: /users.pl
Disallow: /search.pl
Disallow: /submit.pl
Disallow: /pollBooth.pl
Disallow: /pubkey.pl
Disallow: /topics.pl
Disallow: /zoo.pl
Disallow: /palm
Disallow: /slashdot-it.pl
Disallow: slashdot-it.pl
Disallow: authors.pl
Disallow: index.pl
Disallow: comments.pl
Disallow: firehose.pl
Disallow: journal.pl
Disallow: messages.pl
Disallow: metamod.pl
Disallow: users.pl
Disallow: search.pl
Disallow: submit.pl
Disallow: pollBooth.pl
Disallow: pubkey.pl
Disallow: topics.pl
Disallow: zoo.pl
Disallow: /~
Disallow: ~
|
