1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
#
# Bulk downloading of these pages is not permitted for the purposes of
# making archival copies. This includes browsers such as IE which normally
# permit "off-line viewing" (i.e., bulk downloading). Please read the
# page http://mathworld.wolfram.com/about/terms.html for additional explanation.
#
User-agent: *
Disallow: /
# www.google.com
User-agent: googlebot
Disallow:
# Local lucene indexer
User-agent: nutch-crawler
Disallow:
# alexa
User-agent: ia_archiver
Disallow:
# MSNBot
User-Agent: MSNBot
Disallow:
# disable indexing of random page
User-Agent: *
Disallow: /random.html
|
