1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
# robots.txt - for an explanation, see:
# http://www.robotstxt.org/
# http://en.wikipedia.org/wiki/Robots_exclusion_standard
# For all sites
User-agent: *
Disallow: /cgi-bin/ # cgi-bin scripts
Disallow: /php-bin/ # php-bin scripts
Disallow: /phpapps/ # phpapps global scripts
Disallow: /private/ # private directory
Disallow: /test/ # test directory
Disallow: /weblog/ # weblog report directory
# Enable only for Dept. Template System sites (obsolete)
# User-agent: *
# Disallow: /admin/ # admin directory
# Disallow: /data/ # data directory
# Disallow: /include/ # include directory
# Works only for some spiders (e.g. MSN)
User-agent: *
Crawl-delay: 5 # Wait 5 seconds between requests
|
