1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
# Disallow everything until we want to expose the site to external search
# engines.
# 0000-1200 GMT is 6PM to 6AM here
# 4/14/14 Updated for DataExplorer to crawl all state sites.
User-agent: *
Disallow: /cgi-bin
Disallow: /cgi-sys
Disallow: /cd_upload/Search
Disallow: /lawlib/archive/
Disallow: /lawlib/briefs/
Visit-time: 0000-1200
Request-rate: 10
User-agent: Ultraseek
Disallow: /cgi-bin
Disallow: /cgi-sys
Disallow: /cd_upload/Search
User-agent: VSE/1.0
Disallow: /cgi-bin
Disallow: /cgi-sys
|
