1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
# robots.txt for http://www.abc.net.au/ -- ABC Online
User-agent: *
Disallow: /classic/contact/concerts.htm
Disallow: /classic/contact/default.htm
Disallow: /classic/contact/eventsdiary.htm
Disallow: /classic/contact/formerror.htm
Disallow: /classic/contact/formthanks.htm
Disallow: /classic/contact/general.htm
Disallow: /classic/contact/limelight.htm
Disallow: /classic/contact/mailinglist.htm
Disallow: /classic/contact/music.htm
Disallow: /classic/contact/presenter.htm
Disallow: /classic/contact/website.htm
Disallow: /classic/contact/word.htm
Disallow: /xmlcontent/
Disallow: /classicfm/
#OPSSD-340 2015/5/5
Disallow: /iview/
#INNG-46: 2014-12-30
Disallow: /site-archive/
# Added for corporate communications, as they have migrated to a new site
Disallow: /corp/
Disallow: /contact/
# Added for Homepage Beta, prevent indexing during public beta
Disallow: /homepage/2013/
Disallow: /beta/
# Added for WCMS Tennent testing, not a public
Disallow: /abc4000/
User-agent: Googlebot
Crawl-delay: 5
User-agent: Googlebot-Image
Crawl-delay: 5
User-agent: MSNBot
Crawl-delay: 5
User-agent: Slurp
Crawl-delay: 5
|
