1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
User-agent: *
Disallow:/login/
Disallow:/logout/
Disallow:/register/
Disallow:/regPage1/
Disallow:/loginMembersOnly/
Disallow:/tieredRegPage/
Disallow:/vgn-ext-templating/
Disallow:/page/etpk_0/
Disallow:/application/internalSearch/
User-Agent: MediaPartners-Google
Allow: /search/
Allow: /googleNews.xml
User-agent: Updownerbot
Disallow: /
User-agent: 008
Disallow: /
User-Agent: bingbot
Disallow: /search/query
User-Agent: msnbot
Disallow: /search/query
User-agent: bingbot
Crawl-delay: 3
User-agent: msnbot
Crawl-delay: 3
|
