1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
User-agent: *
Disallow: /preview/
Disallow: /redirect/
Disallow: /mailtofriend/
Disallow: /ads/iframe*
Disallow: /profile/
Disallow: /wlive-auth/
Disallow: /embed/
Disallow: /comment/
Disallow: /recherche/?Search
Disallow: /поиск/?Search
Disallow: /tìm-kiếm/?Search
Disallow: /*?page=
Disallow: /*&page=
Disallow: /emissions/titlefilter=
User-agent: Googlebot-News
Disallow: /contenu/ticker/
User-agent: bitlybot
Disallow: /
|
