1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
# robots.txt for www.npr.org
# Changes are tracked in npr_seamus
User-agent: *
Disallow: /mpx/
Disallow: /cgi-bin
Disallow: /ramfiles/
Disallow: /oauth2/
Disallow: /account/
Disallow: /*.smil
Disallow: /*.asx
Disallow: /*.ram
Disallow: /*.wav
Disallow: /*.rmm
Disallow: /*.js
Disallow: /*.au
Disallow: /stations/force/force_localization.php?
Disallow: /rundowns/segment.php?
Disallow: /templates/search/
Disallow: /search
Disallow: /2013/03/21/174840895/
Disallow: /sections/ombudsman/2008/01/frequently_asked_questions_1.html
Disallow: /sections/health-shots/2013/03/11/173816690/new-voices-for-the-voiceless-synthetic-speech-gets-an-upgrade
|
