1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
User-agent: *
Crawl-delay: 2
Disallow: /wgbh/pages/frontline/digitalnation/%3F
Disallow: /wgbh/pages/frontline/digitalnation/%3F*
Disallow: /wgbh/frontline/*elqat*
Disallow: /wgbh/pages/frontline/*elqat*
Disallow: /wgbh/pages/frontline/bushswar*
Disallow: /independentlens/getinvolved/cinema/
Disallow: /cgi-bin/*
Disallow: /cgi-registry/*
Disallow: /survey/
Disallow: /whatson/stations/
Disallow: /whatson/schedulex/
Disallow: /whatson/pbsyou/
Disallow: /whatson/dbs/
Disallow: /remotecontrol/bestpractices/prototype/
Disallow: /remotecontrol/bestpractices/jl-prototype/
Disallow: /pressroom/
Disallow: /als/shakespeare_pilot/
#Disallow: /.
Disallow: /newshour/bb/military/military.html/
Disallow: /wgbh/pages/roadshow/appraiseit/index.html/
Disallow: /producers/webmanual/
Disallow: /devcon2007/
Disallow: /smcot/
Disallow: /pbsholidaycard/
Disallow: /gpra/
Disallow: /tlqa/
Disallow: /remote.control/
Disallow: /teachers/connect/
Disallow: /engage/sites/all/files/blog_images/monkeys.gif
Disallow: /teachers/edcar/
Disallow: /teachers/dll_research/
Disallow: /peerconnection/feeds/
Disallow: /teachers/innovators/toolkit/
Disallow: /mt4/
Disallow: /klru/mt4/
Disallow: /newshour/mt4/
Disallow: /mediashift/mt4/
Disallow: /kcet/mt4/
Disallow: /newshour/mt3/
Disallow: /now/mt3/
Disallow: /itvs/mt3/
Disallow: /pbs/mt3/
Disallow: /pbs/mt3o4/
Disallow: /pbs/mt4/
Disallow: /pbs/teachers/mt3/
Disallow: /pbs/kids/mt3/
Disallow: /pbs/kids/mt4/
Disallow: /pbs/src/mt4/
Disallow: /pbs/parents/mt4/
Disallow: /pbsi/mt3/
Disallow: /moyers/mt3/
Disallow: /moyers/mt4/
Disallow: /nbr/mt3/
Disallow: /nbr/mt4/
Disallow: /cringely/mt3/
Disallow: /cringely/mt4/
Disallow: /wgbh/mt4/
Disallow: /wnet/expose/mt4/
Disallow: /wnet/mt4/
Disallow: /opb/mt4/
Disallow: /search/search_results.html?q=*
Disallow: /wgbh/nova/search/
Disallow: /wgbh/nova/search/results/*
Disallow: /wgbh/nova/beta/search/results/*
Disallow: /nbr/info/search.html?*
Disallow: /independentlens/video/index.php?*
Disallow: /independentlens/video/?filter=*
Disallow: /wnet/religionandethics/*/index.html?*
Disallow: /wnet/religionandethics/search_results.html*
Disallow: /teachers/search/resources/?q=*
Disallow: /teacherline/catalog/browse/
Disallow: /thewar/search_results.php?*
Disallow: */emailthispage.html?*
Disallow: /pov_mt/mt4/
Disallow: /newshour/indepth_coverage/middle_east/iraq/honorroll/search.php?*
Disallow: /includes/
Disallow: */emailthispage.html
Disallow: /about/faq/station-finder/?state-text=*
Disallow: /about/faq/station-finder/?zip-text=*
Disallow: /wgbh/nova/search-results.html
Disallow: /wgbh/americanexperience/about/search/search_results.html
Disallow: /moyers/journal/about/search_google.html
Disallow: /foundation/contact/
Disallow: *?replytocom
Disallow: /social_media/
Disallow: /independentlens/film-lists/watch-online/
Disallow: /show/koth-show/
Sitemap: http://www.pbs.org/mercy-street/sitemap.xml
Sitemap: https://www.pbs.org/newshour/sitemaps
User-Agent: Baiduspider
Disallow: /independentlens/getinvolved/cinema/
Disallow: /independentlens/films/watch-online/*
|
