1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
#Add Robots Exclusion Commands for www below this line
User-agent: *
Disallow: /commencement/book/
Disallow: /academicaffairs/commencement/book/
Disallow: /cos/satoshi-takahashi
Disallow: /cos/old
Disallow: /directory*
#allow the RIT Stormcrawler
User-agent: RITStormCrawler
Disallow:
#allow Googlebot
User-agent: Googlebot
User-agent: google
Disallow:
#Removal of /~w-* URLS from search indexes
#We can't do this globally, since many sites are broken and use these URLs publically
Disallow: /~w-oce/
Disallow: /~pltw/
Disallow: /~w-cosold/
|
