1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
###############################
# robots.txt for USF.edu
###############################
User-agent: *
# list folders robots are not allowed to index
Disallow: /images
Disallow: /css
Disallow: /aspnet_client
Disallow: /javascripts
Disallow: /snapshots
Disallow: /web-templates
Disallow: /inc
Disallow: /utilities
Disallow: /calendar
Disallow: /it/archive
#
# list specific files robots are not allowed to index
#
Disallow: /Atest.html
Disallow: /jstest.asp
Disallow: /About-USF/contact-send.asp
Disallow: /About-USF/page-not-found.asp
#
# End of robots.txt file
#
###############################
|
