1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
# robots.txt file for http://www.utsa.edu
# This character signifies a comment tag
User-agent: webcrawler
Disallow:
User-agent: *
Disallow: /blueprint
Disallow: /lead/adm/
Disallow: /ucomm/experts/
Disallow: /1
Disallow: /forms
Disallow: /logs
Disallow: /cgi
Disallow: /Cfide
Disallow: /today/admin
Disallow: /today/new
Disallow: /roadrunners/admin
Disallow: /internet/iom
Disallow: /adminresources
Disallow: /oracledoc
Disallow: /banner2000
Disallow: /cfdocs
Disallow: /cgi-bin
Disallow: /scsstaff
Disallow: /tlc/
Disallow: /cftemp
Disallow: /java
Disallow: /Cfdocs
Disallow: /_derived
Disallow: /_fpclass
Disallow: /_private
Disallow: /_themes
Disallow: /_vti_cnf
Disallow: /_vti_log
Disallow: /_vti_pvt
Disallow: /_vti_script
Disallow: /_vti_txt
Disallow: /_vit_inf
Disallow: /classes
Disallow: /wams/today/
Disallow: /cias/
Disallow: /sga/inc/
Disallow: /today/2016test/
Disallow: /index1.html
Disallow: /ofponline/ol/ct-training.html
Disallow: /concern
|
