1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
# $Id: robots.txt,v 1.2 1996/06/26 04:15:40 W.Edmunds Exp $
# robots.txt for http://www.ex.ac.uk/
# see <http://web.nexor.co.uk/mak/doc/robots/norobots.html> for an explanation.
User-agent: htdig # ht://Dig should avoid
Disallow: /cgi-bin/ # Script files
Disallow: /errordocs/ # Local Error docs
Disallow: /spool/ # Local MOMspider output
Disallow: /web/ # Local Web Tools
Disallow: /config/ # Local Web Config
Disallow: /cgi-src/ # Local CGI Source
Disallow: /brad/noindex/ # Brads noindex text stuff
Disallow: /~BBagilho/noindex/ # Brads noindex text stuff
Disallow: /alumni/ # Local alumni
Disallow: /alumnidb/ # Local alumni
Disallow: /iais/pgc/ # Causes looping
Disallow: /its/mwr/ # Web Stats
Disallow: /its/meetingmaker/ # MeetingMaker
Disallow: /its/software/cd/localstuff/ # ITS CD info
Disallow: /its/email2003/ # email2003
Disallow: /tusage/ # Web Stats
Disallow: /library/ # Library web pages have moved
Disallow: /wusage/ # Web Stats
Disallow: /web/ # Common Manuals, etc
#Disallow: /admin/personnel/ # Personnel files cause looping process
Disallow: /projects/admin/ # Personnel files
Disallow: /Projects/admin/ # Personnel files
User-agent: * # All other spiders should avoid
Disallow: /cgi-bin/ # Script files
Disallow: /errordocs/ # Local Error docs
Disallow: /spool/ # Local MOMspider output
Disallow: /web/ # Local Web Tools
Disallow: /config/ # Local Web Config
Disallow: /cgi-src/ # Local CGI Source
Disallow: /noindex/ # noindex stuff
Disallow: /no-index/ # noindex stuff
Disallow: /alumni/ # Local alumni
Disallow: /alumnidb/ # Local alumni
Disallow: /alumnisupporters/contactus/updateyourdetails/ # Local alumni
Disallow: /media/universityofexeter/alumniandsupporters/givingtoexeter/disbursement2013/
Disallow: /its/mwr/ # Web Stats
Disallow: /its/meetingmaker/ # MeetingMaker
Disallow: /its/software/cd/localstuff/ # ITS CD info
Disallow: /its/email2003/ # email2003
Disallow: /its/ipc2003/ # ipc2003
Disallow: /tusage/ # Web Stats
Disallow: /wusage/ # Web Stats
Disallow: /web/ # Common Manuals, etc
Disallow: /proxy/ # Local proxy info
Disallow: /admin/personnel/ # Personnel files cause looping process
Disallow: /projects/admin/ # Personnel files
Disallow: /Projects/admin/ # Personnel files
Disallow: /about/governance/business_areas/fundraising/index.shtml
Disallow: /departments/communication/marketing/aboutus/
Disallow: /undergraduate/2010/ # Archived files
Disallow: /undergraduate/2011/ # Archived files
Disallow: /undergraduate/2012/ # Archived files
Disallow: /undergraduate/2013/ # Archived files
Disallow: /undergraduate/2014/ # Archived files
Disallow: /undergraduate/2015/ # Archived files
Disallow: /undergraduate/2016/ # Archived files
Disallow: /undergraduate/2017/ # Archived files
Disallow: /undergraduate/2018/ # Archived files
Disallow: /undergraduate/2019/ # Archived files
Disallow: /undergraduate/degrees2014/ # Archived files
Disallow: /undergraduate/courses-by-subject-2014/ # Archived files
Disallow: /undergraduate/all-courses-2014/ # Archived files
Disallow: /undergraduate/degrees2015/ # Archived files
Disallow: /undergraduate/courses-by-subject-2015/ # Archived files
Disallow: /undergraduate/all-courses-2015/ # Archived files
Disallow: /undergraduate/degrees2016/ # Archived files
Disallow: /undergraduate/courses-by-subject-2016/ # Archived files
Disallow: /undergraduate/all-courses-2016/ # Archived files
Disallow: /undergraduate/degrees2017/ # Archived files
Disallow: /undergraduate/courses-by-subject-2017/ # Archived files
Disallow: /undergraduate/all-courses-2017/ # Archived files
Disallow: /undergraduate/degrees2018/ # Archived files
Disallow: /undergraduate/courses-by-subject-2018/ # Archived files
Disallow: /undergraduate/all-courses-2018/ # Archived files
Disallow: /undergraduate/degrees2019/ # Archived files
Disallow: /undergraduate/courses-by-subject-2019/ # Archived files
Disallow: /undergraduate/all-courses-2019/ # ArchiDisallow: /postgraduate/pgp2010/ # Archived files
Disallow: /postgraduate/pgp2011/ # Archived files
Disallow: /postgraduate/pgp2012/ # Archived files
Disallow: /postgraduate/pgp2013/ # Archived files
Disallow: /postgraduate/pgp2014/ # Archived files
Disallow: /postgraduate/pgp2015/ # Archived files
Disallow: /postgraduate/pgp2016/ # Archived files
Disallow: /postgraduate/pgp2018/ # Archived files
Disallow: /postgraduate/pgp2019/ # Archived files
Disallow: /codebox/ # Archived files
Disallow: /staff/psr/
Disallow: /cornwall/news/articles/index.html # Hidden responsive news pages
Disallow: /business/news/articles/index.html # Hidden responsive news pages
Disallow: /students/news/articles/index.html # Hidden responsive news pages
Disallow: /research/marine/news/articles/index.html # Hidden responsive news pages
Disallow: /teaching-excellence/news/articles/index.html # Hidden responsive news pages
Disallow: /campusservices/news/articles/index.html # Hidden responsive news pages
Disallow: /study-dev/ # Development site
User-Agent: Googlebot
Disallow: /undergraduate/2010/ # Archived files
Disallow: /undergraduate/2011/ # Archived files
Disallow: /undergraduate/2012/ # Archived files
Disallow: /undergraduate/2013/ # Archived files
Disallow: /undergraduate/2014/ # Archived files
Disallow: /undergraduate/2015/ # Archived files
Disallow: /undergraduate/2016/ # Archived files
Disallow: /undergraduate/2017/ # Archived files
Disallow: /undergraduate/2018/ # Archived files
Disallow: /undergraduate/2019/ # Archived files
Disallow: /undergraduate/degrees2014/ # Archived files
Disallow: /undergraduate/courses-by-subject-2014/ # Archived files
Disallow: /undergraduate/all-courses-2014/ # Archived files
Disallow: /undergraduate/degrees2015/ # Archived files
Disallow: /undergraduate/courses-by-subject-2015/ # Archived files
Disallow: /undergraduate/all-courses-2015/ # Archived files
Disallow: /undergraduate/degrees2016/ # Archived files
Disallow: /undergraduate/courses-by-subject-2016/ # Archived files
Disallow: /undergraduate/all-courses-2016/ # Archived files
Disallow: /undergraduate/2016/ # Archived files
Disallow: /undergraduate/degrees2017/ # Archived files
Disallow: /undergraduate/courses-by-subject-2017/ # Archived files
Disallow: /undergraduate/all-courses-2017/ # Archived files
Disallow: /undergraduate/degrees2018/ # Archived files
Disallow: /undergraduate/courses-by-subject-2018/ # Archived files
Disallow: /undergraduate/all-courses-2018/ # Archived files
Disallow: /undergraduate/degrees2019/ # Archived files
Disallow: /undergraduate/courses-by-subject-2019/ # Archived files
Disallow: /undergraduate/all-courses-2019/ # Archived files
Disallow: /postgraduate/pgp2011/ # Archived files
Disallow: /postgraduate/pgp2012/ # Archived files
Disallow: /postgraduate/pgp2013/ # Archived files
Disallow: /postgraduate/pgp2014/ # Archived files
Disallow: /postgraduate/pgp2015/ # Archived files
Disallow: /postgraduate/pgp2016/ # Archived files
Disallow: /postgraduate/pgp2018/ # Archived files
Disallow: /postgraduate/pgp2019/ # Archived files
Disallow: /cornwall/news/articles/index.html # Hidden responsive news pages
Disallow: /business/news/articles/index.html # Hidden responsive news pages
Disallow: /students/news/articles/index.html # Hidden responsive news pages
Disallow: /research/marine/news/articles/index.html # Hidden responsive news pages
Disallow: /teaching-excellence/news/articles/index.html # Hidden responsive news pages
Disallow: /study-dev/ # Development site
Sitemap: https://www.exeter.ac.uk/sitemap.xml
|
