1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
User-agent: *
Disallow: /cgi-bin/
Disallow: /comment
Disallow: /contact/
Disallow: /dev/
Disallow: /esu/unitprogcat/
Disallow: /eopro/
Disallow: /feedback/
Disallow: /fluff/
Disallow: /imagelib/
Disallow: /logs/
Disallow: /news/news-results.html
Disallow: /news/news-search.html?
Disallow: /news/2001/clearing.html
Disallow: /oenology
Disallow: /prospectus/postgraduate/2013/
Disallow: /prospectus/postgraduate/2014/
Disallow: /prospectus/undergraduate/2014/
Disallow: /slink/
Disallow: /sso/login
Disallow: /test/
Disallow: /webcam/sport/archive/
Disallow: /biology/people/person/
Disallow: /composites/media/ICCM17/
Disallow: /composites/media/icons/
Disallow: /composites/media/icshm/
Disallow: /Control_Panel/
Disallow: /Depts/CAS/
Disallow: /depts/CAS/
Disallow: /geography/people/person/
Disallow: /history/customs/resources/
Disallow: /history/customs/datasets/
Disallow: /is/cgi-bin/
Disallow: /is/comment
Disallow: /research/funding/salary_costing/
Disallow: /study/postgraduate/search/research_interests
Disallow: /study/postgraduate/search/search_results
Disallow: /study/undergraduate/search/search_results
Disallow: /media-library/sites/alspac/migrated/documents/
Disallow: /it-services/advice/operatingsystems/mac/archive
Disallow: /it-services/applications/collaboration/office/outlook/
# AFAIK only Googlebot understands these
Disallow: /*/comment$
Disallow: /*/comment?url
Disallow: /*/feedback$
Disallow: /*/news-results.html
Disallow: /*/news-search.html?
Disallow: /*/safe/
|
