1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
|
# robots.txt for http://www.york.ac.uk/
# see <http://info.webcrawler.com/mak/projects/robots/norobots.html> for info.
User-agent: linecker/*
Disallow: /cgi-bin-auth/
Disallow: /cgi-bin-csrv/
Disallow: /test/
User-agent: *
Disallow: /res/thomas/
Disallow: /univ/mis/cfm/gradhunt/
Disallow: /ServerSupport/LinkChecker/
Disallow: /ServerSupport/LinkChecker.TEST/
Disallow: /ServerSupport/UsageStats/
Disallow: /ServerSupport/UsageStats.TEST/
Disallow: /cgi-bin/
Disallow: /cgi-bin-auth/
Disallow: /cgi-bin-csrv/
Disallow: /ftparchive/
Disallow: /test/
Disallow: /~
Disallow: /%7E
Disallow: /%7e
Disallow: /usagestats/
Disallow: /admin/aso/
Disallow: /admin/sro/
Disallow: /admin/uao/ugrad/courses/cupid/?
Disallow: /archaeology/intranet/
Disallow: /commercialservices/catering_and_bars/
Disallow: /media/study-new/
Disallow: /newstudent/
Disallow: /study-new/
Disallow: /study/undergraduate/courses-2017/
Disallow: /univ/coll/
Disallow: /univ/mis/cfm/
Disallow: /univ/mis/fm/
Disallow: /univ/mis/fmport/
Disallow: /univ/mis/fmstudent/
Disallow: /admin/yimshelp/
Disallow: /search/
Disallow: /preview/
|
