1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
#
# Robot exclusion file for K-State
#
# If you are a web publisher at K-State, and would like to have
# some of your documents not available to robots, please contact
# webmaster@k-state.edu.
#
# For more information about robots and what they do, see
# http://www.robotstxt.org/ and http://en.wikipedia.org/wiki/Robots.txt
User-agent: *
Disallow: /KSU_resources/
Disallow: /dev/
Disallow: /test/
Disallow: /archive/
# Don't index the low-graphics version -- the standard page is already indexed.
Disallow: /quick.html
# Disallow the list of all updated files.
Disallow: /search/new/autonew.html
# Disallow anything that has a URL redirect to another location.
# They are indexed under their new name rather than the old.
Disallow: /Directories/ksudepts.html
Disallow: /its/development/
Disallow: /dept_links.html
Disallow: /gen_interest.html
Disallow: /mhk_area_info.html
Disallow: /courses/web-classes.html
Disallow: /creating-home-page.html
Disallow: /aboutuni/copyright.html
Disallow: /aboutuni/help.html
Disallow: /aboutuni/new
Disallow: /wp-query.html
Disallow: /mainemnu.html
Disallow: /welcome.html
Disallow: /admit/viewbook
#Disallow: /internet2
Disallow: /cns_info
#Disallow: /infotech
Disallow: /training
Disallow: /classes
Disallow: /course
Disallow: /ksufcu
Disallow: /sthlt
Disallow: /afact
Disallow: /audit
Disallow: /hrser
Disallow: /jl57
Disallow: /recc
Disallow: /isle
Disallow: /smf
Disallow: /cfc
Disallow: /reg/
Disallow: /blog-feeds/
Disallow: /univpub/
Disallow: /registrar/backup/
Disallow: /registrar/a_r/backup/
Disallow: /registrar/c_d/oldgrad/
Disallow: /registrar/dars/slideshow/backup/
Disallow: /registrar/enroll/backup/
Disallow: /registrar/faqs/backup/
Disallow: /registrar/ferpa/backup/
Disallow: /registrar/t_v/backup/
Disallow: /registrar/unpublished/
Disallow: /registrar/internal/
Disallow: /registrar/statistics/tabs/
Disallow: /safety-new/
# Data files that should never be retrieved
Disallow: /Directories/lsdcodes-save.html
Disallow: /Directories/qsearch.data
Disallow: /Directories/qsearch-save.data
Disallow: /cns/newsletter/
Disallow: /InfoTech/news/tuesday/archive/2001
Disallow: /InfoTech/news/tuesday/archive/2002
Disallow: /InfoTech/news/tuesday/archive/2003
Disallow: /cns/announce/199
Disallow: /cns/announce/2001
Disallow: /cns/announce/2002
Disallow: /cns/announce/2003
Disallow: /elp/univdept
Disallow: /courses/fall2020/
# Remove all eventview because it might be causing load problems and will soon
# be replaced.
Disallow: /cgi-bin/eventview
# No reason to index our Maven repository
Disallow: /repository/
Disallow: /maps2011/
# Somehow Bing started indexing using a path after the filename, e.g.:
# /media/k-statement/main.html/vol30/vol32/vol31/vol31/vol32/vol30/vol30/vol31/vol30/vol30/vol31/vol31/
Disallow: /media/k-statement/main.html/vol
Disallow: /media/k-statement/main.html/aboutus
# Google has gotten ahold of https://www.k-state.edu/uas
Disallow: /uas
# Lots of keywords in this data file:
Disallow: /admissions/js/majors.js
# Old PDF guides
Disallow: /admissions/guides/
# Parking instructions that apply to a small subset of people
Disallow: /admissions/cvparking/
# Archived department heads info that has confused people
Disallow: /dh/archive/
Disallow: /media/WEB
Disallow: /media/Web
Disallow: /ksis/intranet
Disallow: /campaign/
Disallow: /today/tipsheet/
Disallow: /today/tuesday/
Disallow: /challenges/development/
# No reason to index these data feed files
Disallow: /calendar/export/
Disallow: /behemoth/export/
# Old names -- content is available on a new URL.
Disallow: /isis/
#Disallow: /nonviolence/
# Mediasite administratio, per Brandon, INC0298413
Disallow: /mediasite/admin/
# 2017 homepage preview
Disallow: /preview/
|
