1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
User-agent: *
Disallow: /interns/
Disallow: /test/
Disallow: /archives/
Disallow: /www/magic/
Disallow: /*Page=Courses
Disallow: /*Page=DIR
Disallow: /*Page=CatalogCourses
Disallow: /*course_CE
Disallow: /*COURSE
Disallow: /*course=
Disallow: /*Page=R25
Disallow: /*Page=r25
Disallow: /*Page=Faculty
Disallow: /*MM=
Disallow: /talk_to_us/?
Disallow: /sasdoc/
Disallow: /landscape_new
Disallow: /~webshare/
Disallow: /~cliang/
Disallow: /~dgoller/
Disallow: /~davis/davis.mov*
Disallow: /directory/?
Disallow: /~geomorph/gallery/
Disallow: /giee/*category=ucommall
Disallow: /html_templates/
Disallow: /~uvmsnow/*
Disallow: /uvmweb/
# Drupal Do Not Crawl
# --------------------
Disallow: /batch/
Disallow: /content/
Disallow: /entityreference/
Disallow: /factoids/
Disallow: /features/
Disallow: /field_collection/
Disallow: /group/
Disallow: /homepage_feature/
Disallow: /import/
Disallow: /includes/
Disallow: /info_blocks/
Disallow: /misc/
Disallow: /modules/
Disallow: /nodequeue/
Disallow: /panels/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /system/
Disallow: /taxonomy/
Disallow: /themes/
Disallow: /uvmweb/
Disallow: /uvmwebgroups/
Disallow: /views/
Disallow: /sites/default/files/webform/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /user/
Disallow: /users/
|
