1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
#
# robots.txt for http://www.w3.org/
#
# $Id: robots.txt,v 1.84 2019/03/22 22:48:24 gerald Exp $
#
# For use by search.w3.org
User-agent: W3C-gsa
Disallow: /Out-Of-Date
User-agent: W3T_SE
Disallow: /Out-Of-Date
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)
Disallow: /
# W3C Link checker
User-agent: W3C-checklink
Disallow:
# Applebot continues to make hundreds of thousands of reqs/day for this area
# even though it has been returning permanent redirects for years
User-agent: Applebot
Disallow: /People/domain/
# the following settings apply to all bots
User-agent: *
# Blogs - WordPress
# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization
Disallow: /*/wp-admin/
Disallow: /*/wp-includes/
Disallow: /*/wp-content/plugins/
Disallow: /*/wp-content/cache/
Disallow: /*/wp-content/themes/
Disallow: /blog/*/trackback/
Disallow: /blog/*/feed/
Disallow: /blog/*/comments/
Disallow: /blog/*/category/*/*
Disallow: /blog/*/*/trackback/
Disallow: /blog/*/*/feed/
Disallow: /blog/*/*/comments/
Disallow: /blog/*/*?
Disallow: /community/trackback/
Disallow: /community/feed/
Disallow: /community/comments/
Disallow: /community/category/*/*
Disallow: /community/*/trackback/
Disallow: /community/*/feed/
Disallow: /community/*/comments/
Disallow: /community/*/category/*/*
Disallow: /community/*?
Disallow: /Consortium/Offices/trackback/
Disallow: /Consortium/Offices/feed/
Disallow: /Consortium/Offices/comments/
Disallow: /Consortium/Offices/category/*/*
Disallow: /Consortium/Offices/*/trackback/
Disallow: /Consortium/Offices/*/feed/
Disallow: /Consortium/Offices/*/comments/
Disallow: /Consortium/Offices/*?
# Wikis - Mediawiki
# https://www.mediawiki.org/wiki/Manual:Robots.txt
Disallow: /wiki/index.php?
Disallow: /wiki/index.php/Help
Disallow: /wiki/index.php/MediaWiki
Disallow: /wiki/index.php/Special:
Disallow: /wiki/index.php/Template
Disallow: /wiki/skins/
Disallow: /*/wiki/index.php?
Disallow: /*/wiki/index.php/Help
Disallow: /*/wiki/index.php/MediaWiki
Disallow: /*/wiki/index.php/Special:
Disallow: /*/wiki/index.php/Template
Disallow: /*/wiki/Special:
# various other access-controlled or expensive areas
Disallow: /2004/ontaria/basic
Disallow: /Team/
Disallow: /Project
Disallow: /Web
Disallow: /Systems
Disallow: /Out-Of-Date
Disallow: /2005/06/blog/
Disallow: /2004/08/W3CTalks
Disallow: /2007/11/Talks/search
Disallow: /People/all/
Disallow: /RDF/Validator/ARPServlet
Disallow: /RDF/Validator/rdfval
Disallow: /2003/03/Translations/byLanguage
Disallow: /2003/03/Translations/byTechnology
Disallow: /2005/11/Translations/Query
Disallow: /2000/06/webdata/xslt
Disallow: /2000/09/webdata/xslt
Disallow: /2005/08/online_xslt/xslt
Disallow: /Bugs/
Disallow: /Search/Mail/Public/
Disallow: /2006/02/chartergen
Disallow: /2004/01/pp-impl
Disallow: /Consortium/supporters
Disallow: /2007/08/pyRdfa/
Disallow: /2012/pyRdfa/extract
Disallow: /WAI/PF/comments/
Disallow: /participate/conferences.xml
Disallow: /scripts/
Disallow: /2005/01/yacker/
Disallow: /2005/01/yacker?
Disallow: /2003/09/nschecker?
Disallow: /2005/07/pubrules?
Disallow: /ns/hydra/console/?
Disallow: /2007/08/grddl/?
Disallow: /2009/07/webidl-check?
Disallow: /RDF/Validator/ARPServlet?
Disallow: /2000/06/webdata/xsv?
Disallow: /2000/09/webdata/xsv?
Disallow: /Style/CSS/members.be/
Disallow: /services
Disallow: /*,*
# WAI indexing
Disallow: /WAI/beta/
Disallow: /WAI/ut1/
Disallow: /WAI/ut2/
Disallow: /WAI/ut3/
Disallow: /WAI/ut4/
# Disallow: /WAI/EO/Drafts/
Disallow: /WAI/drafts/
|
