1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
User-agent: Googlebot-Mobile
Disallow: /
User-agent: AdsBot-Google
Disallow: /
User-agent: Yahoo Pipes 2.0
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: asterias
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-Agent: Oracle Secure Enterprise Search
Disallow: /
# User-agent: Baiduspider
# Disallow: /
# User-agent: Yandex
# Disallow: /
User-Agent: Mail.RU_Bot
Disallow: /
User-agent: YisouSpider
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: Daumoa
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: NerdyBot
Disallow: /
User-agent: 008
Disallow: /
User-agent: ccbot
Disallow: /
User-agent: discobot
Disallow: /
User-agent: OmegaSeek
Disallow: /
User-agent: discoverybot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: wotbox
Disallow: /
User-agent: yacy
Disallow: /
User-agent: Twitterbot
Disallow:
User-agent: Blekkobot
Disallow: /
User-agent: Abonti
Disallow: /
user-agent: CrystalSemanticsBot
Disallow: /
User-agent: *
Disallow: /etext
Disallow: /dirs/etext
Disallow: /dirs/1
Disallow: /dirs/2
Disallow: /dirs/3
Disallow: /dirs/4
Disallow: /dirs/5
Disallow: /dirs/6
Disallow: /dirs/7
Disallow: /dirs/8
Disallow: /dirs/9
Disallow: /catalog/world/ # old search engine
Disallow: /ebooks/search # new search engine
Disallow: /ebooks/send/ # dropbox et al.
Disallow: /ratelimiter
Disallow: /zipcat
Disallow: /addresses/ # don't remove, ask webmaster
Disallow: /author/
Disallow: /internal/ # private
Disallow: /etext/magnet # against brain damaged robots
Disallow: /cache/ # on-the-fly generated files
Disallow: /cdproject/
Disallow: /w/ # wiki edit pages etc.
Disallow: /wiki/Special:Search
Disallow: /wiki/Special:Random
Crawl-delay: 5
Sitemap: http://www.gutenberg.org/ebooks/sitemaps/
|
