1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
User-agent: Slurp
Disallow:
Crawl-delay: 5
User-agent: Yahoo! Slurp
Disallow:
Crawl-delay: 5
User-agent: IRLbot
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: Gaisbot
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: TeragramWebcrawler
Disallow: /
User-agent: Vagabondo
Disallow: /
User-agent: nogoop-HttpClient
Disallow: /
User-agent: Scirus
Disallow: /
User-agent: Speedy Spider
Disallow: /
User-agent: PicoSearch
Disallow: /
User-agent: Plucker
Disallow: /
User-agent: DISCo Pump
Disallow: /
User-agent: Gulliver
Disallow: /
User-agent: vspider
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: Teleport Pro
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: pamuk
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebCapture
Disallow: /
User-agent: Mass Downloader
Disallow: /
User-agent: WebCopy
Disallow: /
User-agent: AWV0.8d
Disallow: /
User-agent: Crescent Internet ToolPak
Disallow: /
User-agent: JOC Web Spider
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: SiteSucker
Disallow: /
User-agent: Webdup
Disallow: /
User-agent: Scooter
Disallow: /
User-agent: Python-urllib
Disallow: /
User-agent: Python
Disallow: /
User-agent: Franklin Locator
Disallow: /
User-agent: CK-SillyDog
Disallow: /
User-agent: PocketHTTP
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: Xaldon WebSpider
Disallow: /
User-agent: WebCapture
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: sun4u
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: Sisi
Disallow: /
Disallow: /
User-agent: lmspider
Disallow: /
User-agent: NP
Disallow: /
User-agent: Robi
Disallow: /
User-agent: Webster Pro
Disallow: /
User-agent: Webster
Disallow: /
User-agent: Zeus
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Twiceler
Disallow: /
User-agent: MJ12bot
Disallow: /
User-Agent: Charlotte
Disallow: /
User-agent: dotbot
Disallow: /
user-agent: AhrefsBot
disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: msnbot
Disallow: /admin
Crawl-delay: 2
User-agent: *
Disallow: /admin
Sitemap: https://ctext.org/siteindex.txt
|
