1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
# robots.txt for http://www.chrono24.com
User-agent: seekbot
Disallow: /
User-agent: pompos
Disallow: /
User-agent: solomono
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Linguee
Disallow: /
User-agent: LWNutch
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: Riddlerbot
Disallow: /
User-agent: discoverybot
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: endeca webcrawler
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: YisouSpider
Disallow: /
User-agent: HTTrack Website Copier
Disallow: /
User-agent: SentiBot
Disallow: /
User-agent: dotbot
Disallow: /
User-Agent: trendictionbot
Disallow: /
User-Agent: ContextAd Bot
Disallow: /
User-Agent: uMBot
Disallow: /
User-Agent: RubrikkBot
Disallow: /
User-Agent: ADmantX
Disallow: /
User-Agent: Scrapy
Disallow: /
user-agent: SEOdiver
disallow: /
User-agent: plukkie
Disallow: /
User-Agent: XoviBot
Disallow: /
User-agent: WeSEE
Disallow: /
User-agent: psbot
Disallow: /
User-agent: BUbiNG
Disallow: /
User-agent: voltron
Disallow: /
User-agent: proximic
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: BNSBot
Disallow: /
User-agent: MnoGoSearch
Disallow: /
User-agent: um-FC
Disallow: /
User-agent: DomainCrawler
Disallow: /
User-agent: Cliqzbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DomainCrawler
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: BDCbot
Disallow: /
User-agent: SearchmetricsBot
Crawl-delay: 0.5
User-agent: Googlebot
Disallow: /lib/ads/*
Disallow: /lib/generated/js/*
Disallow: */others/ajax_picturemover.htm*
Disallow: */others/popup_*
Disallow: */others/iframe_*
Disallow: */others/banner_link.htm*
Disallow: */others/buynow_link.htm*
Disallow: */others/clicktrack.htm*
Disallow: */others/adIframe*
Disallow: */url/military-watches*
Disallow: *url%2*military-watches*
Disallow: */search/suggest.htm*
Disallow: */search/similar-watches.htm*
Disallow: */search/detail.htm*¬es*
Disallow: */search/detail.htm*&originalNotes*
noindex: */search/detail.htm*¬es*
noindex: */search/detail.htm*&originalNotes*
User-agent: *
Disallow: /lib/ads/*
Disallow: */others/ajax_picturemover.htm*
Disallow: */others/popup_*
Disallow: */others/iframe_*
Disallow: */others/banner_link.htm*
Disallow: */others/buynow_link.htm*
Disallow: */others/clicktrack.htm*
Disallow: */others/adIframe*
Disallow: */search/similar-watches.htm*
Disallow: */search/modelrating.htm*
Disallow: */others/quickreply.htm*
Disallow: */user/activate.htm*
|
