1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
#Updated on 4/9/2018
#User-agent Disallow List with URL Link
User-agent: ZoomSpider #http://www.wrensoft.com/zoom/support/useragent.html
Disallow: /
User-agent: Exabot #http://www.exalead.com/search/webmasterguide
Disallow: /
User-agent: ICC-Crawler #http://www.nict.go.jp/en/univ-com/plan/crawl.html
Disallow: /
User-agent: ichiro #http://search.goo.ne.jp/option/use/sub4/sub4-1/
Disallow: /
User-agent: Yeti #http://www.botopedia.org/user-agent-list/search-bots/item/340-yeti-naverbot
Disallow: /
User-agent: ssearch_bot #http://www.semantissimo.de/
Disallow: /
User-agent: ChangeDetection #http://www.changedetection.com/bot.html
Disallow: /
User-agent: Job Roboter Spider #http://www.webintegration.at/jobroboter_suchmaschine
Disallow: /
User-agent: Xenu Link Sleuth #http://home.snafu.de/tilman/xenulink.html
Disallow: /
User-agent: Hatena Antenna #(BAD)Unknown URL
Disallow: /
User-agent: linkdexbot #http://www.linkdex.com/m/bots/
Disallow: /
User-agent: FacebookExternalHit #http://www.facebook.com/externalhit_uatext.php
Disallow: /
User-agent: Slackbot #https://api.slack.com/robots
Disallow: /
User-agent: Qwantify #https://www.qwant.com/
Disallow: /
User-agent: FeedDemon #http://www.feeddemon.com/
Disallow: /
User-agent: Dotbot #https://moz.com/researchtools/ose/dotbot
Disallow: /
User-agent: SemrushBot #http://www.semrush.com/bot/
Disallow: /
User-agent: SeznamBot #http://napoveda.seznam.cz/cz/seznambot/
Disallow: /
User-agent: Feedly #http://www.feedly.com/fetcher.html
Disallow: /
User-agent: MJ12bot #http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
Disallow: /
User-agent: YisouSpider #(BAD)http://user-agents.me/crawler/yisouspider
Disallow: /
User-agent: magpie-crawler #https://www.brandwatch.com/how-it-works/
Disallow: /
User-agent: Alexabot #https://support.alexa.com/hc/en-us/articles/200462340-Certification-Crawler-Information
Disallow: /
User-agent: Speedy Spider #(BAD)http://www.entireweb.com/
Disallow: /
User-agent: GarlikCrawler #(BAD)http://www.garlik.com/
Disallow: /
User-agent: AhrefsBot #https://ahrefs.com/robot
Disallow: /
User-agent: Slurp #http://www.useragentstring.com/Yahoo!%20Slurp_id_75.php
Disallow: /
User-agent: YandexBot #http://help.yandex.com/search/robots/agent.xml
Disallow: /
User-agent: sogou spider #http://www.sogou.com/docs/help/webmasters.htm#07
Disallow: /
User-agent: maxum #http://www.informedusa.com/t/phantom7.15.html
Disallow: /
User-agent: curious george #http://www.analyticsseo.com/the-analytics-seo-crawler-curious-george/
Disallow: /
User-agent: WeSEE #(BAD)http://www.wesee.com/bot/
Disallow: /
User-agent: rogerbot #http://moz.com/help/pro/what-is-rogerbot-
Disallow: /
User-agent: dotbot #https://moz.com/researchtools/ose/dotbot
Disallow: /
User-agent: Y!J-ASR #https://help.yahoo.com/kb/search/SLN22600.html?impressions=true
Disallow: /
User-agent: Y!J-BSC #https://help.yahoo.com/kb/search/SLN22600.html?impressions=true
Disallow: /
User-agent: ramBot xtreme x.x #(BAD)Unknown URL
Disallow: /
User-agent: Daumoa #(BAD)https://www.webmasterworld.com/search_engine_spiders/3895299.htm
Disallow: /
User-agent: Who.is Bot #https://www.webmasterworld.com/search_engine_spiders/4427797.htm
Disallow: /
User-agent: psbot #http://www.picsearch.com/bot.html
Disallow: /
User-agent: yacybot #http://yacy.net/bot.html
Disallow: /
User-agent: Nutch #http://nutch.apache.org/bot.html
Disallow: /
User-agent: BUbiNG #http://law.di.unimi.it/software.php#buging
Disallow: /
User-agent: bingbot #http://www.bing.com/bingbot.htm
Disallow: /thorproduct.cfm*
User-agent: bingbot #http://www.bing.com/bingbot.htm
Disallow: /ThorProduct.cfm*
User-agent: Mappy #http://mappydata.net/#eng
Disallow: /
#User-agent Crawl Delay Disallow
User-agent: *
Crawl-delay: 30
Disallow: /honey/
Disallow: /thorcat/
Disallow: /Thorcat/
Disallow: /search/
Disallow: /thorsearch.cfm*
Disallow: /advSearch.cfm
Disallow: /advSearchDetail.cfm
Disallow: /*.dxf$
Disallow: /*.sldrpt$
Disallow: /*.step$
Disallow: /*.vbi$
Disallow: /*.zip$
Disallow: /*.eprt$
Disallow: /*.bak$
Disallow: /*.exe$
Disallow: /images/catalog/
Disallow: /trackClick.cfc
Disallow: /NewGroupPage9.cfm?ObjectGroup_ID=5569
Disallow: /Navigation.cfm?Guide_ID=2184
Disallow: /newgrouppage9_pf.cfm*
Disallow: /newgrouppage9pf.cfm*
Disallow: /cfc/familyPage/priceRequest.cfc*
Disallow: /sitemap.cfm*
Disallow: /action.cfm*
Disallow: /RoHS_cert.cfm*
Disallow: /_sd.cfm*
Disallow: /AJAX/
Disallow: /CFC/
Disallow: /cfc/
Disallow: /JS/
Disallow: /CFIDE/
Disallow: /JSON/
Disallow: /*?*CurrencySelect=*
Disallow: /*?*Language=*
Disallow: /*?*isPreview=*
Disallow: /*?*ispreview=*
Disallow: /*.cfc$
Disallow: /contentEditor/
Disallow: /contenteditor/
Disallow: /rest/library/
Disallow: /_volPricing.cfm
Disallow: /_volpricing.cfm
Disallow: /RoHS_cert.cfm
Disallow: /rohs_cert.cfm
|
