1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
---
ADLXBOT: "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)"
ADS_TXT_CRAWLER: "AdsTxtCrawler/1.0"
AMAZONBOT: Amazonbot
ANDERSPINK: "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"
ANTHROPIC_AI: anthropic-ai
APIS_GOOGLE: "APIs-Google; (+https://developers.google.com/webmasters/APIs-Google.html)"
APPLE_BOT: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"
APPLEBOT: Applebot
APPSIGNALBOT: "AppSignalBot/1.0 (+https://appsignal.com)"
ARCHIVEBOT: "ArchiveTeam ArchiveBot/20190617.01 (wpull 2.0.3) and not Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36"
ASK: "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
AWS_ELB: ELB-HealthChecker/1.0
BAIDU: "Baiduspider+(+http://www.baidu.com/search/spider.htm)"
BARKROWLER: "Barkrowler/0.7 (+http://www.exensa.com/crawl)"
BINGBOT: "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
BINGPREVIEW: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b"
BUBING: "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"
BUZZBOT: "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"
BYTESPIDER: Bytespider
CCBOT: CCBot
CHATGPT_USER: ChatGPT-User
CHECKLY: "Checkly/1.0 (https://www.checklyhq.com)"
CHROME_LIGHTHOUSE: "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse"
CIPACRAWLER: "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"
CLAUDE_WEB: Claude-Web
CLAUDEBOT: ClaudeBot
CLOUDFLARE: "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34"
COHERE-AI: cohere-ai
COMMONCRAWL: "CCBot/2.0 (http://commoncrawl.org/faq/)"
COMODO_SSL_CHECKER: "COMODO SSL Checker"
COPYPANTS: "Mozilla/5.0 (compatible; BotPants/1.0; Linux; +info@copypants.com) KHTML/3.5.5 (like Gecko)"
DATAFEEDWATCH: "Datafeedwatch/2.1.x"
DATANYZE: "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
DAUMOA: "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0"
DIFFBOT: Diffbot
DOMAINAREANIMATOR: "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
DOT_BOT: "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"
DUCKDUCKGO: "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"
EZPUBLISH: "eZ Publish Link Validator"
FACEBOOK_BOT: "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
FACEBOOKBOT: FacebookBot
FYREBOT: "Fyrebot/1.0"
GARLIK: "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"
GERMCRAWLER: "GermCrawler"
GO_1.1_PACKAGE_HTTP: "Go 1.1 package http"
GO_HTTP_CLIENT: "Go-http-client"
GOOGLE-EXTENDED: Google-Extended
GOOGLE_BOT: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
GOOGLE_IMAGE_PROXY: "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)"
GOOGLE_PAGE_SPEED_INSIGHTS: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Chrome/22.0.1229 Safari/537.4"
GOOGLE_SHOPPING: "google-xrawler"
GOOGLE_SITE_VERIFICATION: Mozilla/5.0 (compatible; Google-Site-Verification/1.0)
GOOGLE_STACKDRIVER_UPTIME_CHECKS: "GoogleStackdriverMonitoring-UptimeChecks"
GOOGLE_STRUCTURED_DATA_TESTING_TOOL2: "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)"
GOOGLE_STRUCTURED_DATA_TESTING_TOOL: "Mozilla/5.0 (compatible; X11; Linux x86_64; Google-StructuredDataTestingTool; +http://www.google.com/webmasters/tools/richsnippets)"
GPTBOT: GPTBot
GRAPESHOT: "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
HTTRACK: "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"
IMPLISENSEBOT: "ImplisenseBot 1.0"
JAUNT: "Jaunt/1.5"
JOBSEEKER: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) JobBot/5.0 (compatible; +http://www.jobseeker.com.au/bot.html) Safari/538.1"
JOOBLE: "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) Mobile"
KNOWLEDGE_AI: "The Knowledge AI"
LINKDEXBOT: "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)"
LINKEDIN: "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)"
LOAD_TIME_BOT: "Mozilla/5.0 (compatible; LoadTimeBot/0.9; +http://www.loadtime.net/bot.html)"
LTX71: "ltx71 - (http://ltx71.com/)"
MAIL_RU: "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)"
MAPPYDATA: "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)"
MASTODONBOT: "http.rb/3.0.0 (Mastodon/2.2.0; +https://mstdn.io/)"
MAUIBOT: "MauiBot (crawler.feedback+wc@gmail.com)"
MEGAINDEX_RU: "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
MSNBOT: "msnbot/2.0b (+http://search.msn.com/msnbot.htm)"
MSNBOT_MEDIA: "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)"
NETCRAFT2: Netcraft SSL Server Survey - contact info@netcraft.com
NETCRAFT: Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
NEWRELICPINGER: NewRelicPinger/1.0 (12345)
OMGILI: omgili
PAESSLER: Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com); Windows)
PERPLEXITYBOT: PerplexityBot
PR-CY_RU: Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)
PRIVACYAWAREBOT: "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)"
PROXIMIC: "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"
PUINCRAWLER: "Pu_iN Crawler (+http://semanticjuice.com/)"
QUERYSEEKER: "QuerySeekerSpider ( http://queryseeker.com/bot.html )"
QUICKCRAWLER: "Quick-Crawler (+https://www.scrapinghub.com/)"
SCOUT_URL_MONITOR: ScoutURLMonitor/6.2.2
SCRAPY: "Scrapy/0.18.4 (+http://scrapy.org)"
SEMANTICBOT: "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)"
SEO_AUDIT: "Mozilla/5.0 (compatible; seo-audit-check-bot/1.0)"
SEOBILITYBOT: "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)"
SEODIVER: "Mozilla/5.0 (compatible; SEOdiver/1.0; +http://www.seodiver.com/bot)"
SEOKICKS: "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)"
SISTRIX: "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)"
SITECHECK: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) SiteCheck-sitecrawl by Siteimprove.com"
SKYPE: "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"
SOCIALRANKIO: SocialRankIOBot; http://socialrank.io/about
SQUIDER: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 Squider/0.01"
SWIFTYPE: "Swiftbot"
TANGIBLEEBOT: "TangibleeBot"
TEERAID: "Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/)"
TINEYE: "TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)"
TRAACKR: "Traackr.com"
TRENDSMAP: "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)"
UPDOWN_MONITOR: "updown.io daemon 2.2"
WATCHSUMO: "Mozilla/5.0 (compatible) WatchSumo/1.0.0 (http://www.watchsumo.com)"
WEBCEO: "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)"
WHATSAPP: "WhatsApp/2.17.38 Mozilla/5.0 (Linux; U; Android 6.1; en-us; DV Build/Donut) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36"
WIKIDO: "WikiDo/1.1 (http://wikido.com; crawler@wikido.com)"
WOORANK: "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)"
WORIOBOT: "Mozilla/5.0 (compatible; woriobot +http://worio.com)"
YAHOO_AD_MONITORING: "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)"
YAHOO_SLURP: "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
YANDEX_DIRECT: "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)"
YANDEX_METRIKA: "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)"
YANGA: "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"
YOUBOT: YouBot
ZABBIX: "Zabbix"
ZOOMBOT: "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)"
ZOOMINFOBOT: "ZoominfoBot (zoominfobot at zoominfo dot com)"
|
