diff --git a/bots/bots.go b/bots/bots.go
index 1b8d58c..e3f5a83 100644
--- a/bots/bots.go
+++ b/bots/bots.go
@@ -1,467 +1,447 @@
// This package helps the backend determine which clients are bots or crawlers.
// In Write.as, this is used to prevent certain things when viewing posts, like
// incrementing the view count.
package bots
import "strings"
var bots = map[string]bool{
"ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true,
"AcademicBotRTU (https://academicbot.rtu.lv; mailto:caps@rtu.lv)": true,
"AddThis.com robot tech.support@clearspring.com": true,
"AdsBot-Google (+http://www.google.com/adsbot.html)": true,
"AdsTxtCrawlerTP/1.2": true,
"Alwyzbot/1.0": true,
"Amazon-Advertising-ad-standards-bot/1.0": true,
"Apple Color Emoji": true,
"AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)": true,
"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)": true,
"BeeperBot/0 Matrix-Media-Repo/1": true,
"bidswitchbot/1.0": true,
"bitlybot": true,
"BrightEdge Crawler/1.0 (crawler@brightedge.com)": true,
"BSbot 1.1 (monthly copyright check - html/js/css)": true,
"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true,
"CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)": true,
"Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"cis455crawler": true,
"Clickagy Intelligence Bot v2": true,
"COMODO SSL Checker": true,
"companyspotter/1.0.0.0 (robot@companyspotter.com)": true,
"crawler_eb_germany_2.0": true,
"crawlernutchtest/Nutch-1.9": true,
"CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true,
"CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true,
"datebot": true,
"DeadYetBot/1.0 (+http://deadyet.lol)": true,
"Deskyobot/1.0 (+https://www.deskyo.com/bot)": true,
"DingTalkBot-LinkService/1.0 (+https://open-doc.dingtalk.com/microapp/faquestions/ftpfeu)": true,
"DisqusAdstxtCrawler/0.1 (+https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide)": true,
"dj-research/Nutch-1.11 (analytics@@demandjump.com)": true,
"DnBCrawler-Analytics": true,
"DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true,
"DomainCrawler/1.0": true,
"DomainStatsBot/1.0 (https://domainstats.com/pages/our-bot)": true,
"drinespider/Nutch-1.19 (D-RINE Spider; www.d-rine.com/search/about; www.d-rine.com/contact)": true,
"DuckDuckBot-Https/1.1; (+https://duckduckgo.com/duckduckbot)": true,
"ExactSeekCrawler/1.0": true,
"EyeMonIT Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36": true,
"FediCrawl/1.0": true,
"fedistatsCrawler/1.0": true,
"FeedlyBot/1.0 (http://feedly.com)": true,
"GG PeekBot 2.0 ( https://www.gg.pl/ https://www.gg.pl/info/praca/ )": true,
"Gigabot/1.0": true,
"GNUsocialBot 2.0.1-beta0 - https://gnusocial.rocks": true,
"Go 1.1 package http": true,
"Go-http-client/2.0": true,
"Google-Adwords-Instant (+http://www.google.com/adsbot.html)": true,
"Google-Display-Ads-Bot": true,
"Googlebot": true,
"Greppr Bot 1.0/Nutch-1.19": true,
"GroupMeBot/1.0": true,
"GuzzleHttp/7": true,
"Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot": true,
"HubSpot Connect 2.0 (http://dev.hubspot.com/) (namespace: hs_web_crawler) - RoadsModelsJobs-hubspot-53-domain-web-crawl": true,
"ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)": true,
"IABot/2.0 (+https://meta.wikimedia.org/wiki/InternetArchiveBot/FAQ_for_sysadmins) (Checking if link from Wikipedia is broken and needs removal)": true,
"IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)": true,
"LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)": true,
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)": true,
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true,
"LivelapBot/0.2 (http://site.livelap.com/crawler)": true,
"LSSRocketCrawler/1.0 LightspeedSystems": true,
"magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)": true,
"Mastodon server indexer": true,
"MBCrawler/1.0 (https://monitorbacklinks.com/robot)": true,
"Mediatoolkitbot (complaints@mediatoolkit.com)": true,
"Mediatoolkitbot (info@mediatoolkit.com)": true,
"Melvil/1.0": true,
"MetaCommentBot; http://metacomment.io/about": true,
"mfibot/1.1 (http://www.mfisoft.ru/analyst/; <admin@mfisoft.ru>; en-RU)": true,
"Minoru's Fediverse Crawler (+https://nodes.fediverse.party)": true,
"mj12bot": true,
"Mozilla/4.0 (compatible; QwertyNetworksBot/1.0; +https://qwertynetworks.com/faq/QwertyNetworksBot)": true,
"Mozilla/4.0 (compatible; Wibybot; https://wiby.me/)": true,
"Mozilla/5.0 (Android 11; Mobile; rv:109.0) Gecko/114.0 Firefox/114.0": true,
"Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)": true,
"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)": true,
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true,
"Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true,
"Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)": true,
"Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot) Zeno/0030699 warc/v0.8.32": true,
"Mozilla/5.0 (compatible; Atomseobot/2.0; +http://https://error404.atomseo.com/)": true,
"Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)": true,
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html": true,
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true,
- "Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
- "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
"Mozilla/5.0 (compatible; BitSightBot/1.0)": true,
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true,
"Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)": true,
"Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true,
"Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.0.0 Safari/537.36": true,
"Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true,
"Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)": true,
"Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)": true,
"Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)": true,
"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)": true,
"Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true,
"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)": true,
"Mozilla/5.0 (compatible; Dmbot/1.1)": true,
"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true,
"Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)": true,
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true,
"Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)": true,
"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true,
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true,
"Mozilla/5.0 (compatible; FFZBot/2.4.1; +https://www.frankerfacez.com)": true,
"Mozilla/5.0 (compatible; FFZBot/3.0.0; +https://www.frankerfacez.com)": true,
"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true,
"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true,
- "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)": true,
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true,
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; Hkfl-Bot/%s +https://hackerfall.com/)": true,
"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)": true,
"Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true,
"Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)": true,
"Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)": true,
"Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true,
"Mozilla/5.0 (compatible; LinkFeatureBot)": true,
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true,
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Img/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true,
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Target/2.0; +https://help.mail.ru/webmaster/indexing/robots)": true,
"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true,
"Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true,
"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true,
"Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true,
"Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)": true,
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true,
"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)": true,
"Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)": true,
"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true,
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true,
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true,
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true,
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true,
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true,
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true,
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true,
"Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true,
"Mozilla/5.0 (compatible; NetpeakCheckerBot/3.6; +https://netpeaksoftware.com/checker)": true,
"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true,
"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true,
"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true,
"Mozilla/5.0 (compatible; OpsBot/1.0)": true,
"Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true,
"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true,
"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/hc/en-us/articles/360006695637-PaperLiBot)": true,
"Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)": true,
"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true,
"Mozilla/5.0 (compatible; proximic; +https://www.comscore.com/Web-Crawler)": true,
"Mozilla/5.0 (compatible; Qwantify-dev/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev10230/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev19924/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev19967/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev2270/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev22709/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev31653/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev4541/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev5115/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-dev846/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify-worker2994/1.0; +https://help.qwant.com/bot/)": true,
"Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true,
"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true,
"Mozilla/5.0 (compatible; rss2tg bot; +http://komar.in/en/rss2tg_crawler)": true,
"Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true,
"Mozilla/5.0 (compatible; SearchMySiteBot/1.0; +https://searchmysite.net)": true,
"Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true,
"Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true,
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true,
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true,
"Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)": true,
"Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)": true,
"Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true,
"Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true,
"Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true,
"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true,
"Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)": true,
"Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)": true,
"Mozilla/5.0 (compatible; SynapseMediaProxyBot/0.0.1; https://git.pixie.town/f0x/synapse-media-proxy)": true,
"Mozilla/5.0 (compatible; TestCrawler)": true,
"Mozilla/5.0 (compatible; TrendsmapResolver/0.1)": true,
"Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true,
"Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
"Mozilla/5.0 (compatible; WebwikiBot/2.1; +https://www.webwiki.de)": true,
"Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)": true,
"Mozilla/5.0 (compatible; woriobot +http://worio.com)": true,
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) sieve.k8s.crawler-production/1688317128-0": true,
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true,
"Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)": true,
"Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true,
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true,
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true,
"Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)": true,
"Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)": true,
"Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0": true,
"Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots)": true,
"Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true,
"Mozilla/5.0 (compatible; YetiBOT/0.1b; +http://yetibot.ovh)": true,
"Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true,
"Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e YisouSpider/5.0 Safari/602.1": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone14,3__weibo__13.6.3__iphone__os15.1)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone12,8__weibo__13.6.3__iphone__os15.6)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
- "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)": true,
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true,
- "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)": true,
- "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)": true,
"Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1": true,
"Mozilla/5.0 (Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0": true,
"Mozilla/5.0 (Linux x86_64; rv:114.0) Gecko/20100101 Firefox/114.0": true,
"Mozilla/5.0 (Linux; Android 10; BAH3-W59 Build/HUAWEIBAH3-W59; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Safari/537.36 Weibo (HUAWEI-BAH3-W59__weibo__12.5.2__android__android10)": true,
"Mozilla/5.0 (Linux; Android 10; VCE-AL00 Build/HUAWEIVCE-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Mobile Safari/537.36 Weibo (HUAWEI-VCE-AL00__weibo__13.6.3__android__android10)": true,
"Mozilla/5.0 (Linux; Android 13; V2244A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36V2244A_13_WeiboIntlAndroid_6200": true,
"Mozilla/5.0 (Linux; Android 13; V2302A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36 V2302A_13_WeiboIntlAndroid_2612": true,
"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)": true,
"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)": true,
"Mozilla/5.0 (Linux; Android 5.1.1; A37f) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36": true,
- "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
- "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
- "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
- "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.120 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
- "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (Linux; Android 6.0.1; vivo 1606 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 VivoBrowser/5.7.0.6": true,
"Mozilla/5.0 (Linux; Android 6.0.1; vivo Y66 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.141 Mobile Safari/537.36 VivoBrowser/10.9.14.0": true,
"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)": true,
"Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:29.0) Gecko/20100101 Firefox/29.0": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.7 DuckDuckGo/7 Safari/605.1.15": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36 Edg/104.0.1293.54; Bot": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36 webtru_crawler": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 ISSCyberRiskCrawler/1.1.3": true,
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5": true,
"Mozilla/5.0 (Morningscore Bot/1.0)": true,
"Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0": true,
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0": true,
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0": true,
"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko": true,
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.27+ (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27": true,
"Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0": true,
"Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2": true,
"Mozilla/5.0 (Windows NT 6.1) (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)": true,
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true,
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0": true,
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0": true,
"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0": true,
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36": true,
"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0": true,
"Mozilla/5.0 (Windows; Crawler; U; Windows NT 6.0; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7 (.NET CLR 3.5.30729)": true,
"Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.6) Gecko/20040206 Firefox/0.8": true,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler": true,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b4) Gecko/20050908 Firefox/1.4": true,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)": true,
"Mozilla/5.0 (Windows; U; Windows NT 5.1; ja; rv:1.8.0.10) Gecko/20070216 Firefox/1.5.0.10": true,
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11": true,
"Mozilla/5.0 (Windows; U; WinNT4.0; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1": true,
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.47 Safari/536.11": true,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36": true,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36/Gringe": true,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36": true,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36": true,
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)": true,
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20060118 Firefox/1.5": true,
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20061201 Firefox/2.0.0.8": true,
"Mozilla/5.0 (X11; U; Linux i686; fr-FR; rv:1.8.1.6) Gecko/20080208 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12": true,
"Mozilla/5.0 (X11; U; Linux i686; fr; rv:1.9.0.9) Gecko/2009042113 Ubuntu/8.04 (hardy) Firefox/3.0.9": true,
"Mozilla/5.0 (X11; U; Linux i686; hu-HU; rv:1.9.1.9) Gecko/20100330 Fedora/3.5.9-1.fc12 Firefox/3.5.9": true,
"Mozilla/5.0 (X11; U; Linux i686; nl; rv:1.9) Gecko/2008061015 Firefox/3.0": true,
"Mozilla/5.0 (X11; U; Linux i686; pt-BR; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6": true,
"Mozilla/5.0 (X11; U; Linux x86_64; cs-CZ; rv:1.9.1.7) Gecko/20100106 Ubuntu/9.10 (karmic) Firefox/3.5.7": true,
- "Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.8) Googlebot-Compatible Gecko/20100723 Ubuntu/10.04 (lucid) Firefox/3.6.8": true,
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.18) Gecko/20081112 Fedora/2.0.0.18-1.fc8 Firefox/2.0.0.18": true,
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true,
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true,
"Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true,
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Xing Bot": true,
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/100.0.4896.127 Safari/537.36": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/112.0.0.0 Safari/537.36": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/114.0.5735.179 Safari/537.36": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36": true,
- "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0(compatible;Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
"Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true,
"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true,
"Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true,
"Mozilla/9.0 (compatible; Staddlebot/1.0)": true,
"myspider/Nutch-1.10": true,
"navigation": true,
"netEstate NE Crawler (+http://www.website-datenbank.de/)": true,
"Netvibes (crawler; http://www.netvibes.com)": true,
"Netvibes (crawler; https://www.netvibes.com)": true,
"Nextcloud Server Crawler": true,
"omgili/0.5 +http://omgili.com": true,
"OWLer/0.1 (built with StormCrawler; https://ows.eu/owler; owl@ow-s.eu)": true,
"PercolateCrawler/4 (ops@percolate.com)": true,
"PhxBot/0.1 (phxbot@protonmail.com)": true,
"PlayStore-Google Mozilla/5.0 Chrome/114.0.5735.179 (KHTML, like Gecko; compatible; +http://www.google.com/bot.html)": true,
"PlurkBot; Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0(facebookexternalhitTwitterbot compatible; PlurkBot; +https://www.plurk.com/)": true,
"PubMatic Crawler Bot": true,
"PulsePoint-Ads.txt-Crawler/1.1": true,
"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true,
"Quora-Bot/1.0 (http://www.quora.com)": true,
"R6_CommentReader(www.radian6.com/crawler)": true,
"R6_FeedFetcher(www.radian6.com/crawler)": true,
"RandomWebSiteBot 1.0": true,
"read.write.as": true,
"Readybot.io (https://readybot.io)": true,
"repology-linkchecker/1 (+https://repology.org/docs/bots)": true,
"rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true,
"Ruby": true,
- "SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true,
- "SafeDNSBot (https://www.safedns.com/searchbot)": true,
- "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; http://www.google.com/bot.html)": true,
- "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
- "Screaming Frog SEO Spider/4.1": true,
- "Screaming Frog SEO Spider/5.0": true,
- "Screaming Frog SEO Spider/5.1": true,
- "Sellers.Guide Crawler by Primis": true,
- "semanticbot": true,
- "SEMrushBot": true,
- "SeRanking SEOChecker": true,
- "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)": true,
+ "SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true,
+ "SafeDNSBot (https://www.safedns.com/searchbot)": true,
+ "Sellers.Guide Crawler by Primis": true,
+ "semanticbot": true,
+ "SEMrushBot": true,
+ "SeRanking SEOChecker": true,
+ "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)": true,
"serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)": true,
"ShowyouBot (http://showyou.com/crawler)": true,
"SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)": true,
"Slack-ImgProxy (+https://api.slack.com/robots)": true,
"Slackbot 1.0 (+https://api.slack.com/robots)": true,
"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)": true,
"Slzji.com Search Bot (https://www.slzji.com/search_bot)": true,
"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots": true,
"Snap-URL-Preview (bot; snapchat; +https://developers.snap.com/robots)": true,
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)": true,
"spider": true,
"spiderbot": true,
"Stratagems Kumo": true,
"Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1396250262": true,
"Synapse (bot; +https://github.com/matrix-org/synapse)": true,
"TelegramBot (like TwitterBot)": true,
"TheFeedReaderBot/2.0": true,
"tiny.write.as": true,
"Traackr.com Bot": true,
"Twingly Recon": true,
- "Twingly Recon-Imse/1.0 (+https://app.twingly.com/public-docs/crawler)": true,
- "Twingly Recon-Wally/1.0 (+https://app.twingly.com/public-docs/crawler)": true,
- "Twitterbot": true,
- "voltron": true,
- "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)": true,
- "webscraper": true,
- "WhatsApp/2.23.12.78 A": true,
- "Who.is Bot": true,
- "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true,
- "wp.com feedbot/1.0 (+https://wp.com)": true,
- "Write.as v1.7.0; Android": true,
- "write.as": true,
- "WriteFreely.org Crawler (https://writefreely.org/instances)": true,
- "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true,
- "yacybot (-global; amd64 Linux 5.10.0-23-amd64; java 17.0.7; Europe/en) http://yacy.net/bot.html": true,
- "yacybot (/global; amd64 Linux 5.15.0-73-generic; java 1.8.0_372; Etc/en) http://yacy.net/bot.html": true,
- "yacybot (/global; amd64 Windows 10 10.0; java 11.0.18; Europe/pl) http://yacy.net/bot.html": true,
- "yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true,
- "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true,
+ "Twitterbot": true,
+ "voltron": true,
+ "webprosbot/2.0 (+mailto:abuse-6337@webpros.com)": true,
+ "webscraper": true,
+ "Who.is Bot": true,
+ "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true,
+ "wp.com feedbot/1.0 (+https://wp.com)": true,
+ "Write.as v1.7.0; Android": true,
+ "write.as": true,
+ "WriteFreely.org Crawler (https://writefreely.org/instances)": true,
+ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true,
"YisouSpider": true,
"ZoominfoBot (zoominfobot at zoominfo dot com)": true,
}
var botPrefixes = []string{
"AdsTxtCrawler/",
"Akkoma ",
"Apache-HttpClient/",
"bitlybot/",
"curl/",
"DuckDuckBot/",
"facebookexternalhit/",
"FediDB/",
"Friendica",
"Googlebot-Image/",
"Googlebot/",
"hackney/",
"http.rb/",
"kbinBot ",
"libwww-perl/",
"Mediumbot-MetaTagFetcher/",
"Mozilla/5.0 (compatible; AhrefsBot/",
"Mozilla/5.0 (compatible; Applebot/",
"Mozilla/5.0 (compatible; archive.org_bot",
"PHP/",
"PixelFedBot/",
"Pleroma ",
"python-requests/",
"Python-urllib/",
"RepoLookoutBot/",
"Screaming Frog SEO Spider/",
"semanticbot ",
"SummalyBot/",
"TelegramBot",
+ "Twingly Recon-",
"Twitterbot/",
"WhatsApp/",
+ "yacybot ",
+}
+
+var botPhrases = []string{
+ "bingbot",
+ "Googlebot",
}
// IsBot returns whether or not the provided User-Agent string is a known bot
// or crawler.
func IsBot(ua string) bool {
if ua == "" {
return true
}
if _, ok := bots[ua]; ok {
return true
}
for _, p := range botPrefixes {
if strings.HasPrefix(ua, p) {
return true
}
}
+ for _, p := range botPhrases {
+ if strings.Contains(ua, p) {
+ return true
+ }
+ }
return false
}
diff --git a/bots/bots_test.go b/bots/bots_test.go
index 5c6a2ab..e2553ab 100644
--- a/bots/bots_test.go
+++ b/bots/bots_test.go
@@ -1,23 +1,25 @@
package bots
import "testing"
func TestIsBot(t *testing.T) {
tests := map[string]bool{
- "Twitterbot/1.0": true,
+ "Twitterbot/1.0": true,
"http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true,
"http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true,
"http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true,
"http.rb/3.2.0 (Mastodon/2.4.3; +https://qoto.org/)": true,
"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false,
"Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)": true,
+
+ "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
}
for ua, r := range tests {
if IsBot(ua) != r {
t.Errorf("Expected bot = %t on '%s'", r, ua)
}
}
}