diff --git a/crawlers.go b/crawlers.go index 035059b..f5d6f3f 100644 --- a/crawlers.go +++ b/crawlers.go @@ -6,13 +6,13 @@ var crawlers = []string{ `^Aether`, `^Amazon Simple Notification Service Agent$`, `^Amazon-Route53-Health-Check-Service`, - `^Amazon CloudFront`, `^b0t$`, `^bluefish `, `^Calypso v\/`, `^COMODO DCV`, `^Corax`, `^DangDang`, + `^Dart\/[0-9]`, `^DavClnt`, `^DHSH`, `^docker\/[0-9]`, @@ -52,6 +52,7 @@ var crawlers = []string{ `^XRL\/[0-9]`, `^ZmEu`, `008\/`, + `1Pilot`, `13TABS`, `192\.comAgent`, `2GDPR\/`, @@ -66,6 +67,7 @@ var crawlers = []string{ `aboutthedomain`, `Accoona-AI-Agent`, `acebookexternalhit\/`, + `acme\.sectigo\.com`, `acoon`, `acrylicapps\.com\/pulp`, `Acunetix`, @@ -85,6 +87,7 @@ var crawlers = []string{ `affilimate-puppeteer`, `agentslug`, `AHC`, + `Ahrefs`, `aihit`, `aiohttp\/`, `Airmail`, @@ -98,6 +101,7 @@ var crawlers = []string{ `AllSubmitter`, `alyze\.info`, `amagit`, + `amphp\/http-client`, `Anarchie`, `AndroidDownloadManager`, `Anemone`, @@ -224,6 +228,7 @@ var crawlers = []string{ `changedetection`, `ChangesMeter`, `Charlotte`, + `ChatGPT-User`, `chatterino-api-cache`, `CheckHost`, `checkprivacy`, @@ -240,11 +245,13 @@ var crawlers = []string{ `CJNetworkQuality`, `Clarsentia`, `Claude-User`, + `claude-web`, `clips\.ua\.ac\.be`, `Cloud mapping`, `CloudEndure`, `CloudFlare-AlwaysOnline`, `Cloudflare-Healthchecks`, + `Cloudflare-SSLDetector`, `Cloudinary`, `cmcm\.com`, `coccoc`, @@ -265,6 +272,8 @@ var crawlers = []string{ `cortex\/`, `Cosmos4j\.feedback`, `Covario-IDS`, + `Cpanel-HTTP-Client`, + `cpp-httplib`, `Craw\/`, `Crescent`, `Criteo`, @@ -375,6 +384,7 @@ var crawlers = []string{ `faviconarchive`, `faviconkit`, `FavOrg`, + `FediAct Service`, `Feed Wrangler`, `Feedable\/`, `Feedbin`, @@ -404,7 +414,6 @@ var crawlers = []string{ `FlipboardBrowserProxy`, `FlipboardProxy`, `FlipboardRSS`, - `Flock\/`, `Florienzh\/`, `fluffy`, `Flunky`, @@ -469,8 +478,10 @@ var crawlers = []string{ `Google Web Preview`, `Google-Ads`, `Google-Adwords`, + `Google-AMPHTML`, `Google-Apps-Script`, `Google-Calendar-Importer`, + `Google-Firebase`, `Google-HotelAdsVerifier`, `Google-HTTP-Java-Client`, `Google-InspectionTool`, @@ -485,6 +496,7 @@ var crawlers = []string{ `Google-Transparency-Report`, `google-xrawler`, `Google-Youtube-Links`, + `GoogleAssociationService`, `GoogleDocs`, `GoogleHC\/`, `GoogleMessages`, @@ -527,6 +539,7 @@ var crawlers = []string{ `help@dataminr\.com`, `heritrix`, `Hexometer`, + `HF-Research\/`, `historious`, `hkedcity`, `hledejLevne\.cz`, @@ -570,6 +583,7 @@ var crawlers = []string{ `HubSpot-Link-Resolver`, `Humanlinks`, `i2kconnect\/`, + `IAB-Tech-Lab`, `Iblog`, `ichiro`, `Id-search`, @@ -586,6 +600,7 @@ var crawlers = []string{ `ImageVisu\/`, `Imagga`, `imagineeasy`, + `imgproxy\/[0-9]`, `imgsizer`, `InAGist`, `inbound\.li parser`, @@ -666,6 +681,7 @@ var crawlers = []string{ `kubectl`, `kulturarw3`, `KumKie`, + `l9tcpid\/`, `Larbin`, `Lavf\/`, `leakix\.net`, @@ -690,6 +706,7 @@ var crawlers = []string{ `linkCheck`, `linkdex`, `LinkExaminer`, + `Linkfeed\.ru`, `linkfluence`, `linkpeek`, `LinkPreview`, @@ -796,6 +813,7 @@ var crawlers = []string{ `myseosnapshot`, `nagios`, `Najdi\.si`, + `holubem\.eu`, `Name Intelligence`, `NameFo\.com`, `Nameprotect`, @@ -819,6 +837,7 @@ var crawlers = []string{ `nettle`, `NetTrack`, `Netvibes`, + `NetworkingExtension`, `NetZIP`, `Neustar WPM`, `NeutrinoAPI`, @@ -828,6 +847,7 @@ var crawlers = []string{ `newsme`, `newspaper\/`, `Nexgate Ruby Client`, + `Next\.js Middleware`, `NG-Search`, `nghttp2`, `Nibbler`, @@ -931,6 +951,7 @@ var crawlers = []string{ `Pixray`, `Pizilla`, `Plagger\/`, + `PlayStore-Google`, `Pleroma `, `Ploetz \+ Zeller`, `Plukkie`, @@ -975,6 +996,7 @@ var crawlers = []string{ `python-httpx`, `python-requests`, `Python-urllib`, + `Python-xmlrpc`, `Qirina Hurdler`, `QQDownload`, `QrafterPro`, @@ -1064,7 +1086,9 @@ var crawlers = []string{ `SEOCentro`, `SeoCheck`, `seocompany`, + `SEOGets`, `SEOkicks`, + `SeoLik\.ru`, `SEOlizer`, `Seomoz`, `SEOprofiler`, @@ -1166,6 +1190,7 @@ var crawlers = []string{ `Surphace Scout`, `Suzuran`, `swcd `, + `Swisscows Favicons`, `Symfony BrowserKit`, `Symfony2 BrowserKit`, `Synapse\/`, @@ -1235,6 +1260,7 @@ var crawlers = []string{ `UdmSearch`, `ultimate_sitemap_parser`, `unchaos`, + `undici`, `unirest-java`, `UniversalFeedParser`, `unshortenit`, @@ -1289,6 +1315,7 @@ var crawlers = []string{ `WDT\.io`, `Web Auto`, `Web Collage`, + `Web Downloader\/[0-9]`, `Web Enhancer`, `Web Fetch`, `Web Fuck`, diff --git a/exclusions.go b/exclusions.go index 1b8da05..df7ada7 100644 --- a/exclusions.go +++ b/exclusions.go @@ -6,7 +6,7 @@ var exclusions = []string{ `Firefox.[\d\.]*`, ` Chrome.[\d\.]*`, `Chromium.[\d\.]*`, - `MSIE.[\d\.]`, + `MSIE.[\d\.]*`, `Opera\/[\d\.]*`, `Mozilla.[\d\.]*`, `AppleWebKit.[\d\.]*`, diff --git a/testdata/crawlers.txt b/testdata/crawlers.txt index 4df94fe..56aac8f 100644 --- a/testdata/crawlers.txt +++ b/testdata/crawlers.txt @@ -3272,10 +3272,6 @@ Trendsmap Resolver/0.0.1 Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 4.0.30319.17929) Arukereso/1.0 Trendsmap Resolver/0.0.1 -Flock/14.15 (Android 2.9; fr_BE;) -Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.0.16) Gecko/2010021003 Firefox/3.0.16 Flock/2.5.6 -Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.14) Gecko/20080414 Firefox/2.0.0.14 Flock/1.1.2 -Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070801 Firefox/2.0.0.6 Flock/0.9.0.2 AppStoreScraperZ/1 CFNetwork/889.9 Darwin/17.2.0 (x86_64) Everwall Link Expander ( http://everwall.com ) Wfuzz/2.2.2 @@ -3666,7 +3662,6 @@ Mollie HTTP client/1.0 Inboxb0t-Image/2.0 fragFINN.de HTTP-Collector SpeedKit -Amazon CloudFront link-check/5.2.0 Mozilla/5.0/Firefox/42.0 - nbertaupete95(at)gmail.com Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/100.0.4896.127 Safari/537.36 @@ -3691,4 +3686,35 @@ Mozilla/5.0 (compatible; Odin; https://docs.getodin.com/) Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/) Mozilla/5.0 (compatible; IbouBot/1.0; +bot@ibou.io; +https://ibou.io/iboubot.html) Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 DebugBear -Mozilla/5.0 (Linux; Android 11.0; moto g power (2022)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 DebugBear \ No newline at end of file +Mozilla/5.0 (Linux; Android 11.0; moto g power (2022)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 DebugBear +Mozilla/5.0 (compatible; monitoring360bot/1.1; +https://app.360monitoring.com/bot.html) +1Pilot Monitoring (+http://1pilot.io/) +ChatGPT-User/1.0 +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot) +Mozilla/5.0 (Linux; arm_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 claude-web +NetworkingExtension/8621.3.11.10.3 Network/4277.140.33 iOS/18.6.2 +NetworkingExtension/8620.2.5.10.7 Network/4277.42.2 iOS/17.6.1 +Dart/3.5 (dart:io) +acme.sectigo.com/v2/DV +amphp/http-client/5.x +Cloudflare-SSLDetector +Cpanel-HTTP-Client/1.0 +cpp-httplib/0.15.3 +FediAct Service +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.7049.52 Mobile Safari/537.36 (compatible; Google-AMPHTML) +Google-Firebase +GoogleAssociationService +HF-Research/1.0 +IAB-Tech-Lab +imgproxy/3.27.2 +l9tcpid/v1.1.0 +Linkfeed.ru +Next.js Middleware +PlayStore-Google +Python-xmlrpc/3.9 +SeoLik.ru +Swisscows Favicons +undici +Web Downloader/6.9 +Najdu (Vyhledavac sluzeb (obsah) najdu.s.holubem.eu) +Najdu (Vyhledavac sluzeb (hlavicka) najdu.s.holubem.eu) diff --git a/testdata/devices.txt b/testdata/devices.txt index 89ec1be..ab2ee32 100644 --- a/testdata/devices.txt +++ b/testdata/devices.txt @@ -165634,4 +165634,7 @@ Mozilla/5.0 (Linux; Android 5.0; SM-N9100 Build/LRX21V) > AppleWebKit/537.36 (KH Mozilla/5.0 (Linux; U; Android 5.0; en-US; SM-N900S;iPhone: Build/LRX21V) Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 YaBrowser/24.1.2.86 (lite) TV Safari/537.36 -Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36 \ No newline at end of file +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36 +Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.0.16) Gecko/2010021003 Firefox/3.0.16 Flock/2.5.6 +Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.8.1.14) Gecko/20080414 Firefox/2.0.0.14 Flock/1.1.2 +Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070801 Firefox/2.0.0.6 Flock/0.9.0.2 \ No newline at end of file