From ce539ffa7d4afd85138d6cd76c445e6f918f9b04 Mon Sep 17 00:00:00 2001 From: Andreas Jaggi Date: Tue, 14 Jul 2026 05:48:34 +0200 Subject: [PATCH] Update patterns Add new crawlers: - add Palo Alto Cortex Xpanse - add Infrawatch - add ModatScanner - add krynos - add Visionheight - add ExposureScanner - add foda-scanner - add HaveInListEnricher - add pathscan - add Goosee-Audit Corresponds to upstream v1.4.1 --- crawlers.go | 10 ++++++++++ testdata/crawlers.txt | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/crawlers.go b/crawlers.go index f5d6f3f..8c9434b 100644 --- a/crawlers.go +++ b/crawlers.go @@ -17,11 +17,14 @@ var crawlers = []string{ `^DHSH`, `^docker\/[0-9]`, `^Expanse`, + `^ExposureScanner`, `^FDM `, `^git\/`, `^Goose\/`, + `^Goosee-Audit`, `^Grabber`, `^Gradle\/`, + `^Hello from Palo Alto Networks`, `^HTTPClient\/`, `^HTTPing`, `^Java\/`, @@ -47,6 +50,7 @@ var crawlers = []string{ `^TLS tester `, `^twine\/`, `^ureq`, + `^visionheight\.com\/scan`, `^VSE\/[0-9]`, `^WordPress\.com`, `^XRL\/[0-9]`, @@ -418,6 +422,7 @@ var crawlers = []string{ `fluffy`, `Flunky`, `flynxapp`, + `foda-scanner\/[0-9]`, `forensiq`, `ForusP`, `FoundSeoTool`, @@ -532,6 +537,7 @@ var crawlers = []string{ `HappyApps-WebCheck`, `Hardenize`, `Hatena`, + `HaveInListEnricher`, `Havij`, `HaxerMen`, `HEADMasterSEO`, @@ -611,6 +617,7 @@ var crawlers = []string{ `infohelfer`, `InfoTekies`, `InfoWizards Reciprocal Link`, + `Infrawatch\/[0-9]`, `inpwrd\.com`, `instabid`, `Instapaper`, @@ -677,6 +684,7 @@ var crawlers = []string{ `KOCMOHABT`, `kouio`, `krawler\.dk`, + `krynos`, `kube-probe`, `kubectl`, `kulturarw3`, @@ -786,6 +794,7 @@ var crawlers = []string{ `mixed-content-scan`, `mixnode`, `Mnogosearch`, + `ModatScanner\/[0-9]`, `mogimogi`, `Mojeek`, `Mojolicious \(Perl\)`, @@ -920,6 +929,7 @@ var crawlers = []string{ `panscient`, `Papa Foto`, `parsijoo`, + `pathscan\/[0-9]`, `Pavuk`, `PayPal IPN`, `pcBrowser`, diff --git a/testdata/crawlers.txt b/testdata/crawlers.txt index 56aac8f..3fcb275 100644 --- a/testdata/crawlers.txt +++ b/testdata/crawlers.txt @@ -3718,3 +3718,13 @@ undici Web Downloader/6.9 Najdu (Vyhledavac sluzeb (obsah) najdu.s.holubem.eu) Najdu (Vyhledavac sluzeb (hlavicka) najdu.s.holubem.eu) +Hello from Palo Alto Networks, find out more about our scans in https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity +Mozilla/5.0 (compatible; Infrawatch/1.0; +https://infrawat.ch/) +Mozilla/5.0 (compatible; ModatScanner/1.2; +https://modat.io/) +mozilla firefox 5.0/krynos made +Mozilla/5.0 (compatible; pathscan/1.0) +Mozilla/5.0 (compatible; foda-scanner/0.1) +ExposureScanner/1.0 (internal-security-audit) +visionheight.com/scan Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/126.0.0.0 Safari/537.36 +Goosee-Audit/1.0 (+internal-security-audit) +Mozilla/5.0 (compatible; HaveInListEnricher/1.0)