From e3f78ce5b9523bfc2c05dfe005945d66b6a0415b Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Wed, 11 Nov 2026 17:23:19 -0600 Subject: [PATCH] Regenerate agent registry (235 agents) --- sdk/src/registry/agents.generated.php | 184 +++++++++++++++++- .../registry/parity-vectors.generated.json | 168 ++++++++++++++++ 2 files changed, 351 insertions(+), 1 deletion(-) diff --git a/sdk/src/registry/agents.generated.php b/sdk/src/registry/agents.generated.php index e85006e..b65fc83 100644 --- a/sdk/src/registry/agents.generated.php +++ b/sdk/src/registry/agents.generated.php @@ -14,7 +14,7 @@ return [ 'schema' => 2, - 'agent_count' => 221, + 'agent_count' => 235, 'categories' => ['training_crawler', 'ai_search_crawler', 'ai_agent', 'ai_assistant', 'search_crawler', 'seo_crawler', 'security_scanner', 'generic_scraper', 'archiver', 'fetcher', 'monitoring', 'headless_browser', 'feed_reader', 'commercial_scraper', 'none'], 'agents' => [ [ @@ -381,6 +381,19 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'riverbot', + 'name' => 'RiverBot', + 'category' => 'training_crawler', + 'behavior' => 'training', + 'organization' => 'River', + 'patterns' => ['riverbot'], + 'robots_name' => 'RiverBot', + 'website' => 'https://river.ai/bot', + 'base_score' => 80, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'kimibot', 'name' => 'KimiBot', @@ -654,6 +667,19 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'isonsearchbot', + 'name' => 'IsonSearchBot', + 'category' => 'ai_search_crawler', + 'behavior' => 'search', + 'organization' => 'Ison Search', + 'patterns' => ['isonsearchbot'], + 'robots_name' => 'IsonSearchBot', + 'website' => 'https://search.isonai.com/bot', + 'base_score' => 65, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'keenablebot', 'name' => 'KeenableBot', @@ -1304,6 +1330,45 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'barkrowler', + 'name' => 'Barkrowler', + 'category' => 'seo_crawler', + 'behavior' => 'seo', + 'organization' => 'Babbar', + 'patterns' => ['barkrowler'], + 'robots_name' => 'Barkrowler', + 'website' => 'https://babbar.tech/crawler', + 'base_score' => 50, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'scrunchbot', + 'name' => 'Scrunchbot', + 'category' => 'seo_crawler', + 'behavior' => 'seo', + 'organization' => 'Scrunch', + 'patterns' => ['scrunchbot'], + 'robots_name' => 'Scrunchbot', + 'website' => 'https://scrunchai.com/bots', + 'base_score' => 50, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'webxtrendbot', + 'name' => 'webXtrendBot', + 'category' => 'seo_crawler', + 'behavior' => 'seo', + 'organization' => 'webXtrend', + 'patterns' => ['webxtrendbot'], + 'robots_name' => 'webXtrendBot', + 'website' => 'https://webxtrend.com/crawler', + 'base_score' => 50, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'serankingbacklinksbot', 'name' => 'SERankingBacklinksBot', @@ -1525,6 +1590,58 @@ 'respects_robots' => false, 'http_client' => false, ], + [ + 'id' => 'paloalto-xpanse', + 'name' => 'Palo Alto Networks Cortex Xpanse', + 'category' => 'security_scanner', + 'behavior' => 'security_testing', + 'organization' => 'Palo Alto Networks', + 'patterns' => ['palo alto networks'], + 'robots_name' => '', + 'website' => 'https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity', + 'base_score' => 65, + 'respects_robots' => false, + 'http_client' => false, + ], + [ + 'id' => 'recordedfuture', + 'name' => 'Recorded Future Crawler', + 'category' => 'security_scanner', + 'behavior' => 'security_testing', + 'organization' => 'Recorded Future', + 'patterns' => ['recordedfuture'], + 'robots_name' => '', + 'website' => 'https://www.recordedfuture.com', + 'base_score' => 65, + 'respects_robots' => false, + 'http_client' => false, + ], + [ + 'id' => 'cloudflare-security-center', + 'name' => 'Cloudflare Security Center', + 'category' => 'security_scanner', + 'behavior' => 'security_testing', + 'organization' => 'Cloudflare', + 'patterns' => ['developers.cloudflare.com/security-center'], + 'robots_name' => '', + 'website' => 'https://developers.cloudflare.com/security-center/', + 'base_score' => 40, + 'respects_robots' => false, + 'http_client' => false, + ], + [ + 'id' => 'virustotal-cloud', + 'name' => 'VirusTotal', + 'category' => 'security_scanner', + 'behavior' => 'security_testing', + 'organization' => 'Google', + 'patterns' => ['virustotalcloud'], + 'robots_name' => '', + 'website' => 'https://www.virustotal.com', + 'base_score' => 40, + 'respects_robots' => false, + 'http_client' => false, + ], [ 'id' => 'intruder', 'name' => 'Intruder', @@ -2175,6 +2292,19 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'stripebot', + 'name' => 'Stripebot', + 'category' => 'fetcher', + 'behavior' => 'link_preview', + 'organization' => 'Stripe', + 'patterns' => ['stripebot'], + 'robots_name' => 'Stripebot', + 'website' => 'https://docs.stripe.com/stripebot-crawler', + 'base_score' => 25, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'google-read-aloud', 'name' => 'Google-Read-Aloud', @@ -2474,6 +2604,19 @@ 'respects_robots' => true, 'http_client' => false, ], + [ + 'id' => 'livelapbot', + 'name' => 'LivelapBot', + 'category' => 'feed_reader', + 'behavior' => 'feeds', + 'organization' => 'Livelap', + 'patterns' => ['livelapbot'], + 'robots_name' => 'LivelapBot', + 'website' => 'https://site.livelap.com/crawler', + 'base_score' => 30, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'feedfetcher-google', 'name' => 'Feedfetcher-Google', @@ -2617,6 +2760,45 @@ 'respects_robots' => false, 'http_client' => false, ], + [ + 'id' => 'brightbot', + 'name' => 'Brightbot', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'Bright Data', + 'patterns' => ['brightbot'], + 'robots_name' => 'Brightbot', + 'website' => 'https://brightdata.com/brightbot', + 'base_score' => 60, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'cloudflare-crawl', + 'name' => 'Cloudflare Browser Rendering Crawler', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'Cloudflare', + 'patterns' => ['cloudflarebrowserrenderingcrawler'], + 'robots_name' => 'CloudflareBrowserRenderingCrawler', + 'website' => 'https://developers.cloudflare.com/browser-rendering/rest-api/crawl-endpoint', + 'base_score' => 55, + 'respects_robots' => true, + 'http_client' => false, + ], + [ + 'id' => 'ev-crawler', + 'name' => 'ev-crawler', + 'category' => 'commercial_scraper', + 'behavior' => '', + 'organization' => 'Headline', + 'patterns' => ['ev-crawler'], + 'robots_name' => 'ev-crawler', + 'website' => 'https://headline.com/web-crawler', + 'base_score' => 55, + 'respects_robots' => true, + 'http_client' => false, + ], [ 'id' => 'oxylabs', 'name' => 'Oxylabs', diff --git a/sdk/src/registry/parity-vectors.generated.json b/sdk/src/registry/parity-vectors.generated.json index b304192..8e68595 100644 --- a/sdk/src/registry/parity-vectors.generated.json +++ b/sdk/src/registry/parity-vectors.generated.json @@ -227,6 +227,18 @@ "category": "search_crawler", "matched": true }, + { + "userAgent": "barkrowler", + "id": "barkrowler", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; barkrowler/1.0; +http://example.com/bot)", + "id": "barkrowler", + "category": "seo_crawler", + "matched": true + }, { "userAgent": "beautifulsoup", "id": "beautifulsoup", @@ -323,6 +335,18 @@ "category": "commercial_scraper", "matched": true }, + { + "userAgent": "brightbot", + "id": "brightbot", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; brightbot/1.0; +http://example.com/bot)", + "id": "brightbot", + "category": "commercial_scraper", + "matched": true + }, { "userAgent": "browser-use", "id": "browser-use", @@ -491,6 +515,30 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "cloudflarebrowserrenderingcrawler", + "id": "cloudflare-crawl", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; cloudflarebrowserrenderingcrawler/1.0; +http://example.com/bot)", + "id": "cloudflare-crawl", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "developers.cloudflare.com/security-center", + "id": "cloudflare-security-center", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; developers.cloudflare.com/security-center/1.0; +http://example.com/bot)", + "id": "cloudflare-security-center", + "category": "security_scanner", + "matched": true + }, { "userAgent": "coccocbot", "id": "coccocbot", @@ -1271,6 +1319,18 @@ "category": "security_scanner", "matched": true }, + { + "userAgent": "isonsearchbot", + "id": "isonsearchbot", + "category": "ai_search_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; isonsearchbot/1.0; +http://example.com/bot)", + "id": "isonsearchbot", + "category": "ai_search_crawler", + "matched": true + }, { "userAgent": "java/", "id": "java", @@ -1451,6 +1511,18 @@ "category": "ai_search_crawler", "matched": true }, + { + "userAgent": "livelapbot", + "id": "livelapbot", + "category": "feed_reader", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; livelapbot/1.0; +http://example.com/bot)", + "id": "livelapbot", + "category": "feed_reader", + "matched": true + }, { "userAgent": "mj12bot", "id": "mj12bot", @@ -1883,6 +1955,18 @@ "category": "generic_scraper", "matched": true }, + { + "userAgent": "palo alto networks", + "id": "paloalto-xpanse", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; palo alto networks/1.0; +http://example.com/bot)", + "id": "paloalto-xpanse", + "category": "security_scanner", + "matched": true + }, { "userAgent": "perplexity-user", "id": "perplexity-user", @@ -2075,6 +2159,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "recordedfuture", + "id": "recordedfuture", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; recordedfuture/1.0; +http://example.com/bot)", + "id": "recordedfuture", + "category": "security_scanner", + "matched": true + }, { "userAgent": "reddit", "id": "reddit", @@ -2099,6 +2195,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "riverbot", + "id": "riverbot", + "category": "training_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; riverbot/1.0; +http://example.com/bot)", + "id": "riverbot", + "category": "training_crawler", + "matched": true + }, { "userAgent": "rogerbot", "id": "rogerbot", @@ -2207,6 +2315,18 @@ "category": "seo_crawler", "matched": true }, + { + "userAgent": "scrunchbot", + "id": "scrunchbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; scrunchbot/1.0; +http://example.com/bot)", + "id": "scrunchbot", + "category": "seo_crawler", + "matched": true + }, { "userAgent": "searchgpt", "id": "searchgpt", @@ -2447,6 +2567,18 @@ "category": "monitoring", "matched": true }, + { + "userAgent": "stripebot", + "id": "stripebot", + "category": "fetcher", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; stripebot/1.0; +http://example.com/bot)", + "id": "stripebot", + "category": "fetcher", + "matched": true + }, { "userAgent": "tavily", "id": "tavily", @@ -2555,6 +2687,18 @@ "category": "training_crawler", "matched": true }, + { + "userAgent": "virustotalcloud", + "id": "virustotal-cloud", + "category": "security_scanner", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; virustotalcloud/1.0; +http://example.com/bot)", + "id": "virustotal-cloud", + "category": "security_scanner", + "matched": true + }, { "userAgent": "wayback", "id": "wayback", @@ -2735,6 +2879,18 @@ "category": "fetcher", "matched": true }, + { + "userAgent": "ev-crawler", + "id": "ev-crawler", + "category": "commercial_scraper", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; ev-crawler/1.0; +http://example.com/bot)", + "id": "ev-crawler", + "category": "commercial_scraper", + "matched": true + }, { "userAgent": "img2dataset", "id": "img2dataset", @@ -2795,6 +2951,18 @@ "category": "seo_crawler", "matched": true }, + { + "userAgent": "webxtrendbot", + "id": "webxtrendbot", + "category": "seo_crawler", + "matched": true + }, + { + "userAgent": "Mozilla/5.0 (compatible; webxtrendbot/1.0; +http://example.com/bot)", + "id": "webxtrendbot", + "category": "seo_crawler", + "matched": true + }, { "userAgent": "xai-searchbot", "id": "xai-searchbot",