Merge pull request #153 from sergiospagnuolo/Poseidon

ai.robots.txt · ai.robots.txt · commit eb05f2f5276f · 2025-06-14T14:04:03.000Z
Update robots.json with new crawler
diff --git a/.htaccess b/.htaccess
@@ -1,3 +1,3 @@
 RewriteEngine On
-RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot) [NC]
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot) [NC]
 RewriteRule !^/?robots\.txt$ - [F,L]
diff --git a/Caddyfile b/Caddyfile
@@ -1,3 +1,3 @@
 @aibots {
-        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)"
+        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)"
 }
diff --git a/haproxy-block-ai-bots.txt b/haproxy-block-ai-bots.txt
@@ -56,6 +56,7 @@ Perplexity-User
 PerplexityBot
 PetalBot
 PhindBot
+Poseidon Research Crawler
 QualifiedBot
 QuillBot
 quillbot.com
diff --git a/nginx-block-ai-bots.conf b/nginx-block-ai-bots.conf
@@ -1,3 +1,3 @@
-if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)") {
+if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|bedrockbot|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|EchoboxBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|MyCentralAIScraperBot|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot|SemrushBot\-BA|SemrushBot\-CT|SemrushBot\-OCOB|SemrushBot\-SI|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)") {
     return 403;
 }
diff --git a/robots.txt b/robots.txt
@@ -56,6 +56,7 @@ User-agent: Perplexity-User
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: PhindBot
+User-agent: Poseidon Research Crawler
 User-agent: QualifiedBot
 User-agent: QuillBot
 User-agent: quillbot.com
diff --git a/table-of-bot-metrics.md b/table-of-bot-metrics.md
@@ -58,6 +58,7 @@
 | PerplexityBot | [Perplexity](https://www.perplexity.ai/) | [Yes](https://docs.perplexity.ai/guides/bots) | Search result generation. | No information. | Crawls sites to surface as results in Perplexity. |
 | PetalBot | [Huawei](https://huawei.com/) | Yes | Used to provide recommendations in Hauwei assistant and AI search services. | No explicit frequency provided. | Operated by Huawei to provide search and AI assistant services. |
 | PhindBot | [phind](https://www.phind.com/) | Unclear at this time. | AI-enhanced search engine. | No explicit frequency provided. | Company offers an AI agent that uses AI and generate extra web query on the fly |
+| Poseidon Research Crawler | [Poseidon Research](https://www.poseidonresearch.com) | Unclear at this time. | AI research crawler | No explicit frequency provided. | Lab focused on scaling the interpretability research necessary to make better AI systems possible. |
 | QualifiedBot | [Qualified](https://www.qualified.com) | Unclear at this time. | Company offers AI agents and other related products; usage can be assumed to support said products. | No explicit frequency provided. | Operated by Qualified as part of their suite of AI product offerings. |
 | QuillBot | [Quillbot](https://quillbot.com) | Unclear at this time. | Company offers AI detection, writing tools and other services. | No explicit frequency provided. | Operated by QuillBot as part of their suite of AI product offerings. |
 | quillbot\.com | [Quillbot](https://quillbot.com) | Unclear at this time. | Company offers AI detection, writing tools and other services. | No explicit frequency provided. | Operated by QuillBot as part of their suite of AI product offerings. |

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`@aibots {`
`2`		- header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|bedrockbot\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|EchoboxBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|MyCentralAIScraperBot\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Panscient\|panscient\.com\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|QuillBot\|quillbot\.com\|SBIntuitionsBot\|Scrapy\|SemrushBot\|SemrushBot\-BA\|SemrushBot\-CT\|SemrushBot\-OCOB\|SemrushBot\-SI\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)"
	`2`	+ header_regexp User-Agent "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|bedrockbot\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|EchoboxBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|MyCentralAIScraperBot\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Panscient\|panscient\.com\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|Poseidon\ Research\ Crawler\|QualifiedBot\|QuillBot\|quillbot\.com\|SBIntuitionsBot\|Scrapy\|SemrushBot\|SemrushBot\-BA\|SemrushBot\-CT\|SemrushBot\-OCOB\|SemrushBot\-SI\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)"
`3`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`		-if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|bedrockbot\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|EchoboxBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|MyCentralAIScraperBot\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Panscient\|panscient\.com\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|QualifiedBot\|QuillBot\|quillbot\.com\|SBIntuitionsBot\|Scrapy\|SemrushBot\|SemrushBot\-BA\|SemrushBot\-CT\|SemrushBot\-OCOB\|SemrushBot\-SI\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)") {
	`1`	+if ($http_user_agent ~* "(AI2Bot\|Ai2Bot\-Dolma\|aiHitBot\|Amazonbot\|Andibot\|anthropic\-ai\|Applebot\|Applebot\-Extended\|bedrockbot\|Brightbot\ 1\.0\|Bytespider\|CCBot\|ChatGPT\-User\|Claude\-SearchBot\|Claude\-User\|Claude\-Web\|ClaudeBot\|cohere\-ai\|cohere\-training\-data\-crawler\|Cotoyogi\|Crawlspace\|Diffbot\|DuckAssistBot\|EchoboxBot\|FacebookBot\|Factset_spyderbot\|FirecrawlAgent\|FriendlyCrawler\|Google\-CloudVertexBot\|Google\-Extended\|GoogleOther\|GoogleOther\-Image\|GoogleOther\-Video\|GPTBot\|iaskspider/2\.0\|ICC\-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|meta\-externalagent\|Meta\-ExternalAgent\|meta\-externalfetcher\|Meta\-ExternalFetcher\|MistralAI\-User/1\.0\|MyCentralAIScraperBot\|NovaAct\|OAI\-SearchBot\|omgili\|omgilibot\|Operator\|PanguBot\|Panscient\|panscient\.com\|Perplexity\-User\|PerplexityBot\|PetalBot\|PhindBot\|Poseidon\ Research\ Crawler\|QualifiedBot\|QuillBot\|quillbot\.com\|SBIntuitionsBot\|Scrapy\|SemrushBot\|SemrushBot\-BA\|SemrushBot\-CT\|SemrushBot\-OCOB\|SemrushBot\-SI\|SemrushBot\-SWA\|Sidetrade\ indexer\ bot\|TikTokSpider\|Timpibot\|VelenPublicWebCrawler\|Webzio\-Extended\|wpbot\|YandexAdditional\|YandexAdditionalBot\|YouBot)") {
`2`	`2`	`return 403;`
`3`	`3`	`}`