mastodon/public/robots.txt

# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

User-agent: *
Disallow: /media_proxy/
Disallow: /interact/

## AI Bots - Research from darkvisitors.com and multiple other sources.

# OpenAI Scraper/Crawler/Assistant - https://platform.openai.com/docs/bots
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# AI Search Crawler - https://developer.amazon.com/amazonbot
User-agent: Amazonbot

# ??
User-agent: anthropic-ai

# AI Search Crawler - https://support.apple.com/en-au/HT204683
User-agent: Applebot

# AI Data Scraper - https://support.apple.com/en-au/119829
User-agent: Applebot-Extended

# AI Data Scraper
User-agent: Bytespider

# AI Data Scraper - https://commoncrawl.org/ccbot
User-agent: CCBot

# AI Data Scraper - https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: ClaudeBot

# ??
User-agent: Claude-Web

# ??
User-agent: cohere-ai

# AI Data Scraper
User-agent: Diffbot

# AI Data Scraper - https://developers.facebook.com/docs/sharing/bot/
User-agent: FacebookBot

# AI Data Scraper
User-agent: Google-Extended

# Search Engine Crawler - http://google.com/bot.html
User-agent: GoogleOther

# ?? - https://developers.google.com/search/docs/crawling-indexing/googlebot
User-agent: GoogleOther-Image

# ?? - https://developers.google.com/search/docs/crawling-indexing/googlebot
User-agent: GoogleOther-Video

# Intelligence Gatherer - https://imagesift.com/about
User-agent: ImagesiftBot

# ??
User-agent: img2dataset

# AI Data Scraper - https://developers.facebook.com/docs/sharing/webmasters/crawler
User-agent: Meta-ExternalAgent

# AI Assistant - https://developers.facebook.com/docs/sharing/webmasters/crawler
User-agent: Meta-ExternalFetcher

# AI Data Scraper
User-agent: omgili

# ??
User-agent: omgilibot

# AI Search Crawler - https://docs.perplexity.ai/docs/perplexitybot
User-agent: PerplexityBot

# ??
User-agent: Scrapy

# AI Data Scraper
User-agent: Timpibot

# AI Data Scraper - https://velen.io/
User-agent: VelenPublicWebCrawler

# AI Search Crawler - https://about.you.com/es/youbot/
User-agent: YouBot

Disallow: /