From cc7e07f12f461a9209ba7b9e7862912e7269a604 Mon Sep 17 00:00:00 2001 From: Shlee Date: Thu, 1 Aug 2024 20:41:10 +0930 Subject: [PATCH] Update robots.txt --- public/robots.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 2c04383b56c..d7d5dfd8c6a 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -8,7 +8,7 @@ Disallow: /interact/ # AI Bots - Crawlers/scrapers and data harvesters. Blocking these should not negatively effect the end user. ############################################################################################################################### -# OpenAI Scraper/Crawler/Assistant - https://platform.openai.com/docs/bots +# OpenAI Scraper/Crawler/Assistant https://platform.openai.com/docs/bots User-agent: GPTBot User-agent: ChatGPT-User User-agent: OAI-SearchBot @@ -22,7 +22,7 @@ User-agent: Applebot # Apple AI models https://support.apple.com/en-au/119829 User-agent: Applebot-Extended -# https://commoncrawl.org/ccbot +# Common Crawl https://commoncrawl.org/ccbot User-agent: CCBot # Facebook AI models https://developers.facebook.com/docs/sharing/bot/ @@ -33,8 +33,6 @@ User-agent: Meta-ExternalAgent # Googles Gemini Apps and Vertex AI https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers User-agent: Google-Extended -############################################################################################################################### - # https://docs.perplexity.ai/docs/perplexity-bot User-agent: PerplexityBot