diff --git a/public/robots.txt b/public/robots.txt index 453f4f0dd3a..91c35f87fb9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -35,6 +35,7 @@ User-agent: Applebot-Extended Disallow: / # Common Crawl https://commoncrawl.org/ccbot +# "democratizing access to web information by producing and maintaining an open repository of web crawl data that is universally accessible and analyzable by anyone." User-agent: CCBot Disallow: /