diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 0000000..5575d73 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,23 @@ +# Used for many other (non-commercial) purposes as well +User-agent: CCBot +Disallow: / + +# For new training only +User-agent: GPTBot +Disallow: / + +# Not for training, only for user requests +User-agent: ChatGPT-User +Disallow: / + +# Marker for disabling Bard and Vertex AI +User-agent: Google-Extended +Disallow: / + +# Speech synthesis only? +User-agent: FacebookBot +Disallow: / + +# Multi-purpose, commercial uses; including LLMs +User-agent: Omgilibot +Disallow: / \ No newline at end of file