diff --git a/src/frontend/public/robots.txt b/src/frontend/public/robots.txt index ad88bc9cd..41ec3e67a 100644 --- a/src/frontend/public/robots.txt +++ b/src/frontend/public/robots.txt @@ -1,3 +1,31 @@ +# robots.txt for https://aspire.dev +# Aspire welcomes AI/LLM crawlers and assistants. For an AI-optimized index of +# the site, see https://aspire.dev/llms.txt. + +# --- AI / LLM crawlers: explicitly allowed --- +# Training, search indexing, and user-triggered assistant fetches are all +# permitted, consistent with the site-wide Content-Signal declared below. +User-agent: GPTBot +User-agent: OAI-SearchBot +User-agent: ChatGPT-User +User-agent: ClaudeBot +User-agent: Claude-User +User-agent: Claude-SearchBot +User-agent: anthropic-ai +User-agent: Google-Extended +User-agent: PerplexityBot +User-agent: Perplexity-User +User-agent: Applebot-Extended +User-agent: Amazonbot +User-agent: Meta-ExternalAgent +User-agent: MistralAI-User +User-agent: DuckAssistBot +User-agent: cohere-ai +User-agent: CCBot +Content-Signal: ai-train=yes, search=yes, ai-input=yes +Allow: / + +# --- Default: all other crawlers --- User-agent: * Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: /