From 922836e207cea6fa229199cb0d1ead63545406f7 Mon Sep 17 00:00:00 2001
From: David Pine <7679720+IEvangelist@users.noreply.github.com>
Date: Mon, 6 Jul 2026 23:22:07 -0500
Subject: [PATCH] Add explicit AI crawler rules to robots.txt

Add per-agent User-agent groups for major AI/LLM crawlers (OpenAI, Anthropic, Google, Perplexity, Apple, Amazon, Meta, Mistral, DuckDuckGo, Cohere, Common Crawl) that explicitly Allow: / and carry the existing Content-Signal (ai-train=yes, search=yes, ai-input=yes), while preserving the wildcard default group and the sitemap reference.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 src/frontend/public/robots.txt | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/frontend/public/robots.txt b/src/frontend/public/robots.txt
index ad88bc9cd..41ec3e67a 100644
--- a/src/frontend/public/robots.txt
+++ b/src/frontend/public/robots.txt
@@ -1,3 +1,31 @@
+# robots.txt for https://aspire.dev
+# Aspire welcomes AI/LLM crawlers and assistants. For an AI-optimized index of
+# the site, see https://aspire.dev/llms.txt.
+
+# --- AI / LLM crawlers: explicitly allowed ---
+# Training, search indexing, and user-triggered assistant fetches are all
+# permitted, consistent with the site-wide Content-Signal declared below.
+User-agent: GPTBot
+User-agent: OAI-SearchBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-User
+User-agent: Claude-SearchBot
+User-agent: anthropic-ai
+User-agent: Google-Extended
+User-agent: PerplexityBot
+User-agent: Perplexity-User
+User-agent: Applebot-Extended
+User-agent: Amazonbot
+User-agent: Meta-ExternalAgent
+User-agent: MistralAI-User
+User-agent: DuckAssistBot
+User-agent: cohere-ai
+User-agent: CCBot
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
+Allow: /
+
+# --- Default: all other crawlers ---
 User-agent: *
 Content-Signal: ai-train=yes, search=yes, ai-input=yes
 Allow: /