# Bot Appétit (botappetit.ai) — an AEO/SEO tool that eats its own cooking.
# We WANT both search engines and AI answer engines to read and cite this site.

User-agent: *
Allow: /
Disallow: /r/
Disallow: /site-scan/
Content-Signal: search=yes, ai-input=yes

# Major AI answer-engine crawlers — explicitly welcomed (deliberate intent, not just the * default).
User-agent: GPTBot
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: OAI-SearchBot
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: ChatGPT-User
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: ClaudeBot
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: Claude-User
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: PerplexityBot
Allow: /
Disallow: /r/
Disallow: /site-scan/

User-agent: Google-Extended
Allow: /
Disallow: /r/
Disallow: /site-scan/

Sitemap: https://botappetit.ai/sitemap-index.xml
Sitemap: https://botappetit.ai/state-of-the-web-sitemap.xml

# Site-scan reports (/site-scan/<job-id>) hold the SAME third-party findings at site scale, and are
# disallowed for the same reason and in the same groups (added 2026-08-26, with shareable links).
# ⚠️ THE TRAILING SLASH IS LOAD-BEARING. `Disallow: /site-scan/` covers `/site-scan/<job-id>` and NOT
# `/site-scan` itself, so the page people actually start a scan from stays crawlable and indexable.
#
# Scan permalinks (/r/<id>) hold results for OTHER PEOPLE'S websites. Publishing third-party audit
# findings into Google — or into an AI answer engine — under our own domain is not ours to do, so
# /r/ is disallowed in EVERY group above, not just `*`: RFC 9309 says a crawler obeys the single
# most-specific matching group and ignores the rest, so a rule in `*` alone would never reach the
# named AI crawlers. The route also sends X-Robots-Tag: noindex, which fails differently — robots.txt
# stops the crawl, the header stops indexing of a URL discovered some other way.