# robots.txt — agent-readiness compliant
# Satisfies isitagentready.com Discoverability and Bot Access Control checks.
# Content Signals follow the Cloudflare spec: https://contentsignals.org/
# Last reviewed: 2026-04-19

# ----------------------------------------------------------------------------
# Default policy — public marketing and docs pages are crawlable.
# Content-Signal declares preferences for downstream AI use (contentsignals.org
# draft-romm-aipref-contentsignals). Placed directly after User-agent so
# parsers see it before any Allow/Disallow rule.
# ----------------------------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /auth/
Disallow: /login/
Disallow: /admin/
Disallow: /oauth/
Disallow: /auth/mcp-connect/

# ----------------------------------------------------------------------------
# AI training crawlers — blocked.
# We do not license our content for model training.
# ----------------------------------------------------------------------------
User-agent: GPTBot
Content-Signal: ai-train=no
Disallow: /

User-agent: ClaudeBot
Content-Signal: ai-train=no
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ----------------------------------------------------------------------------
# AI answer / search assistants — allowed.
# These fetch live content to answer user questions. We want Metrifyr
# documentation to appear in their responses.
# ----------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot-User
Allow: /

User-agent: Claude-Web
Allow: /

# ----------------------------------------------------------------------------
# Discovery pointers
# ----------------------------------------------------------------------------
Sitemap: https://www.metrifyr.cloud/sitemap.xml