# robots.txt - https://scaledown.ai
# RFC 9309: https://www.rfc-editor.org/rfc/rfc9309

# ── general crawlers ─────────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /settings
Disallow: /getapikey
Disallow: /blog/admin

# ── ai search crawlers (indexing allowed, no training) ────────────────────────
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /settings
Disallow: /getapikey
Disallow: /blog/admin

User-agent: PerplexityBot
Allow: /
Disallow: /dashboard
Disallow: /profile
Disallow: /settings
Disallow: /getapikey
Disallow: /blog/admin

# ── ai training crawlers (disallowed) ────────────────────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

# ── content signals (https://contentsignals.org) ─────────────────────────────
Content-Signal: ai-train=no, search=yes, ai-input=no

# ── sitemap ───────────────────────────────────────────────────────────────────
Sitemap: https://scaledown.ai/sitemap.xml