# Coderblock robots.txt
# Public marketing pages are indexable; app pages are not.
#
# ── AI crawler policy (piano M5a — founder decision, 2026-08-22) ──────────
# EXPLICIT choice, not a default: ALL AI crawlers get the same rules as any
# other bot — marketing pages allowed, app/api blocked below.
#   * Answer/search bots (OAI-SearchBot, Claude-SearchBot, PerplexityBot)
#     and user-triggered fetchers (ChatGPT-User, Claude-User,
#     Perplexity-User): WELCOME — they cite and send traffic.
#   * Training bots (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended,
#     CCBot, Meta-ExternalAgent): ADMITTED on marketing pages — models
#     learning our promotional material is the goal.
# One shared rule-group on purpose: per-bot groups with identical rules
# drift apart over time. If a future decision DIFFERS per bot, add a named
# group then. Re-verify agent names against official docs when editing.
# ──────────────────────────────────────────────────────────────────────────

User-agent: *
Allow: /
Allow: /features
Allow: /how-it-works
Allow: /pricing
Allow: /about
Allow: /waitlist
Allow: /offerta-lancio
Allow: /ai
Allow: /templates
Allow: /solutions
Allow: /blog
Allow: /tutorial
Allow: /learn
Allow: /news
Allow: /terms
Allow: /privacy
Allow: /cookie-policy

Allow: /login
Allow: /signup

Disallow: /apps
Disallow: /apps/
# v2 in costruzione (piano v2-public-pages, founder 03/09): cintura oltre al
# noindex — si rimuove SOLO a migrazione completata
Disallow: /v2
Disallow: /v2/
# Archivio v1 (swap 08/09): vecchie pagine parcheggiate sotto /v1, noindex
# in pagina + cintura qui — le canoniche sono le v2 prefissless.
Disallow: /v1
Disallow: /v1/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /forgot-password
Disallow: /sso-login
# /api/ is blocked EXCEPT the dynamic content sitemap (articles, referenced by
# the sitemap index) — the longer Allow rule wins over Disallow /api/ in
# Google's robots evaluation.
Allow: /api/v1/content/sitemap.xml
Disallow: /api/
Disallow: /teams

Sitemap: https://coderblock.ai/sitemap.xml