User-agent: *
Allow: /

# Account / auth surfaces — never useful to index, must not leak
# user-specific URLs into search results. Prefix form (no trailing slash)
# covers both the bare route (/admin) and everything under it (/admin/...).
Disallow: /admin
# Public developer docs live under /api-guide and /api-docs — keep them
# crawlable. Only the JSON API itself (/api/...) is blocked, hence the
# trailing slash: "/api/" matches /api/foo but NOT /api-guide or /api-docs.
Allow: /api-guide
Disallow: /api/
Disallow: /dashboard
Disallow: /profile
Disallow: /subscription
Disallow: /partner
Disallow: /support
Disallow: /onboarding
Disallow: /auth
Disallow: /verify-email
Disallow: /reset-password
Disallow: /forgot-password
# OAuth consent screen — a functional step in a flow, not a document. It was
# answering 200 with robots "index, follow", so it was eligible for the index.
Disallow: /oauth
# Internal design/preview scratch pages (/dev/design-system, /dev/blog-rich-
# content). Same problem: reachable, indexable, and of no use in search.
Disallow: /dev

# Crawl politeness — most large bots respect this; SemrushBot/Ahrefs
# typically don't, but the targeted block below handles them.
Crawl-delay: 1

# Aggressive SEO scrapers — heavy on origin, no benefit to us.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# AI / LLM engines (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, …) are
# intentionally allowed by the wildcard rule above — astrocodex content is
# meant to be citable. LLM content index (GEO):
# LLM-Index: https://astrocodex.ai/llms.txt
# AI usage policy: https://astrocodex.ai/.well-known/ai.txt

Sitemap: https://astrocodex.ai/sitemap.xml