# OpenBAZI - BaZi Astrology Platform
# robots.txt — last updated: 2026-06-08

# ── Standard web crawlers ────────────────────────────────────────────────────
User-agent: *
Allow: /

# Disallow private / functional areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /settings/
Disallow: /payment/
Disallow: /checkout/
Disallow: /user/
Disallow: /profile/

# Disallow dev / test pages
Disallow: /test/
Disallow: /dev/
Disallow: /debug/
Disallow: /temp/
Disallow: /env-test.html
Disallow: /performance-test.html
Disallow: /test-*.html
Disallow: /debug-*.html

# Disallow source & config file types that should never be public.
# JavaScript and CSS are intentionally crawlable so search engines can render the SPA.
Disallow: /*.ts$
Disallow: /*.tsx$
Disallow: /*.md$
Disallow: /*.sh$
Disallow: /*.config.js$
Disallow: /firebase-admin-key.json
Disallow: /*.json$

# Explicitly allow important public files (override /*.json$ and /*.txt$ above)
Allow: /manifest.json
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /llms.txt
Allow: /favicon.ico
Allow: /og-image.svg
Allow: /logo-horizontal.png

# ── Googlebot ────────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /payment/

# ── Bingbot ──────────────────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /payment/

# ── AI / LLM crawlers (GEO — Generative Engine Optimization) ─────────────────
# Allow all AI crawlers full access to public content

# OpenAI (ChatGPT, SearchGPT)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common AI research bots
User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

# ── Sitemap ───────────────────────────────────────────────────────────────────
Sitemap: https://www.openbazi.com/sitemap.xml