# robots.txt for Claire AI - TCF Canada Prep Platform
# Updated: 2026-04-22
# LLM crawlers: see /llms.txt for a structured site description
# AI content discovery: /.well-known/ai-answers.json
# IndexNow: /indexnow-key.txt (Bing instant indexing)
#
# Note on ?lang=xx query params: NOT disallowed here. Edge middleware
# (middleware.ts) returns a 301 redirect to the canonical path-prefixed URL
# (e.g. /en/listening/40?lang=en → /en/listening/40). If we added
# `Disallow: /*?lang=`, Google would stop crawling those URLs, never see
# the 301, and the historical entries stay stuck in the index instead of
# merging to canonical. Leave it crawlable so the 301 can do its job.

# Google — primary search engine, no crawl delay
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /reset-password
Disallow: /refund
Disallow: /mistake-book/practice
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# Google AI / Gemini
User-agent: GoogleOther
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# Bing — no crawl-delay to maximize indexing speed
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /reset-password
Disallow: /refund
Disallow: /mistake-book/practice
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# Bing preview bot
User-agent: BingPreview
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# MSNBot (legacy Bing)
User-agent: msnbot
Allow: /
Disallow: /admin
Disallow: /account
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# Major LLM crawlers — allowed for GEO indexing, but not assets
User-agent: GPTBot
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: OAI-SearchBot
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: ChatGPT-User
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: ClaudeBot
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: PerplexityBot
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: Google-Extended
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: Applebot-Extended
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: cohere-ai
Allow: /
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /account
Disallow: /reset-password
Disallow: /refund
Disallow: /mistake-book/practice
Disallow: /audio/
Disallow: /images/reading/
Disallow: /images/listening/

# Sitemaps — multilingual (static + dynamic)
Sitemap: https://www.tcfcanada.ai/sitemap.xml
Sitemap: https://www.tcfcanada.ai/sitemap-zh.xml
Sitemap: https://www.tcfcanada.ai/sitemap-en.xml
Sitemap: https://www.tcfcanada.ai/sitemap-fr.xml
Sitemap: https://www.tcfcanada.ai/api/sitemap-guides