# ═══════════════════════════════════════════════════════════════════
# Little Wheels - robots.txt
# Last updated: 2025-12-11
# Humans: https://littlewheels.app/humans.txt
# Educational toddler apps - vehicle-themed speech & creative learning
# ═══════════════════════════════════════════════════════════════════

# Default rules for all crawlers
User-agent: *
Disallow: /admin/
Disallow: /api/
# /llm/*.txt files are for AI crawlers (llms.txt), not web search results.
# Deliberately NOT Disallow'd: these are served with `X-Robots-Tag: noindex,
# follow` (see server/index.ts), which keeps them out of search results while
# still letting any crawler fetch them. Disallow did the opposite of what we
# wanted - it blocked every agent without a named Allow group below, including
# ones following the ~277 paths advertised in llms.txt, and a Disallow'd URL can
# still be indexed from external links, just with no content to show for it.
Allow: /llm/
# Machine-readable API surface. `Disallow: /api/` above keeps crawlers out of
# anything under /api/ that is not a published document; these are published
# documents, and llms.txt, agents.md and /developers all link to them by name.
# A crawler that is told about a URL and then forbidden from fetching it is the
# same failure mode the /llm/ note above describes.
Allow: /api/features.json
Allow: /api/apps.json
Allow: /api/vehicles.json
Allow: /api/curriculum.json
Allow: /api/index.json
Allow: /api/tools.json
Allow: /api/openapi.json
Allow: /openapi.json
Allow: /agents.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# App Store redirect utility pages (always noindex,nofollow; linked from 100+
# vehicle pages, nav, footer, hero, sticky CTA, etc. with distinct ?src=
# tracking params). Blocking here prevents crawlers from wasting crawl
# budget fetching many near-duplicate query-string variants of the same
# redirect page. Real users are unaffected; this only blocks bots.
Disallow: /go/

# ═══════════════════════════════════════════════════════════════════
# LLM CRAWLER DIRECTIVES
# We welcome AI crawlers to index our educational content
# ═══════════════════════════════════════════════════════════════════

# OpenAI (GPT-4, ChatGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Allow: /

# Google AI (Gemini, Bard)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /

# Perplexity AI
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# xAI (Grok)
User-agent: Grok
Allow: /

# DeepSeek
User-agent: DeepSeek
Allow: /

# Meta AI (Llama)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /

# Microsoft/Bing AI (Copilot)
User-agent: Bingbot
User-agent: BingPreview
Allow: /

# Common Crawl (training data for many AI systems)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Apple (Siri, Apple Intelligence)
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Amazon (Alexa)
User-agent: Amazonbot
Allow: /

# Brave Search AI
User-agent: Brave
Allow: /

# ByteDance (TikTok AI)
User-agent: Bytespider
Allow: /

# ═══════════════════════════════════════════════════════════════════
# SITEMAPS & AI CONTENT
# ═══════════════════════════════════════════════════════════════════
Sitemap: https://littlewheels.app/sitemap.xml

# LLM-Optimized Content
# llms.txt: Structured index of AI-friendly content
# llms-full.txt: Complete concatenated content for full context
# See: https://littlewheels.app/llms.txt