# robots.txt for SEO and AI/LLM crawlers
# Place at https://your-domain/robots.txt

# Sitemap location
Sitemap: https://citespark.com/api/sitemap/sitemap.xml

# Default: allow everything for all bots
User-agent: *
Allow: /

# Explicitly allow AI-related crawlers for training/indexing
# OpenAI (training crawler)
User-agent: GPTBot
Allow: /

# OpenAI (ChatGPT browsing fetcher)
User-agent: ChatGPT-User
Allow: /

# Google AI training control (separate from Googlebot)
User-agent: Google-Extended
Allow: /

# Common Crawl (data used by many AI systems)
User-agent: CCBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /