# robots.txt for UnderstandingBaptism.com
# Maximum discoverability configuration - all crawlers welcome

# Allow all traditional search engine crawlers
User-agent: *
Allow: /

# Explicitly allow all major AI and LLM crawlers
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Gemini
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta/Facebook AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Apple AI
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft AI
User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Bytedance/TikTok
User-agent: Bytespider
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# AI2 (Allen Institute)
User-agent: AI2Bot
Allow: /

User-agent: AI2Bot-Dolma
Allow: /

# YouBot (You.com)
User-agent: YouBot
Allow: /

# Seekr
User-agent: Seekr
Allow: /

# Timpi
User-agent: Timpibot
Allow: /

# Duck.ai
User-agent: DuckAssistBot
Allow: /

# Omgili/Webz.io
User-agent: Omgili
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: webzio-extended
Allow: /

# DataForSEO
User-agent: DataForSeoBot
Allow: /

# Kangaroo Bot
User-agent: Kangaroo Bot
Allow: /

# PanguBot
User-agent: PanguBot
Allow: /

# img2dataset
User-agent: img2dataset
Allow: /

# ICC Crawler
User-agent: ICC-Crawler
Allow: /

# Scrapy
User-agent: Scrapy
Allow: /

# Firecrawl
User-agent: FirecrawlAgent
Allow: /

# Sitemap location
Sitemap: https://understandingbaptism.com/sitemap.xml