# ER Of Fort Worth — robots.txt
# https://emergencyroomfortworth.com/
# Last updated: 2026-06-25
# Policy: Index everything. Block only the private admin panel.

# ============================================================
# GLOBAL — all compliant crawlers welcome
# ============================================================
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /admin/*

# ============================================================
# GOOGLE
# ============================================================
User-agent: Googlebot
Allow: /
Disallow: /admin/

User-agent: Googlebot-Image
Allow: /assets/images/

User-agent: Googlebot-News
Allow: /blog/

User-agent: Googlebot-Video
Allow: /

# Google AI (Gemini / AI Overviews training) — allow
User-agent: Google-Extended
Allow: /

User-agent: AdsBot-Google
Allow: /

# ============================================================
# MICROSOFT / BING
# ============================================================
User-agent: Bingbot
Allow: /
Disallow: /admin/

User-agent: BingPreview
Allow: /

User-agent: MicrosoftPreview
Allow: /

# ============================================================
# OPENAI / CHATGPT
# ============================================================
# GPTBot — ChatGPT training and browsing
User-agent: GPTBot
Allow: /

# OAI-SearchBot — OpenAI search
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User — live ChatGPT web browsing
User-agent: ChatGPT-User
Allow: /

# ============================================================
# ANTHROPIC / CLAUDE
# ============================================================
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# ============================================================
# PERPLEXITY AI
# ============================================================
User-agent: PerplexityBot
Allow: /

# ============================================================
# META AI (Facebook / Instagram AI)
# ============================================================
User-agent: facebookexternalhit
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ============================================================
# APPLE (Siri / Apple Intelligence)
# ============================================================
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ============================================================
# AMAZON ALEXA / AMAZON AI
# ============================================================
User-agent: Amazonbot
Allow: /

# ============================================================
# YOU.COM AI SEARCH
# ============================================================
User-agent: YouBot
Allow: /

# ============================================================
# COHERE AI
# ============================================================
User-agent: cohere-ai
Allow: /

# ============================================================
# BYTEDANCE / TIKTOK AI
# ============================================================
User-agent: Bytespider
Allow: /

# ============================================================
# DIFFBOT (knowledge graph / AI extraction)
# ============================================================
User-agent: Diffbot
Allow: /

# ============================================================
# COMMON CRAWL (used to train GPT-3/4, LLaMA, Mistral, etc.)
# ============================================================
User-agent: CCBot
Allow: /

# ============================================================
# INTERNET ARCHIVE (Wayback Machine)
# ============================================================
User-agent: ia_archiver
Allow: /

# ============================================================
# OTHER SEARCH ENGINES
# ============================================================
User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Sogou
Allow: /

User-agent: Exabot
Allow: /

# ============================================================
# SEO AUDIT TOOLS
# ============================================================
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DataForSeoBot
Allow: /

# ============================================================
# SITEMAP
# ============================================================
Sitemap: https://emergencyroomfortworth.com/sitemap.xml