# Optimized robots.txt for physiciansgroupllc.com

# -----------------------------------------

# Default Rules for Search Engines & Live AI Retrieval Bots
# Includes Googlebot, Bingbot, OAI-SearchBot, PerplexityBot,
# Claude-User, Claude-SearchBot, and other standard crawlers

# -----------------------------------------

User-agent: *

# Block admin, dev, staging, backup, and low-value paths

Disallow: /admin/
Disallow: /wp-admin/
Disallow: /dev/
Disallow: /staging/
Disallow: /test/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /trackback/

# Allow Autoptimize cache (critical for CSS/JS rendering)
Allow: /wp-content/cache/autoptimize/

# Keep general cache blocked if it contains other non-essential items
Disallow: /wp-content/cache/

# Only keep this if /lp-forms/ is not a public landing page or conversion path
Disallow: /lp-forms/

# Optional low-value WordPress feeds
Disallow: /feed/

# Allow essential WordPress frontend functionality
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# Block internal search and duplicate-prone parameters
Disallow: /?s=*
Disallow: /*?preview=*
Disallow: /*?replytocom=*

# -----------------------------------------

# AI Training Crawler Controls
# Blocks training/model-development crawlers while preserving
# normal Google Search and live AI search/retrieval visibility

# -----------------------------------------

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

# -----------------------------------------

# Sitemap

# -----------------------------------------

Sitemap: https://physiciansgroupllc.com/sitemap_index.xml