# ======================================================
#   Elite Robots.txt (2026 Edition)
#   Strategy: High-Visibility Allowlist; Scraper Blocklist
#   Extima fleet standard — canonical source:
#   _operations/hosting/fleet-standards/robots.txt.template
# ======================================================

# -------------------------
# Global Directives (single, merged * block)
# NOTE: there must be exactly ONE "User-agent: *" block in this file.
# Crawlers obey only the FIRST matching group — a second * block is
# silently ignored. (This is the bug that shipped in the v1 file.)
# -------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Disallow: /wp-json/wp/v2/
Disallow: /readme.html
Disallow: /refer/
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /cgi-bin/
# Crawl traps — infinite/duplicate URL surfaces that waste crawl budget.
# (Adopted fleet-wide 2026-07-21 from the connections-marcom file.)
Disallow: /*?amp
Disallow: /*?replytocom
# WP-Optimize's plugin-tables dump. WP-Optimize appends this itself via the
# robots_txt filter WITH ITS OWN "User-agent: *" line — which creates a second
# * block and silently voids everything after it. Declaring it here means the
# virtual injection is redundant; on a physical file the filter never runs.
Disallow: /wp-content/uploads/wpo/wpo-plugins-tables-list.json

# -------------------------
# SECTION 1: Beneficial AI & Search
# Logic: These bots index you for Search, AI Overviews, and Copilots.
# -------------------------
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Google-Extended
User-agent: bingbot
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: SemrushBot
Allow: /
Allow: /wp-json/wp/v2/

# -------------------------
# SECTION 2: Social Media & Discovery
# Logic: Ensures rich previews on Facebook, X, LinkedIn, etc.
# -------------------------
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: Xbot
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: Discordbot
User-agent: Slackbot
User-agent: Bluesky
User-agent: ByteDance
Allow: /
Allow: /wp-json/wp/v2/

# -------------------------
# SECTION 3: Explicit Blocklist (The "No Value" Zone)
# Logic: Block scrapers and training bots that don't send traffic.
# -------------------------
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: Bytespider
User-agent: Yandex
User-agent: DataForSeoBot
User-agent: IASpider
User-agent: Diffbot
Disallow: /

# -------------------------
# SECTION 4: AI content policy pointer
# Logic: points LLM crawlers at a machine-readable usage policy. Harmless if
# /llms.txt is absent — it is a hint, not a directive. Adopted fleet-wide
# 2026-07-21 (pattern taken from full-circle-artists).
# NOTE: only meaningful on sites that actually publish /llms.txt. Rank Math's
# "llms-txt" module generates one; confirm before relying on it.
# -------------------------
LLM-Policy: /llms.txt

Sitemap: https://cplinc.com/sitemap_index.xml