User-agent: *
Allow: /

# Block legacy WordPress admin and auth surfaces from being indexed
# (Netlify will 404 these anyway, but make it explicit for crawlers).
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-content/
Disallow: /wp-includes/

# Block WordPress legacy URL patterns that all 301-redirect via netlify.toml.
# These have no crawl value and generate large amounts of wasted traffic when
# bots enumerate them or construct malformed URLs by mis-resolving relative
# hrefs on blog nav (see 2026.8 CSP + crawler audit).
Disallow: /category/
Disallow: /tag/
Disallow: /author/
Disallow: /feed/
Disallow: /comments/
Disallow: /2023/
Disallow: /2024/
Disallow: /2025/
Disallow: /2026/

# Query-string variants Google flagged as duplicates
Disallow: /?p=
Disallow: /?page_id=

# Utility embed views (already noindex on the pages themselves; belt-and-braces)
Disallow: /faq-embed/
Disallow: /case-studies-embed/

# Post-booking / gated pages that should not be crawled
Disallow: /call-booked

Sitemap: https://xfusion.io/sitemap.xml

# Page-shell sources used by the prerender build; not for indexing.
Disallow: /*.src.html

# --------------------------------------------------------------------
# AI training crawlers — throttle to stop bandwidth burn.
# Their fetches are for training/indexing, not for real-time answers,
# so slowing them down doesn't hurt discoverability. Real-time answer
# bots (Claude-User, ChatGPT-User, Perplexity-User, OAI-SearchBot,
# Claude-SearchBot) are intentionally left unrestricted below.
# --------------------------------------------------------------------

# Anthropic AI training
User-agent: ClaudeBot
Crawl-delay: 30

# Anthropic real-time — unrestricted (small footprint anyway)
User-agent: Claude-User
Crawl-delay: 5

User-agent: Claude-SearchBot
Crawl-delay: 5

# Anthropic older UA (belt-and-braces)
User-agent: anthropic-ai
Crawl-delay: 30

# Meta AI training + agent fetcher
User-agent: meta-externalagent
Crawl-delay: 30

User-agent: meta-externalfetcher
Crawl-delay: 30

# Meta link preview (facebookexternalhit) is intentionally not restricted
# so shared xFusion links keep rendering rich cards on Facebook/Instagram/Threads.

# OpenAI AI training
User-agent: GPTBot
Crawl-delay: 30

# OpenAI real-time (ChatGPT-User, OAI-SearchBot) intentionally unrestricted.

# Perplexity training
User-agent: PerplexityBot
Crawl-delay: 30

# Perplexity real-time (Perplexity-User) intentionally unrestricted.

# Common Crawl (feeds many AI training pipelines)
User-agent: CCBot
Crawl-delay: 60

# Amazon AI training
User-agent: Amazonbot
Crawl-delay: 30

# Apple AI training (separate from Applebot search — search is unrestricted)
User-agent: Applebot-Extended
Crawl-delay: 30

# Google AI training (separate from Googlebot search — search is unrestricted)
User-agent: Google-Extended
Crawl-delay: 30

# --------------------------------------------------------------------
# ByteDance / TikTok AI training — full block.
# Zero B2B customer-support discovery value for xFusion (feeds TikTok
# recommendation + Doubao AI). Also documented aggressive behavior that
# does not reliably honor Crawl-delay. Escalated from throttle to block
# after the 2026.8 crawler audit.
# --------------------------------------------------------------------
User-agent: Bytespider
Disallow: /

# --------------------------------------------------------------------
# Search crawlers (Googlebot, Bingbot, DuckDuckBot, Applebot) are
# intentionally NOT restricted here — they carry SEO value. They
# still honor the User-agent: * Disallow rules above for the WP
# legacy paths, which is what we want.
# --------------------------------------------------------------------