# BudgetFitter SEO Engine v2.0
# Optimized for LLM Agents & Semantic Search

User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Allow access to assets needed for page rendering
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/budgetfitter-web-theme/
Allow: /wp-content/plugins/budgetfitter-seo-engine/assets/

User-agent: Googlebot
Allow: /

User-agent: BingBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: archive.org_bot
Allow: /

# ── Tier 1: AI Real-Time Search Agents ──────────────────────────
# Live retrieval for AI responses. Full access.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Grok
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: ProRataInc
Allow: /

User-agent: Novellum
Allow: /

User-agent: Timpibot
Allow: /

# ── Tier 2: AI Training Data Crawlers ───────────────────────────
# Training pipelines. Restricted to AI-discovery files only.
# Content-Signal: search=no, ai-input=yes, ai-train=no
User-agent: Amazonbot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Allow: /.well-known/
Disallow: /

User-agent: Bytespider
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Allow: /.well-known/
Disallow: /

User-agent: CCBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.yaml
Allow: /.well-known/
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Disallow internal search and sorting
Disallow: /?s=
Disallow: /?deal=
Disallow: /?replytocom=
Disallow: /?orderby=
Disallow: /?filter=
Disallow: /*?lang=
Disallow: /*&lang=

# Block tag archives (internal use only)
Disallow: /tag/
Disallow: /tags/

# Affiliate redirect links (cloaked)
Disallow: /go/
Disallow: /link/

# LLM & AI Maps
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai-manifest.json
Allow: /openapi.yaml
Allow: /.well-known/

# Content Signals — AI content usage preferences
Content-Signal: search=yes, ai-input=yes, ai-train=no

Sitemap: https://budgetfitter.uk/sitemap_index.xml
Sitemap: https://budgetfitter.uk/llms.txt
Schemamap: https://budgetfitter.uk/sitemap_index.xml