# ---------------------------------------------------------
# Personal Injury Solicitors London - ROBOTS.TXT
# Goal: maximise search and AI citation visibility while
# limiting access for selected model-training crawlers.
# ---------------------------------------------------------

# ---------------------------------------------------------
# GROUP 1: TRADITIONAL SEARCH CRAWLERS
# Required for Google Search, Bing Search, AI Overviews,
# AI Mode, Copilot-style search and normal organic discovery.
# ---------------------------------------------------------

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: bingbot
User-agent: Applebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=
Allow: /

# ---------------------------------------------------------
# GROUP 2: AI SEARCH, CITATION AND USER-REQUESTED FETCHERS
# These are allowed because they support AI search visibility,
# answer-engine citation, user-directed retrieval and linking.
# ---------------------------------------------------------

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=
Allow: /

# ---------------------------------------------------------
# GROUP 3: MODEL TRAINING OPT-OUT
# These crawlers are associated with model training or broader
# AI model development rather than direct search/citation.
# ---------------------------------------------------------

User-agent: GPTBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: Applebot-Extended
Disallow: /

# ---------------------------------------------------------
# GROUP 4: HIGH-VOLUME AI TRAINING / DATASET CRAWLERS
# Block unless you make a deliberate commercial decision to allow.
# ---------------------------------------------------------

User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: ImagesiftBot
Disallow: /

# ---------------------------------------------------------
# GROUP 5: GLOBAL CATCH-ALL
# Default access for ordinary crawlers, SEO tools and unknown bots.
# ---------------------------------------------------------

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cdn-cgi/
Disallow: /*seraph_accel_gp=

# ---------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------

Sitemap: https://www.personalinjurysolicitorslondon.co.uk/sitemap_index.xml