# Eatpol robots.txt
# Served identically on all domains: eatpol.com, eatpol.co.uk, eatpol.nl,
# eatpol.de, eatpol.it, eatpol.fr
#
# AI-readable company manifest: /llms.txt
# (each non-English domain serves its own localized manifest at that path —
#  see the llms.txt rewrites in netlify.toml)

User-agent: *
Allow: /
Allow: /js/
Allow: /css/
Allow: /img/
Allow: /fontawesome/

# Disallow sensitive files.
# NOTE: robots.txt supports only `*` and `$` — NOT regex. The previous
# `/.*\.php$` was read as a literal path beginning `/.`, so it matched nothing
# and the PHP files were never actually disallowed.
Disallow: /*.php$
Disallow: /*db-config*
Disallow: /admin/
Disallow: /.git/
Disallow: /.env
Disallow: /node_modules/

# Allow all PDFs and images
Allow: /*.pdf$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$

# Sitemaps. Google only acts on the sitemap that matches the host it fetched
# robots.txt from, so every domain finds its own here; the rest are ignored
# rather than harmful. Each domain also serves its sitemap at the canonical
# /sitemap.xml (rewritten in netlify.toml).
Sitemap: https://eatpol.com/sitemap.xml
Sitemap: https://eatpol.co.uk/sitemap-couk.xml
Sitemap: https://eatpol.nl/sitemap-nl.xml
Sitemap: https://eatpol.de/sitemap-de.xml
Sitemap: https://eatpol.it/sitemap-it.xml
Sitemap: https://eatpol.fr/sitemap-fr.xml

# ──────────────────────────────────────────────
# AI / answer-engine crawlers (GEO + AEO)
# ──────────────────────────────────────────────
# These are already permitted by the `User-agent: *` group above; the explicit
# groups exist so the intent is unambiguous and survives any future tightening
# of the wildcard rules. Eatpol publishes llms.txt specifically to be cited by
# these systems, so blocking them would defeat the point.
#
# Two distinct behaviours are covered:
#   - TRAINING / index crawlers (GPTBot, ClaudeBot, Google-Extended, …)
#   - LIVE RETRIEVAL agents that fetch a page to answer a question right now
#     (OAI-SearchBot, ChatGPT-User, Claude-User, Perplexity-User). Blocking
#     these removes Eatpol from AI answers even when the model already knows
#     the brand.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Bingbot
Allow: /

User-agent: cohere-ai
Allow: /

# ──────────────────────────────────────────────
# Other specific bots
# ──────────────────────────────────────────────
# AdsBot-Google crawls Google Ads landing pages to score their quality.
# It previously carried `Disallow: /test*`, which also matched /testers and
# /testers.html — the consumer app-install page, i.e. exactly the page any
# tester-recruitment ad campaign would point at. Scoped to the intended
# target (stray /test-* scratch pages) so ad landing pages stay crawlable.
User-agent: AdsBot-Google
Disallow: /test-*

User-agent: MJ12bot
Crawl-delay: 10