# Aggressive AI / SEO-scraper crawlers: still allowed to index, but asked to slow down so
# they don't hammer the data-heavy /sold-prices pages (nginx also rate-limits these per-IP
# as a hard backstop — see /etc/nginx/conf.d/ratelimit.conf on the server). Googlebot and
# Bingbot are intentionally NOT throttled. NOTE: robots.txt groups do not inherit from the
# "User-agent: *" group, so the Allow/Disallow rules below must mirror that group.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DataForSeoBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: PetalBot
User-agent: CCBot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: YandexBot
Crawl-delay: 10
Allow: /
Disallow: /Identity/
Disallow: /AgentsShortlist
Disallow: /my-shortlist
Disallow: /SellerDashboard
Allow: /vendor/online-valuation
Allow: /vendor/epc-checker
Allow: /vendor/listing-monitor
Allow: /vendor/agent-valuation
Disallow: /vendor/
Disallow: /missioncontrol
Disallow: /admin
Disallow: /my-reports
Disallow: /my-saved-reports
Disallow: /property-data-test
Disallow: /database-diagnostic
Disallow: /property-gallery
Disallow: /counter
Disallow: /fetchdata

User-agent: *
Allow: /
Disallow: /Identity/
Disallow: /AgentsShortlist
Disallow: /my-shortlist
Disallow: /SellerDashboard

# Vendor tools: allow public-facing pages, block authenticated dashboards
Allow: /vendor/online-valuation
Allow: /vendor/epc-checker
Allow: /vendor/listing-monitor
Allow: /vendor/agent-valuation
Disallow: /vendor/

# Admin, diagnostics & authenticated pages — not for indexing
Disallow: /missioncontrol
Disallow: /admin
Disallow: /my-reports
Disallow: /my-saved-reports
Disallow: /property-data-test
Disallow: /database-diagnostic
Disallow: /property-gallery
Disallow: /counter
Disallow: /fetchdata

Sitemap: https://agentseeker.co.uk/sitemap.xml