Sitemap: https://www.headway.org.uk/xmlsitemap

# AI Crawlers - Block
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Baiduspider
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: PerplexityBot
User-agent: YouBot
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: Diffbot
User-agent: omgilibot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: PetalBot
Disallow: /

# Search Engines - Allow
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
Allow: /
Disallow: /search/

# Allow Headway Public Website crawler
User-agent: Mozilla/5.0 SalesforceDataCloudCrawler headway_public_website
User-agent: Mozilla/5.0 SalesforceDataCloudCrawler HeadwayAgentforcePilot
User-agent: SalesforceDataCloudCrawler
User-agent: headway_public_website
User-agent: HeadwayAgentforcePilot
Allow: /

# Everyone else - slow down
User-agent: *
Crawl-delay: 10
Disallow: /