# Block Meta crawlers from data pages. meta-externalagent was being used as a scraping
# proxy via the Facebook Graph API (?scrape=true) to bulk-harvest /addresses/ data at
# ~30 req/sec. meta-webindexer is Meta's general web indexer — hit 18k req/day on
# people/company pages after noindex was removed. Marketing pages remain crawlable so
# WhatsApp/Facebook link previews still work for company profiles and marketing pages.
# Note: robots.txt does not stop the Graph API scrape=true endpoint — the Cloudflare
# WAF rule scoped to /addresses/ is the effective block for that.
User-agent: meta-externalagent
Disallow: /addresses/
Disallow: /companies/
Disallow: /people/
Disallow: /search

User-agent: meta-webindexer
Disallow: /addresses/
Disallow: /companies/
Disallow: /people/
Disallow: /search

User-agent: facebookexternalhit
Disallow: /addresses/
Disallow: /companies/
Disallow: /people/
Disallow: /search

# Block AI training crawlers from bulk company data pages
# Marketing pages (/, /products, /pricing, /docs, /use-cases, etc.) remain crawlable
User-agent: GPTBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: anthropic-ai
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: ClaudeBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: CCBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: Google-Extended
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: PerplexityBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: YouBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: Amazonbot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

# Applebot (Siri Suggestions / Spotlight indexing) — separate from Applebot-Extended
# (AI training) above. Blocked from data pages after it was observed crawling
# /companies/, /people/, /addresses/ at ~100+ req/min (~150k+ req/day), ~95% of
# which is uncached and hits the origin directly — Apple states Applebot respects
# robots.txt. Marketing pages remain crawlable.
User-agent: Applebot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /officers/
Disallow: /search

User-agent: Applebot-Extended
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: DuckAssistBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: cohere-ai
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: Diffbot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

User-agent: HubSpot Crawler
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search

# Block live AI-search bots from data pages — these fetch URLs at query
# time to summarise answers in chat (no training, but also no click-through).
# Marketing pages remain crawlable so the assistants can still learn what
# CompanyLens is and surface it in conversations.
# NOTE: when a bot has a specific User-agent rule, robots.txt spec says it uses
# ONLY that rule (not User-agent: *), so /dashboard/, /api/, /v1/ must be listed
# explicitly here rather than relying on the wildcard block below.
User-agent: Claude-SearchBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

User-agent: OAI-SearchBot
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

User-agent: ChatGPT-User
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

User-agent: Perplexity-User
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

User-agent: Manus
Disallow: /companies/
Disallow: /people/
Disallow: /addresses/
Disallow: /search
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

# Block SEO/data crawlers that provide no value and generate significant load.
# These respect robots.txt — malicious scanners (l9scan, TLM-Audit-Scanner, Wget)
# are handled exclusively via the Cloudflare WAF rule.
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SleepBot
Disallow: /

User-agent: BizzLenz
Disallow: /

# Allow all other crawlers (Googlebot, Bingbot, etc.)
User-agent: *
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /v1/
Disallow: /subscribe

Sitemap: https://companylens.io/sitemap.xml
Sitemap: https://companylens.io/sitemap-index.xml
Sitemap: https://companylens.io/sitemap-companies