# WatchPolice robots.txt
# Bulk collection of watch/brand data is prohibited without written permission.
# Terms: https://watchpolice.com/terms (Section 7 — Automated Data Collection)
# LLM-friendly endpoints: https://watchpolice.com/llms.txt
# Per-brand AI summaries: https://watchpolice.com/api/llm-summary/:slug
# Facts API: https://watchpolice.com/api/facts.json

User-agent: *
Allow: /api/sitemap
Allow: /api/sitemap-index.xml
Allow: /api/llm-summary
Allow: /api/facts.json
Disallow: /api/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /sign-up
Disallow: /sign-in
Disallow: /settings
Disallow: /my-collection
Disallow: /admin/
Disallow: /embed/
Disallow: /brands/*/manage
Disallow: /brands/*/claim
Disallow: /brands/claim/
Disallow: /u/*/edit
Disallow: /u/create

# ── AI search assistants — allowed to index and summarise ─────────────────
# These assistants refer traffic back to the site and respect crawl limits.
# NOTE: a named User-agent group fully REPLACES the `*` group for that bot, so
# the LLM-endpoint Allow lines must be repeated in every group below or these
# assistants would inherit only their own Disallow: /api/ and be locked out of
# /api/llm-summary and /api/facts.json.

User-agent: Google-Extended
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: DuckAssistBot
Allow: /
Allow: /api/sitemap
Allow: /api/sitemap-index.xml
Allow: /api/llm-summary
Allow: /api/facts.json
Disallow: /api/
Disallow: /admin/
Disallow: /embed/

# ── AI training crawlers — bulk data collection disallowed ────────────────
# Proprietary watch/brand data must not be ingested for model training.
# (Applebot-Extended is Apple's training opt-in token — blocked here; the
#  regular Applebot search crawler still indexes via the `*` group above.)

User-agent: GPTBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Applebot-Extended
Disallow: /

# ── SEO / backlink & marketing audit crawlers — fully disallowed ──────────
# High-volume commercial crawlers that send no referral traffic. Blocked to
# reduce server load and prevent competitive backlink / data harvesting.
# (The LLM search assistants above remain fully allowed.)
# NOTE: "spoofed" SemrushBot traffic uses a faked user-agent and ignores
# robots.txt entirely — it can only be stopped server-side, not here.

User-agent: Barkrowler
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SWA
User-agent: SiteAuditBot
User-agent: MJ12bot
User-agent: PetalBot
User-agent: HubSpot Crawler
Disallow: /

# ── Search-engine spiders — disallowed by request ─────────────────────────
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou Pic Spider
Disallow: /

Sitemap: https://watchpolice.com/sitemap.xml
Sitemap: https://watchpolice.com/api/sitemap-index.xml