User-agent: *
Disallow: /errors/
Disallow: /fancybox/
Disallow: /fancybox*
Disallow: /layout/
Disallow: /mode/
Disallow: /media/
Disallow: /print/
Disallow: /scripts/
Disallow: /search/
Disallow: /subs/
Disallow: /tmp/
Disallow: /webadmin/
Disallow: /webadmin*
Disallow: /xml/


#block crawlers from entire website

User-agent: bingbot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: python
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: crawl
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: bot
Disallow: /

User-agent: crawl
Disallow: /

User-agent: crawler
Disallow: /



# ============================================================
# AI CRAWLER POLICY (2026)
# ============================================================

# --- Citation & live-answer bots: MUST be allowed ---
User-agent: OAI-SearchBot
#Allow: /
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

# --- Training bots: also allowed (visibility strategy).
# Only block these if a content-licensing position ever requires it.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

#or block when to intense
#Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /