# Allow Google ad crawlers to verify public ad metadata.
User-agent: Mediapartners-Google
User-agent: Google-Display-Ads-Bot
Allow: /
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /builder
Disallow: /checkout/
Disallow: /decks/builder
Disallow: /login
Disallow: /me/
Disallow: /upgrade/plus-check
Disallow: /assets/
Disallow: /404_
Disallow: /500_
Disallow: /matchmaking

# Allow public pages for search and AI-search discovery.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /builder
Disallow: /clan/*/tracking
Disallow: /clan/*/plus/daily-tracking
Disallow: /checkout/
Disallow: /decks/builder
Disallow: /login
Disallow: /me/
Disallow: /open/
Disallow: /player/*/battles/war
Disallow: /upgrade/plus-check
Disallow: /assets/
Disallow: /404_
Disallow: /500_
Disallow: /matchmaking

# OpenAI search visibility. This is separate from GPTBot training.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /builder
Disallow: /clan/*/tracking
Disallow: /clan/*/plus/daily-tracking
Disallow: /checkout/
Disallow: /decks/builder
Disallow: /login
Disallow: /me/
Disallow: /open/
Disallow: /player/*/battles/war
Disallow: /upgrade/plus-check
Disallow: /assets/
Disallow: /404_
Disallow: /500_
Disallow: /matchmaking

# Allow user-triggered assistant fetches for public pages.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /builder
Disallow: /checkout/
Disallow: /decks/builder
Disallow: /login
Disallow: /me/
Disallow: /upgrade/plus-check
Disallow: /assets/
Disallow: /404_
Disallow: /500_
Disallow: /matchmaking

# Block AI training and broad AI dataset crawlers.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Block noisy SEO and bulk crawlers from the old site posture.
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

Sitemap: https://cwstats.com/sitemap.xml