# ── Googlebot (Search + Images + News + Video) ───────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /assets/

User-agent: Googlebot-Image
Allow: /
Disallow: /assets/

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

# ── Google AdsBot ─────────────────────────────────────────────────────────────
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

# ── Google AI (SGE / Gemini training) ────────────────────────────────────────
User-agent: Google-Extended
Allow: /

# ── Bing / Microsoft ─────────────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /assets/

User-agent: msnbot
Allow: /

User-agent: msnbot-media
Allow: /

# ── OpenAI / ChatGPT ─────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ── Anthropic / Claude ───────────────────────────────────────────────────────
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# ── Perplexity ───────────────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

# ── Apple / Applebot ─────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

# ── DuckDuckGo ───────────────────────────────────────────────────────────────
User-agent: DuckDuckBot
Allow: /

# ── Yahoo ────────────────────────────────────────────────────────────────────
User-agent: Slurp
Allow: /

# ── Yandex ───────────────────────────────────────────────────────────────────
User-agent: YandexBot
Allow: /

# ── Facebook / Meta (link preview) ───────────────────────────────────────────
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# ── Twitter / X (link preview) ───────────────────────────────────────────────
User-agent: Twitterbot
Allow: /

# ── LinkedIn (link preview) ──────────────────────────────────────────────────
User-agent: LinkedInBot
Allow: /

# ── Telegram (link preview) ──────────────────────────────────────────────────
User-agent: TelegramBot
Allow: /

# ── Archive.org (Wayback Machine) ────────────────────────────────────────────
User-agent: ia_archiver
Allow: /

# ══════════════════════════════════════════════════════════════════════════════
# BLOCK — Known scrapers, content thieves, AI training harvesters
# ══════════════════════════════════════════════════════════════════════════════

# Python scrapers (requests, scrapy, httpx default UA)
User-agent: python-requests
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: httpx
Disallow: /

User-agent: aiohttp
Disallow: /

# Generic curl / wget
User-agent: curl
Disallow: /

User-agent: Wget
Disallow: /

# Known content scrapers / spam bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: Exabot
Disallow: /

# AI training harvesters (opt-out)
User-agent: CCBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

# ─── Fallback: all other bots ────────────────────────────────────────────────
# Allow crawling the entire site (blog posts + store product pages) for SEO.
User-agent: *
Allow: /
Allow: /blog$
Allow: /blog/
Allow: /store$
Allow: /store/
Disallow: /assets/
Crawl-delay: 10

# ─── Sitemap ────────────────────────────────────────────────────────────────
Sitemap: https://cryplink.xyz/sitemap.xml