# =========================================
# Global Rules (All Crawlers)
# =========================================
User-agent: *
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Allow: /wp-admin/admin-ajax.php

# START WPFORMS BLOCK
User-agent: *
Disallow: /wp-content/uploads/wpforms/
# END WPFORMS BLOCK


# =========================================
# Major Search Engines
# =========================================

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /


# =========================================
# AI Search / Generative AI Crawlers (AISO READY)
# =========================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Microsoft Copilot / Bing AI
User-agent: BingAI
Allow: /

User-agent: MicrosoftPreview
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Amazon AI / Alexa
User-agent: Amazonbot
Allow: /

# Apple AI / Siri / Spotlight
User-agent: Applebot
Allow: /


# =========================================
# Voice Search & Assistants
# =========================================

User-agent: Google-Voice
Allow: /

User-agent: Assistant
Allow: /

User-agent: SiriBot
Allow: /

User-agent: AlexaBot
Allow: /

User-agent: Cortana
Allow: /


# =========================================
# News Discovery & Content Platforms
# =========================================

# Google News
User-agent: Googlebot-News
Allow: /

# Apple News
User-agent: AppleNewsBot
Allow: /

# Bing News
User-agent: BingNews
Allow: /

# Flipboard
User-agent: FlipboardProxy
Allow: /

# Feed Readers & Discovery
User-agent: Feedly
Allow: /

User-agent: Feedspot
Allow: /

User-agent: Feedspotbot
Allow: /

User-agent: Inoreader
Allow: /

User-agent: NewsBlur
Allow: /


# =========================================
# USA & Enterprise Crawlers / Indexers
# =========================================

User-agent: USABot
Allow: /

User-agent: CommonCrawl
Allow: /

User-agent: CCBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /


# =========================================
# Block Aggressive / Low-Value / Risk Bots
# =========================================

# TikTok / ByteDance (aggressive, low SEO value)
User-agent: Bytespider
Disallow: /

# Other heavy scrapers
User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /


# =========================================
# Sitemap
# =========================================
Sitemap: https://www.bitcot.com/sitemap_index.xml