# ═══════════════════════════════════════════════════════════ # BENO.COM — robots.txt # Updated: June 2026 # Purpose: Allow all AI crawlers for GEO/AEO optimisation # ═══════════════════════════════════════════════════════════ # ── GOOGLE ────────────────────────────────────────────────── # Googlebot = Google Search rankings (critical) # Google-Extended = Gemini + AI Overview training (critical for GEO) # GoogleOther = Google product teams, Bard, SGE User-agent: Googlebot Allow: / User-agent: Google-Extended Allow: / User-agent: GoogleOther Allow: / # ── OPENAI / CHATGPT ──────────────────────────────────────── # GPTBot = ChatGPT training data (brand visibility in GPT) # OAI-SearchBot = ChatGPT live search index (citation critical) # ChatGPT-User = User-triggered live fetches (direct referral traffic) User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / # ── ANTHROPIC / CLAUDE ────────────────────────────────────── # Anthropic updated bot docs Feb 2026 — three separate bots # ClaudeBot = Training data only (does NOT drive citations) # Claude-SearchBot = Claude search indexing (blocking = invisible in Claude) # Claude-User = User query live fetches (direct citation traffic) # Note: Each bot is fully independent — one rule does NOT cover others User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / # ── PERPLEXITY ────────────────────────────────────────────── # PerplexityBot = Standard crawler + search index # Perplexity-User = User-triggered fetches (may not follow robots.txt) User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # ── MICROSOFT COPILOT ─────────────────────────────────────── # Bingbot = Bing Search AND Microsoft Copilot AI (one crawler, two products) User-agent: Bingbot Allow: / # ── META AI ───────────────────────────────────────────────── # Meta-ExternalAgent = AI training + indexing (16.7% of AI crawler traffic) # Meta-ExternalFetcher = Real-time fetches for Meta AI on Instagram/WhatsApp User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / # ── APPLE INTELLIGENCE ────────────────────────────────────── # Applebot = Apple Search + Siri # Applebot-Extended = Apple Intelligence AI training (activity up 2026) User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # ── AMAZON ────────────────────────────────────────────────── # Amazonbot = Amazon Alexa AI + product knowledge base User-agent: Amazonbot Allow: / # ── ALL OTHER CRAWLERS ────────────────────────────────────── # Wildcard covers any bot not explicitly named above # Disallow: /error = Error page, no value to crawlers # Disallow: /v3/ = API endpoints, not for public indexing # NOTE: /*.js$ and /*.json$ rules have been REMOVED # Reason: Blocking .js files prevents Google from rendering pages # and reading JavaScript-based schema. This was hurting SEO and # AI indexing. All legitimate .js and .json files should be crawlable. User-agent: * Allow: / Disallow: /error Disallow: /v3/ # ── SITEMAP ───────────────────────────────────────────────── Sitemap: https://www.beno.com/sitemap.xml