# === Googlebot ===
User-agent: Googlebot
Allow: /
Allow: /signup
Allow: /login
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

# === Social / Preview Bots ===
User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# =====================================================================
# AI crawlers
#
# These were previously split into "search = allow" and "training =
# block". That split cost us the thing we actually want.
#
# Being *recommended* by an assistant has two separate paths, and the
# retrieval-time crawler is only one of them:
#
#   1. Retrieval — the assistant fetches a page mid-answer and cites it.
#      OAI-SearchBot, ChatGPT-User, PerplexityBot, ClaudeBot.
#   2. Recall — the model already knows what Brainbaby is, because the
#      marketing pages were in its training corpus. No fetch, no
#      citation, no link required. This is the path that produces
#      "you could try Brainbaby" unprompted, and it is by far the more
#      valuable of the two.
#
# Blocking GPTBot, Google-Extended and CCBot closed path 2 entirely.
# CCBot in particular feeds Common Crawl, which is an input to nearly
# every frontier model — one blocked line removed us from the training
# data of models we are trying to be recommended by.
#
# The usual reason to block training crawlers is to protect proprietary
# content. That reasoning does not transfer here: everything these bots
# can reach is marketing copy whose entire purpose is to be repeated by
# other people. The private surface — chat, dashboard, health, company
# data — is behind auth and is disallowed below for every agent.
#
# Bytespider stays blocked: it is widely reported to ignore crawl-delay
# and robots directives, and it buys us no assistant visibility.
# =====================================================================

# --- Retrieval: fetch a page to answer a live question ---
User-agent: OAI-SearchBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: ChatGPT-User
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: PerplexityBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: ClaudeBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: Claude-SearchBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: YouBot
Allow: /

# --- Recall: corpus inclusion, so the model knows us without fetching ---
User-agent: GPTBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: Google-Extended
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: CCBot
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: anthropic-ai
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/

User-agent: meta-externalagent
Allow: /
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/

# --- Blocked: ignores directives, no visibility upside ---
User-agent: Bytespider
Disallow: /

# === Default ===
User-agent: *
Allow: /
Allow: /signup
Allow: /login
Allow: /glossary
Allow: /customers
Allow: /use-cases
Allow: /changelog
Allow: /status
Allow: /answers.json
Allow: /ai
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /dashboard$
Disallow: /dashboard?
Disallow: /dashboard/
Disallow: /chat$
Disallow: /chat?
Disallow: /chat/
Disallow: /video$
Disallow: /video?
Disallow: /video/
Disallow: /email$
Disallow: /email?
Disallow: /email/
Disallow: /calendar$
Disallow: /calendar?
Disallow: /calendar/
Disallow: /health$
Disallow: /health?
Disallow: /health/
Disallow: /companies$
Disallow: /companies?
Disallow: /companies/
Disallow: /settings$
Disallow: /settings?
Disallow: /settings/
Disallow: /admin$
Disallow: /admin?
Disallow: /admin/
Disallow: /checkout$
Disallow: /checkout?
Disallow: /checkout/
Disallow: /onboarding$
Disallow: /onboarding?
Disallow: /onboarding/
Disallow: /presentations$
Disallow: /presentations?
Disallow: /presentations/
Disallow: /auth$
Disallow: /auth?
Disallow: /auth/
Disallow: /delete-account$
Disallow: /delete-account?
Disallow: /delete-account/

Sitemap: https://brainbaby.ai/sitemap.xml