# robots.txt for ibl.ai
# https://ibl.ai

User-agent: *
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Block internal/API routes — but allow the Open Graph image endpoint so
# social scrapers (LinkedIn, X, Slack, Facebook) can fetch link-preview cards.
Allow: /api/og
Disallow: /api/
Disallow: /_vercel/
Disallow: /admin/
Disallow: /debug/
Disallow: /presentations/
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete

# Allow Next.js static assets (CSS, JS, images) for proper page rendering
# Only block internal data fetching routes
Disallow: /_next/data/

# /proposals.txt policy, in two tiers:
#   - INDEXING / TRAINING crawlers (below, and "User-agent: *") are disallowed —
#     the file is internal working material and must never be indexed, cached,
#     or trained on.
#   - USER-INITIATED fetchers (ChatGPT-User, Claude-User, Perplexity-User) are
#     ALLOWED. Those agents only retrieve a URL because a human in a live
#     session pointed at it; they do not index or train. Blocking them returned
#     ROBOTS_DISALLOWED to our own team pasting the link into a chat session.
#     The X-Robots-Tag: noindex on the file (next.config.mjs) is what keeps it
#     out of any index even after such a fetch.
#
# /sso-login and /sso-login-complete are disallowed for EVERY agent, INCLUDING
# the user-initiated fetchers — the one documented exception to the two-tier
# rule above. The criterion: a URL whose only content is a credential hand-off.
# /proposals.txt is material a human might legitimately paste into a chat
# session; these are the Auth-SPA callback, they render a spinner and redirect,
# and the URL they arrive on carries a live bearer token. There is nothing to
# fetch and every reason not to hand one to an agent.

# Specific AI bots (allowed for LLM training)
User-agent: Googlebot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: GPTBot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# OpenAI on-demand fetch (a user in ChatGPT asked about a specific URL) —
# user-initiated, not a crawler, so /proposals.txt is allowed.
User-agent: ChatGPT-User
Allow: /
Allow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: ClaudeBot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: Anthropic-AI
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Anthropic on-demand fetch (powers citations when a user asks Claude about a
# URL) — user-initiated, not a crawler, so /proposals.txt is allowed.
User-agent: Claude-User
Allow: /
Allow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Anthropic search-index crawler (powers Claude's cited search answers)
User-agent: Claude-SearchBot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# OpenAI ChatGPT Search crawler (powers answers/citations in ChatGPT)
User-agent: OAI-SearchBot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Perplexity crawler (indexes) — /proposals.txt disallowed.
User-agent: PerplexityBot
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Perplexity on-demand fetch — user-initiated, so /proposals.txt is allowed.
User-agent: Perplexity-User
Allow: /
Allow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Google-Extended — opt in to Gemini / AI Overviews grounding & training
User-agent: Google-Extended
Allow: /
Disallow: /proposals.txt
Disallow: /sso-login
Disallow: /sso-login-complete
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Sitemap location
Sitemap: https://ibl.ai/sitemap.xml