# robots.txt for CoSama
#
# Policy: this site is open to AI crawlers and answer engines by design,
# including model-training crawlers. That is deliberate, not an oversight.
# Every agent below is named explicitly so the intent is unambiguous.
#
# No opt-out tokens appear here. Google-Extended and Applebot-Extended are
# robots.txt controls rather than crawlers — both vendors state they issue
# no HTTP requests — and absence of a Disallow is already the permissive
# state. Leaving them out is how we decline to opt out of Gemini and Apple
# Intelligence training.
#
# The Disallow rules repeated in every block are crawl-budget hygiene
# (no content lives at those paths). They are not a stance on AI.
#
# No Crawl-delay anywhere: full-speed crawl is intended.
#
# Generated from src/seo/crawlers.ts. Do not hand-edit.

# --- Search engines ---
# Google Search primary crawler.
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Bing. Lowercase is Microsoft's own spelling; matching is case-insensitive either way. [token not confirmed against vendor docs]
User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# DuckDuckGo's own search crawler.
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Powers Siri, Spotlight and Safari suggestions.
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# --- Answer engines and retrieval ---
# ChatGPT search. Blocking it removes the site from ChatGPT search answers.
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Anthropic's search-quality crawler.
User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Perplexity's search index. Documented as not used for foundation-model training. [observed behaviour diverges from documented policy; rules here are not enforceable]
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Powers DuckDuckGo's AI-assisted answers. DuckDuckGo states this data is not used to train models, and opting out does not affect organic ranking.
User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Mistral's search index. Documented as indexing only, not training.
User-agent: MistralAI-Index
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Fetches pages for Vertex AI Agents, and only when a site owner has asked it to. It will not appear in logs unless someone builds such an agent against this site. Listed for completeness, not because traffic is expected.
User-agent: Google-CloudVertexBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# --- User-triggered fetchers (a person asked; not scheduled crawling) ---
# User-initiated ChatGPT and Custom GPT fetches. [vendor states robots.txt may not apply to this agent]
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# User-initiated Claude fetches. One of only two user-triggered fetchers that actually honours robots.txt, so a rule here would genuinely take effect.
User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# User-initiated Perplexity fetches. [vendor states robots.txt may not apply to this agent]
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# User-initiated Mistral fetches. The other fetcher that honours robots.txt.
User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# --- Model training crawlers ---
# OpenAI foundation-model training.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Anthropic model training.
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Amazon. Its documentation states content may be used to train Amazon AI models, which is why it sits here rather than under retrieval.
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Meta. Documented as training foundation models or indexing content directly. [vendor does not state whether this agent obeys robots.txt]
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Mistral training datasets. Its docs state webmasters may disallow this agent.
User-agent: MistralAI-Training
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Common Crawl. Feeds many downstream training sets.
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# ByteDance. [token not confirmed against vendor docs] [observed behaviour diverges from documented policy; rules here are not enforceable]
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Diffbot structured-content extraction and Knowledge Graph.
User-agent: Diffbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# --- Everyone else ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

Sitemap: https://www.cosama.co/sitemap.xml