# DeepValues Research: open-source value-investing research (CC BY 4.0). All crawlers welcome,
# including AI search, assistant and training crawlers. Please keep request rates reasonable.
# Cite as: DeepValues Research, <title>, <url> and link the canonical URL.
# Machine-readable: /llms.txt, /llms-full.txt, /api/stances.json, /index.json, <report-url>index.md, /for-agents/

# Content Signals Policy (https://contentsignals.org): yes = content may be collected for that use.
# search = build a search index and show results; ai-input = feed content into AI models (RAG, grounding,
# AI answers); ai-train = train or fine-tune AI models.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Google Search and AI Overviews
User-agent: Googlebot
Allow: /

# Gemini apps and grounding (control token)
User-agent: Google-Extended
Allow: /

# Bing search and Microsoft Copilot
User-agent: Bingbot
Allow: /

# ChatGPT search index (decides whether a site appears in ChatGPT search)
User-agent: OAI-SearchBot
Allow: /

# ChatGPT user-initiated page fetches
User-agent: ChatGPT-User
Allow: /

# OpenAI model training
User-agent: GPTBot
Allow: /

# Claude search index
User-agent: Claude-SearchBot
Allow: /

# Claude user-initiated page fetches
User-agent: Claude-User
Allow: /

# Anthropic model training
User-agent: ClaudeBot
Allow: /

# Perplexity search index
User-agent: PerplexityBot
Allow: /

# Perplexity user-initiated page fetches
User-agent: Perplexity-User
Allow: /

# Apple Siri, Spotlight and Safari suggestions
User-agent: Applebot
Allow: /

# Apple Intelligence training (control token)
User-agent: Applebot-Extended
Allow: /

# DuckDuckGo AI answers
User-agent: DuckAssistBot
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /

# Meta AI training and indexing
User-agent: meta-externalagent
Allow: /

# Meta AI user-initiated page fetches
User-agent: meta-externalfetcher
Allow: /

# Mistral Le Chat user-initiated page fetches
User-agent: MistralAI-User
Allow: /

# Common Crawl (open web archive used by many models)
User-agent: CCBot
Allow: /

# ByteDance / Doubao (allowed: the site has Chinese-language reports; compliance is reportedly inconsistent)
User-agent: Bytespider
Allow: /

# You.com
User-agent: YouBot
Allow: /

Sitemap: https://research.deepvalues.ai/sitemap.xml