{
  "checked": "2026-11-11",
  "bots": [
    {
      "name": "GPTBot",
      "vendor": "OpenAI",
      "kind": "training",
      "purpose": "Model training",
      "robots": "yes",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "name": "ClaudeBot",
      "vendor": "Anthropic",
      "kind": "training",
      "purpose": "Model training",
      "robots": "yes",
      "source": "https://support.claude.com/en/articles/8896518-robots-txt-controls"
    },
    {
      "name": "Meta-ExternalAgent",
      "vendor": "Meta",
      "kind": "training",
      "purpose": "Model training and indexing",
      "robots": "yes",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
    },
    {
      "name": "Amazonbot",
      "vendor": "Amazon",
      "kind": "training",
      "purpose": "Model training and Amazon products",
      "robots": "yes",
      "source": "https://developer.amazon.com/docs/amazonbot/amazonbot.html"
    },
    {
      "name": "MistralAI-Training",
      "vendor": "Mistral",
      "kind": "training",
      "purpose": "Model training",
      "robots": "yes",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "name": "CCBot",
      "vendor": "Common Crawl",
      "kind": "training",
      "purpose": "An open web corpus that AI labs train on",
      "robots": "yes",
      "source": "https://commoncrawl.org/ccbot"
    },
    {
      "name": "Bytespider",
      "vendor": "ByteDance",
      "kind": "training",
      "purpose": "Crawling for ByteDance",
      "robots": "not stated"
    },
    {
      "name": "AI2Bot",
      "vendor": "Allen Institute for AI",
      "kind": "training",
      "purpose": "Training open language models",
      "robots": "not stated"
    },
    {
      "name": "Diffbot",
      "vendor": "Diffbot",
      "kind": "training",
      "purpose": "A web knowledge graph and datasets",
      "robots": "not stated"
    },
    {
      "name": "Omgilibot",
      "vendor": "Webz.io",
      "kind": "training",
      "purpose": "Web datasets",
      "robots": "not stated"
    },
    {
      "name": "ChatGPT-User",
      "vendor": "OpenAI",
      "kind": "live-fetch",
      "purpose": "Loads a page when a ChatGPT user asks",
      "robots": "not always",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "name": "Claude-User",
      "vendor": "Anthropic",
      "kind": "live-fetch",
      "purpose": "Loads a page when a Claude user asks",
      "robots": "yes",
      "source": "https://support.claude.com/en/articles/8896518-robots-txt-controls"
    },
    {
      "name": "Perplexity-User",
      "vendor": "Perplexity",
      "kind": "live-fetch",
      "purpose": "Loads a page when a Perplexity user asks",
      "robots": "not always",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "name": "Google-Agent",
      "vendor": "Google",
      "kind": "live-fetch",
      "purpose": "Agents on Google infrastructure that browse when a user asks",
      "robots": "not always",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers"
    },
    {
      "name": "Meta-ExternalFetcher",
      "vendor": "Meta",
      "kind": "live-fetch",
      "purpose": "Loads a page when a user asks",
      "robots": "not always",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
    },
    {
      "name": "Amzn-User",
      "vendor": "Amazon",
      "kind": "live-fetch",
      "purpose": "Loads a page when a user asks",
      "robots": "not always",
      "source": "https://developer.amazon.com/docs/amazonbot/amazonbot.html"
    },
    {
      "name": "MistralAI-User",
      "vendor": "Mistral",
      "kind": "live-fetch",
      "purpose": "Loads a page when a user asks",
      "robots": "not stated",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "kind": "ai-search",
      "purpose": "The index for ChatGPT search",
      "robots": "yes",
      "cost": "Not shown in ChatGPT search answers",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "name": "Claude-SearchBot",
      "vendor": "Anthropic",
      "kind": "ai-search",
      "purpose": "The index for Claude's search",
      "robots": "yes",
      "cost": "Less visible in Claude's search results",
      "source": "https://support.claude.com/en/articles/8896518-robots-txt-controls"
    },
    {
      "name": "PerplexityBot",
      "vendor": "Perplexity",
      "kind": "ai-search",
      "purpose": "The index for Perplexity's results; not for training",
      "robots": "yes",
      "cost": "Out of Perplexity's results",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "name": "Meta-WebIndexer",
      "vendor": "Meta",
      "kind": "ai-search",
      "purpose": "Citations and links in Meta AI",
      "robots": "not stated",
      "cost": "Fewer citations in Meta AI",
      "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
    },
    {
      "name": "Amzn-SearchBot",
      "vendor": "Amazon",
      "kind": "ai-search",
      "purpose": "Alexa search; not for training",
      "robots": "yes",
      "cost": "Out of Alexa search",
      "source": "https://developer.amazon.com/docs/amazonbot/amazonbot.html"
    },
    {
      "name": "MistralAI-Index",
      "vendor": "Mistral",
      "kind": "ai-search",
      "purpose": "Mistral's search index; not for training",
      "robots": "yes",
      "cost": "Out of Mistral's search",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "name": "DuckAssistBot",
      "vendor": "DuckDuckGo",
      "kind": "ai-search",
      "purpose": "DuckAssist answers; not for training",
      "robots": "yes",
      "cost": "Out of DuckAssist; search unaffected",
      "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
    },
    {
      "name": "YouBot",
      "vendor": "You.com",
      "kind": "ai-search",
      "purpose": "You.com search",
      "robots": "yes",
      "source": "https://you.com/docs/youbot"
    },
    {
      "name": "Googlebot",
      "vendor": "Google",
      "kind": "search",
      "purpose": "Google Search, including AI Overviews and AI Mode",
      "robots": "yes",
      "cost": "Out of Google Search",
      "source": "https://developers.google.com/search/docs/appearance/ai-features"
    },
    {
      "name": "Bingbot",
      "vendor": "Microsoft",
      "kind": "search",
      "purpose": "Bing and Copilot",
      "robots": "yes",
      "cost": "Out of Bing and Copilot",
      "source": "https://blogs.bing.com/search/2026/10/your-content-your-choice-clearer-preferences-for-training-and-grounding/"
    },
    {
      "name": "Applebot",
      "vendor": "Apple",
      "kind": "search",
      "purpose": "Spotlight, Siri and Safari, and context for Apple's AI answers",
      "robots": "yes",
      "cost": "Out of Spotlight, Siri and Safari",
      "source": "https://support.apple.com/en-us/119829"
    },
    {
      "name": "Google-Extended",
      "vendor": "Google",
      "kind": "token",
      "purpose": "Gemini training, and grounding in Gemini Apps and Vertex AI; Google Search unaffected",
      "robots": "yes",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
    },
    {
      "name": "Applebot-Extended",
      "vendor": "Apple",
      "kind": "token",
      "purpose": "Apple's model training; search unaffected",
      "robots": "yes",
      "source": "https://support.apple.com/en-us/119829"
    },
    {
      "name": "__usage_ai-training",
      "vendor": "Shared (IETF draft)",
      "kind": "token",
      "purpose": "Training, for every crawler that honors it: Anthropic now, OpenAI announced, Google early 2027, Microsoft from 1 Feb 2027",
      "robots": "yes",
      "source": "https://datatracker.ietf.org/doc/draft-kelley-aipref-training-preference-signal/"
    }
  ]
}
