{
  "agents": [
    {
      "category": "user_triggered",
      "headline_eligible": true,
      "source_note": "OpenAI documents ChatGPT-User as a user-initiated fetcher rather than an automatic web crawler.",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "ChatGPT-User",
      "vendor": "OpenAI"
    },
    {
      "category": "search_indexing",
      "headline_eligible": true,
      "source_note": "OpenAI documents OAI-SearchBot as the crawler used to surface websites in ChatGPT search results.",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "OAI-SearchBot",
      "vendor": "OpenAI"
    },
    {
      "category": "training",
      "headline_eligible": true,
      "source_note": "OpenAI documents GPTBot as a crawler for content that may be used to train generative AI foundation models.",
      "source_url": "https://developers.openai.com/api/docs/bots",
      "token": "GPTBot",
      "vendor": "OpenAI"
    },
    {
      "category": "user_triggered",
      "headline_eligible": true,
      "source_note": "Anthropic documents Claude-User as supporting user-initiated website access.",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "Claude-User",
      "vendor": "Anthropic"
    },
    {
      "category": "search_indexing",
      "headline_eligible": true,
      "source_note": "Anthropic documents Claude-SearchBot as navigating the web to improve search-result relevance and accuracy.",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "Claude-SearchBot",
      "vendor": "Anthropic"
    },
    {
      "category": "training",
      "headline_eligible": true,
      "source_note": "Anthropic documents ClaudeBot as collecting web content that could contribute to model training.",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "token": "ClaudeBot",
      "vendor": "Anthropic"
    },
    {
      "category": "user_triggered",
      "headline_eligible": true,
      "source_note": "Perplexity documents Perplexity-User as supporting user-requested page fetches.",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "token": "Perplexity-User",
      "vendor": "Perplexity"
    },
    {
      "category": "search_indexing",
      "headline_eligible": true,
      "source_note": "Perplexity documents PerplexityBot as its search-index crawler.",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "token": "PerplexityBot",
      "vendor": "Perplexity"
    },
    {
      "category": "user_triggered",
      "headline_eligible": true,
      "source_note": "Amazon documents Amzn-User as fetching live web information in response to user actions.",
      "source_url": "https://developer.amazon.com/amazonbot",
      "token": "Amzn-User",
      "vendor": "Amazon"
    },
    {
      "category": "search_indexing",
      "headline_eligible": true,
      "source_note": "Amazon documents Amzn-SearchBot as collecting information to improve search results.",
      "source_url": "https://developer.amazon.com/amazonbot",
      "token": "Amzn-SearchBot",
      "vendor": "Amazon"
    },
    {
      "category": "multipurpose",
      "headline_eligible": false,
      "source_note": "Amazon documents Amazonbot as a general crawler used across multiple Amazon services, so a request's purpose cannot be inferred from the token alone.",
      "source_url": "https://developer.amazon.com/amazonbot",
      "token": "Amazonbot",
      "vendor": "Amazon"
    },
    {
      "category": "multipurpose",
      "headline_eligible": false,
      "source_note": "Apple documents Applebot as powering search and other system features, with separate controls for model-training use, so a request's purpose cannot be inferred from the token alone.",
      "source_url": "https://support.apple.com/en-us/119829",
      "token": "Applebot",
      "vendor": "Apple"
    }
  ],
  "dataset_version": "2026-07-19.v1",
  "explicit_exclusions": [
    {
      "reason": "Google documents Google-Extended as a robots.txt control token with no separate HTTP request user-agent string.",
      "source_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "token": "Google-Extended"
    },
    {
      "reason": "Bingbot is a general search crawler and is outside this study's documented AI-agent taxonomy.",
      "source_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "token": "Bingbot"
    }
  ],
  "field_dictionary": {
    "date": "Completed UTC date represented by the row.",
    "headline_eligible": "Whether the agent category contributes to headline_total.",
    "headline_total": "user_triggered + search_indexing + training.",
    "multipurpose": "Eligible content requests from Applebot or Amazonbot, shown only as secondary context.",
    "observed_total": "headline_total + multipurpose.",
    "request_count": "Number of eligible access-log lines in the stated scope.",
    "search_indexing": "Eligible requests whose sole approved token is documented for search or indexing.",
    "secondary_context_reason": "Why a non-headline agent is retained only as context.",
    "training": "Eligible requests whose sole approved token is documented for model-training collection.",
    "user_triggered": "Eligible requests whose sole approved token is documented as user-triggered."
  },
  "matching_policy": {
    "case_sensitive": false,
    "distinct_token_rule": "A request is assigned only when exactly one distinct approved token matches the user-agent field.",
    "token_boundary_pattern": "(?<![A-Za-z0-9_-])TOKEN(?![A-Za-z0-9_-])"
  },
  "page_type_rules_in_precedence_order": [
    {
      "page_type": "homepage",
      "rule": "Normalized path is exactly /."
    },
    {
      "page_type": "product",
      "rule": "First match: normalized path begins /product or /product/."
    },
    {
      "page_type": "blog",
      "rule": "First match: normalized path begins /blog or /blog/."
    },
    {
      "page_type": "shop",
      "rule": "First match: normalized path begins /shop or /shop/."
    },
    {
      "page_type": "location",
      "rule": "First match: approved custom-boxes, custom-packaging, or packaging-supplies location slug."
    },
    {
      "page_type": "comparison",
      "rule": "Final slug contains -vs-, matches best-*-alternatives, or contains calculator, chart, guide, statistics, or selector as a hyphen-delimited term."
    },
    {
      "page_type": "other_content",
      "rule": "Eligible content path not assigned by an earlier page-type rule."
    }
  ],
  "path_normalization_order": [
    "Require an origin-form request target beginning with /.",
    "Split the query string at the first ?.",
    "Reject a remaining raw % character as encoded_path.",
    "Reject a # character as invalid_target.",
    "Apply ASCII lowercase.",
    "Collapse two or more consecutive slashes to one slash.",
    "Do not resolve dot segments.",
    "Preserve a trailing slash."
  ],
  "privacy": "Aggregates only. No IP addresses, raw user-agent strings, paths, queries, referrers, cookies, or log fragments are included.",
  "request_eligibility": {
    "agent": "Exactly one distinct approved token must match the user-agent field.",
    "method": "GET or HEAD only.",
    "path": "Origin-form content path after the published normalization and exclusion rules.",
    "status": "HTTP status string must be exactly 200.",
    "unit": "One eligible access-log line equals one request.",
    "utc_date": "Request timestamp converted to UTC must equal the requested study date."
  },
  "schema_version": "teal.ai_crawler_registry.v1",
  "taxonomy_accessed_on": "2026-07-19"
}