{
  "schema": "mxdata.site/v1",
  "generated_at": "2026-10-16T07:44:31Z",
  "snapshot_date": "2026-10-16",
  "snapshot_finished_at": "2026-10-16T07:32:12Z",
  "panel": {
    "name": "Tranco top 10,000",
    "list_id": "Y8JGG",
    "list_date": "2026-10-15",
    "size": 10000,
    "file_sha256": "1e49477006b7d63dced741c2ce51666463c31d9881248df5ec7f4fafbb1376cf"
  },
  "crawler": {
    "user_agent": "MXDATA-signals/1.0 (+AI usage-rights signal archive; daily, low-rate; robots.txt respected)",
    "duration_seconds": 1287
  },
  "integrity": {
    "manifest_sha256": "7e56985581858d76305bd18d7a9562b36950ef7a54c965acdbdc331615feb130",
    "prev_snapshot_date": null,
    "prev_manifest_sha256": null
  },
  "stats": {
    "sites_total": 10000,
    "sites_reachable": 7886,
    "robots_valid": 5767,
    "robots_name_ai_bot": 1450,
    "robots_full_block_any_ai_bot": 1044,
    "robots_full_block_major4": 878,
    "content_signals": 167,
    "tdmrep_any": 55,
    "ai_txt": 11,
    "rsl_any": 7,
    "noai_any": 22,
    "any_machine_readable_signal": 1534
  },
  "rates_pct": {
    "full_block_any_ai_of_robots_valid": 18.1,
    "full_block_major4_of_robots_valid": 15.2,
    "name_ai_bot_of_robots_valid": 25.1,
    "any_signal_of_reachable": 19.5
  },
  "major4_bots": [
    "CCBot",
    "ClaudeBot",
    "GPTBot",
    "Google-Extended"
  ],
  "bots": [
    {
      "name": "CCBot",
      "operator": "Common Crawl",
      "purpose": "training",
      "full_block": 723,
      "partial_block": 98,
      "named": 890
    },
    {
      "name": "GPTBot",
      "operator": "OpenAI",
      "purpose": "training",
      "full_block": 706,
      "partial_block": 203,
      "named": 1068
    },
    {
      "name": "Bytespider",
      "operator": "ByteDance",
      "purpose": "training",
      "full_block": 691,
      "partial_block": 61,
      "named": 781
    },
    {
      "name": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "training",
      "full_block": 634,
      "partial_block": 149,
      "named": 918
    },
    {
      "name": "Google-Extended",
      "operator": "Google",
      "purpose": "training",
      "full_block": 529,
      "partial_block": 149,
      "named": 823
    },
    {
      "name": "meta-externalagent",
      "operator": "Meta",
      "purpose": "training",
      "full_block": 507,
      "partial_block": 76,
      "named": 627
    },
    {
      "name": "Applebot-Extended",
      "operator": "Apple",
      "purpose": "training",
      "full_block": 498,
      "partial_block": 78,
      "named": 624
    },
    {
      "name": "cohere-ai",
      "operator": "Cohere",
      "purpose": "mixed",
      "full_block": 481,
      "partial_block": 49,
      "named": 554
    },
    {
      "name": "anthropic-ai",
      "operator": "Anthropic",
      "purpose": "legacy",
      "full_block": 480,
      "partial_block": 71,
      "named": 603
    },
    {
      "name": "Amazonbot",
      "operator": "Amazon",
      "purpose": "mixed",
      "full_block": 477,
      "partial_block": 83,
      "named": 610
    },
    {
      "name": "Diffbot",
      "operator": "Diffbot",
      "purpose": "mixed",
      "full_block": 471,
      "partial_block": 18,
      "named": 502
    },
    {
      "name": "omgili",
      "operator": "Webz.io",
      "purpose": "mixed",
      "full_block": 463,
      "partial_block": 13,
      "named": 479
    },
    {
      "name": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "search",
      "full_block": 454,
      "partial_block": 172,
      "named": 783
    },
    {
      "name": "Claude-Web",
      "operator": "Anthropic",
      "purpose": "legacy",
      "full_block": 388,
      "partial_block": 47,
      "named": 480
    },
    {
      "name": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "user-fetch",
      "full_block": 377,
      "partial_block": 197,
      "named": 727
    },
    {
      "name": "YouBot",
      "operator": "You.com",
      "purpose": "search",
      "full_block": 376,
      "partial_block": 37,
      "named": 435
    },
    {
      "name": "FacebookBot",
      "operator": "Meta",
      "purpose": "legacy",
      "full_block": 370,
      "partial_block": 42,
      "named": 443
    },
    {
      "name": "Timpibot",
      "operator": "Timpi",
      "purpose": "mixed",
      "full_block": 327,
      "partial_block": 8,
      "named": 337
    },
    {
      "name": "ImagesiftBot",
      "operator": "ImageSift",
      "purpose": "mixed",
      "full_block": 308,
      "partial_block": 10,
      "named": 321
    },
    {
      "name": "Webzio-Extended",
      "operator": "Webz.io",
      "purpose": "training",
      "full_block": 271,
      "partial_block": 8,
      "named": 280
    },
    {
      "name": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "search",
      "full_block": 262,
      "partial_block": 200,
      "named": 618
    },
    {
      "name": "AI2Bot",
      "operator": "Ai2",
      "purpose": "training",
      "full_block": 257,
      "partial_block": 20,
      "named": 280
    },
    {
      "name": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "purpose": "user-fetch",
      "full_block": 254,
      "partial_block": 47,
      "named": 318
    },
    {
      "name": "Meta-ExternalFetcher",
      "operator": "Meta",
      "purpose": "user-fetch",
      "full_block": 250,
      "partial_block": 28,
      "named": 294
    },
    {
      "name": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "user-fetch",
      "full_block": 245,
      "partial_block": 83,
      "named": 381
    },
    {
      "name": "Claude-User",
      "operator": "Anthropic",
      "purpose": "user-fetch",
      "full_block": 234,
      "partial_block": 87,
      "named": 384
    },
    {
      "name": "cohere-training-data-crawler",
      "operator": "Cohere",
      "purpose": "training",
      "full_block": 229,
      "partial_block": 16,
      "named": 248
    },
    {
      "name": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "search",
      "full_block": 226,
      "partial_block": 98,
      "named": 384
    },
    {
      "name": "Google-CloudVertexBot",
      "operator": "Google",
      "purpose": "mixed",
      "full_block": 213,
      "partial_block": 18,
      "named": 240
    },
    {
      "name": "MistralAI-User",
      "operator": "Mistral AI",
      "purpose": "user-fetch",
      "full_block": 196,
      "partial_block": 28,
      "named": 242
    },
    {
      "name": "PanguBot",
      "operator": "Huawei",
      "purpose": "training",
      "full_block": 195,
      "partial_block": 8,
      "named": 203
    },
    {
      "name": "Kangaroo Bot",
      "operator": "Kangaroo LLM",
      "purpose": "training",
      "full_block": 169,
      "partial_block": 5,
      "named": 175
    },
    {
      "name": "Ai2Bot-Dolma",
      "operator": "Ai2",
      "purpose": "training",
      "full_block": 163,
      "partial_block": 11,
      "named": 175
    },
    {
      "name": "Brightbot",
      "operator": "Bright Data",
      "purpose": "mixed",
      "full_block": 83,
      "partial_block": 3,
      "named": 86
    },
    {
      "name": "Gemini-Deep-Research",
      "operator": "Google",
      "purpose": "user-fetch",
      "full_block": 81,
      "partial_block": 21,
      "named": 109
    },
    {
      "name": "TikTokSpider",
      "operator": "ByteDance",
      "purpose": "mixed",
      "full_block": 77,
      "partial_block": 10,
      "named": 102
    }
  ],
  "content_signal_values": {
    "ai-train": {
      "yes": 58,
      "no": 101
    },
    "ai-input": {
      "yes": 114,
      "no": 18
    },
    "search": {
      "yes": 164,
      "no": 1
    }
  },
  "changes": {
    "baseline": true,
    "prev_snapshot_date": null,
    "signal": 0,
    "raw": 0,
    "availability": 0
  }
}