{
  "id": "crawlers/ccbot",
  "url": "https://www.proxies.sx/crawlers/ccbot",
  "license": "CC BY 4.0",
  "reviewed": "2026-09-06",
  "name": "CCBot",
  "operator": "Common Crawl",
  "operatorUrl": "https://commoncrawl.org",
  "docsUrl": "https://commoncrawl.org/ccbot",
  "purpose": "Archive crawler",
  "userAgentToken": "CCBot",
  "fullUserAgentExample": "CCBot/2.0 (https://commoncrawl.org/faq/)",
  "robotsToken": "CCBot",
  "robotsTokenIsPolicyOnly": false,
  "honorsRobotsTxt": "yes",
  "verificationMethod": "Common Crawl publishes its IP ranges at https://index.commoncrawl.org/ccbot.json and lists the same CIDRs inline in its FAQ, where the IPv6 range is labelled \"IPv6 (no reverse DNS)\"; the file's notes read: \"IP ranges used by CCBot. For verification of IPv4 addresses, FCrDNS is also recommended.\" The operator states: \"CCBot is now run on dedicated IP address ranges with reverse DNS (except over IPv6 where reverse DNS is not yet supported.)\" and gives the example that 18.97.14.84 resolves to 18-97-14-84.crawl.commoncrawl.org, which resolves back to 18.97.14.84. The documentation warns: \"we are aware of crawlers falsely identifying themselves as CCBot. We recommend verifying UserAgent strings to ensure authenticity.\"",
  "reverseDnsSuffix": ".crawl.commoncrawl.org",
  "ipRanges": {
    "sourceUrl": "https://index.commoncrawl.org/ccbot.json",
    "prefixes": 5,
    "ipv4": 4,
    "ipv6": 1,
    "sha256": "0a4c3b6707326d7f083139a979062604a57edfbb251bd29f23f5a86c5107ea30"
  },
  "sources": [
    "https://commoncrawl.org/ccbot",
    "https://commoncrawl.org/faq",
    "https://index.commoncrawl.org/ccbot.json"
  ]
}