{
  "metadata": {
    "title": "When AI Comes to Your Website",
    "subtitle": "A Behavioral Analysis of AI Crawlers in the Wild",
    "analysis_date": "2026-06-08T13:00:11.373572+00:00",
    "total_visits": 16390735,
    "brands": 144,
    "unique_urls": 1838373,
    "date_range": "2025-06-11 to 2026-06-08",
    "exclusions": "Dominant e-commerce brand excluded to ensure generalizable patterns"
  },
  "findings": [
    {
      "id": 1,
      "title": "OpenAI Dominates",
      "stat": "20%",
      "detail": "GPTBot + OAI-SearchBot account for 20% of all AI crawler traffic",
      "validated": true,
      "visualization": {
        "type": "donut",
        "data": {
          "Bytespider": 49.0,
          "ClaudeBot": 10.6,
          "ChatGPT-User": 9.4,
          "Applebot": 9.2,
          "Amazonbot": 6.2,
          "OAI-SearchBot": 5.5,
          "GPTBot": 5.3
        }
      }
    },
    {
      "id": 2,
      "title": "Claude Prefers Homepages",
      "stat": "1x",
      "detail": "ClaudeBot visits homepages 1x more than GPTBot (controlled for same 78 brands)",
      "validated": true,
      "visualization": {
        "type": "comparison_bar",
        "data": {
          "ClaudeBot": 1.1,
          "GPTBot": 1.8
        },
        "unit": "% of visits to homepage"
      }
    },
    {
      "id": 3,
      "title": "Weekends Are Peak",
      "stat": "+-25%",
      "detail": "GPTBot is -25% more active on weekends than weekdays",
      "validated": true,
      "visualization": {
        "type": "grouped_bar",
        "data": {
          "GPTBot": {
            "weekday_avg": 2674,
            "weekend_avg": 2019,
            "ratio": 0.75
          },
          "ClaudeBot": {
            "weekday_avg": 2596,
            "weekend_avg": 10664,
            "ratio": 4.11
          },
          "OAI-SearchBot": {
            "weekday_avg": 3040,
            "weekend_avg": 1585,
            "ratio": 0.52
          },
          "Claude-SearchBot": {
            "weekday_avg": 83,
            "weekend_avg": 166,
            "ratio": 2.0
          }
        },
        "note": "Opposite of traditional web traffic patterns"
      }
    },
    {
      "id": 4,
      "title": "Blog Entry Points",
      "stat": "17%",
      "detail": "1 in 5 OAI-SearchBot crawl sessions start on blog pages",
      "validated": true,
      "visualization": {
        "type": "pie",
        "data": {
          "Blog": 17,
          "Product": 6,
          "Homepage": 8,
          "Other": 69
        }
      }
    },
    {
      "id": 5,
      "title": "One-and-Done",
      "stat": "1 visit",
      "detail": "Most pages visited exactly once (median = 1)",
      "validated": true,
      "visualization": {
        "type": "histogram",
        "data": {
          "1": 63.5,
          "10+": 2.1,
          "2": 13.9,
          "3-5": 15.1,
          "6-10": 5.3
        },
        "stats": {
          "median": 1,
          "p90": 4,
          "p99": 15
        }
      }
    },
    {
      "id": 6,
      "title": "The 3-Click Rule",
      "stat": "33%",
      "detail": "GPTBot visits Depth 3 pages (e.g. /blog/2024/post) 33% of the time",
      "validated": true,
      "note": "Depth 0 = homepage (/), Depth 1 = /about, Depth 2 = /blog/post, Depth 3 = /blog/2024/post",
      "visualization": {
        "type": "stacked_bar",
        "data": {
          "GPTBot": {
            "Depth 0": 1.8,
            "Depth 1": 9.6,
            "Depth 2": 46.5,
            "Depth 3": 33.2,
            "Depth 4": 7.2,
            "Depth 5+": 1.8
          },
          "ClaudeBot": {
            "Depth 0": 1.1,
            "Depth 1": 75.8,
            "Depth 2": 9.7,
            "Depth 3": 9.3,
            "Depth 4": 3.3,
            "Depth 5+": 1.0
          },
          "OAI-SearchBot": {
            "Depth 0": 0.8,
            "Depth 1": 6.2,
            "Depth 2": 79.9,
            "Depth 3": 7.8,
            "Depth 4": 3.3,
            "Depth 5+": 2.0
          },
          "Claude-SearchBot": {
            "Depth 0": 4.3,
            "Depth 1": 14.0,
            "Depth 2": 49.3,
            "Depth 3": 20.7,
            "Depth 4": 11.6,
            "Depth 5+": 0.2
          }
        }
      }
    },
    {
      "id": 7,
      "title": "SearchBot Reaches Everyone",
      "stat": "85%",
      "detail": "GPTBot visits 85% of all brands (widest reach)",
      "validated": true,
      "visualization": {
        "type": "coverage_bar",
        "data": {
          "GPTBot": 84.7,
          "OAI-SearchBot": 70.1,
          "ClaudeBot": 59.7,
          "Claude-SearchBot": 8.3
        }
      }
    },
    {
      "id": 8,
      "title": "Velocity Gap",
      "stat": "1x",
      "detail": "GPTBot crawls 1x faster (56 req/hr) than ClaudeBot (84 req/hr)",
      "validated": true,
      "visualization": {
        "type": "horizontal_bar",
        "data": {
          "GPTBot": 55.7,
          "ClaudeBot": 83.5,
          "OAI-SearchBot": 45.3,
          "Claude-SearchBot": 14.3
        },
        "unit": "requests per active hour"
      }
    }
  ],
  "new_insights": [
    {
      "id": 9,
      "title": "Bytespider: The Homepage Bot",
      "stat": "2%",
      "detail": "TikTok's Bytespider visits homepages 2% of the time - checking brand existence, not training on content",
      "visualization": {
        "type": "pie",
        "data": {
          "Deep page": 98.0,
          "Homepage": 1.9,
          "Robots.txt": 0.1,
          "Favicon": 0.0,
          "Sitemap": 0.0
        }
      }
    },
    {
      "id": 10,
      "title": "The Triple Crown",
      "stat": "46%",
      "detail": "Only 46% of brands (62/135) get visited by ALL three major AI crawlers",
      "visualization": {
        "type": "stacked_bar",
        "data": {
          "1 crawler": 25.9,
          "2 crawlers": 19.3,
          "3 crawlers": 45.9,
          "4 crawlers": 8.9
        }
      }
    },
    {
      "id": 11,
      "title": "The Evening Crawl",
      "stat": "1.6x",
      "detail": "OAI-SearchBot is 1.6x more active during US evening hours (4pm-11pm PST) than US morning hours",
      "visualization": {
        "type": "line_chart",
        "data": {
          "GPTBot": {
            "peak_hour_utc": 16,
            "peak_visits": 50231,
            "peak_pct": 5.8,
            "trough_hour_utc": 2,
            "hourly_data": {
              "0": 28714,
              "1": 30064,
              "2": 24870,
              "3": 28411,
              "4": 36415,
              "5": 35715,
              "6": 33191,
              "7": 35844,
              "8": 39684,
              "9": 41407,
              "10": 36645,
              "11": 29344,
              "12": 31051,
              "13": 41521,
              "14": 42508,
              "15": 46424,
              "16": 50231,
              "17": 43245,
              "18": 30016,
              "19": 33818,
              "20": 33217,
              "21": 33589,
              "22": 40393,
              "23": 42050
            }
          },
          "ClaudeBot": {
            "peak_hour_utc": 3,
            "peak_visits": 415927,
            "peak_pct": 24.0,
            "trough_hour_utc": 6,
            "hourly_data": {
              "0": 36513,
              "1": 41040,
              "2": 35204,
              "3": 415927,
              "4": 38214,
              "5": 38041,
              "6": 32735,
              "7": 39656,
              "8": 41110,
              "9": 48843,
              "10": 321325,
              "11": 41411,
              "12": 49411,
              "13": 41306,
              "14": 55495,
              "15": 56387,
              "16": 51404,
              "17": 64219,
              "18": 44344,
              "19": 41037,
              "20": 47573,
              "21": 75603,
              "22": 40402,
              "23": 39391
            }
          },
          "OAI-SearchBot": {
            "peak_hour_utc": 8,
            "peak_visits": 299176,
            "peak_pct": 33.2,
            "trough_hour_utc": 17,
            "hourly_data": {
              "0": 21595,
              "1": 20184,
              "2": 20064,
              "3": 24532,
              "4": 25166,
              "5": 23159,
              "6": 24456,
              "7": 28248,
              "8": 299176,
              "9": 23555,
              "10": 21196,
              "11": 21161,
              "12": 22649,
              "13": 16680,
              "14": 17070,
              "15": 17379,
              "16": 17616,
              "17": 13843,
              "18": 15060,
              "19": 151068,
              "20": 16230,
              "21": 22450,
              "22": 20601,
              "23": 18248
            }
          },
          "Claude-SearchBot": {
            "peak_hour_utc": 8,
            "peak_visits": 857,
            "peak_pct": 7.6,
            "trough_hour_utc": 2,
            "hourly_data": {
              "0": 612,
              "1": 333,
              "2": 201,
              "3": 362,
              "4": 645,
              "5": 732,
              "6": 323,
              "7": 579,
              "8": 857,
              "9": 432,
              "10": 465,
              "11": 694,
              "12": 473,
              "13": 361,
              "14": 418,
              "15": 547,
              "16": 274,
              "17": 223,
              "18": 373,
              "19": 437,
              "20": 543,
              "21": 535,
              "22": 384,
              "23": 401
            }
          }
        }
      }
    },
    {
      "id": 12,
      "title": "Deep Dive Sessions",
      "stat": "46 pages",
      "detail": "GPTBot averages 46 pages per crawl session vs ClaudeBot's 70",
      "visualization": {
        "type": "comparison_bar",
        "data": {
          "OAI-SearchBot": 36.2,
          "ClaudeBot": 69.7,
          "GPTBot": 46.5,
          "Claude-SearchBot": 13.4
        }
      }
    }
  ],
  "visualization_data": {
    "market_share": {
      "Bytespider": {
        "visits": 8030373,
        "share": 49.0,
        "brands": 82
      },
      "ClaudeBot": {
        "visits": 1736590,
        "share": 10.6,
        "brands": 86
      },
      "ChatGPT-User": {
        "visits": 1540508,
        "share": 9.4,
        "brands": 40
      },
      "Applebot": {
        "visits": 1504844,
        "share": 9.2,
        "brands": 28
      },
      "Amazonbot": {
        "visits": 1015896,
        "share": 6.2,
        "brands": 38
      },
      "OAI-SearchBot": {
        "visits": 901386,
        "share": 5.5,
        "brands": 101
      },
      "GPTBot": {
        "visits": 868367,
        "share": 5.3,
        "brands": 122
      },
      "Scrapy": {
        "visits": 339806,
        "share": 2.1,
        "brands": 12
      },
      "Meta-ExternalFetcher": {
        "visits": 197356,
        "share": 1.2,
        "brands": 50
      },
      "PerplexityBot": {
        "visits": 140159,
        "share": 0.9,
        "brands": 36
      },
      "GoogleOther": {
        "visits": 16606,
        "share": 0.1,
        "brands": 1
      },
      "Claude-User": {
        "visits": 11841,
        "share": 0.1,
        "brands": 20
      },
      "Meta-ExternalAgent": {
        "visits": 11760,
        "share": 0.1,
        "brands": 9
      },
      "Bingbot": {
        "visits": 11375,
        "share": 0.1,
        "brands": 9
      },
      "Claude-SearchBot": {
        "visits": 11204,
        "share": 0.1,
        "brands": 12
      },
      "CCBot": {
        "visits": 11193,
        "share": 0.1,
        "brands": 13
      },
      "ChatGPT": {
        "visits": 9388,
        "share": 0.1,
        "brands": 8
      },
      "GrokBot": {
        "visits": 8300,
        "share": 0.1,
        "brands": 12
      },
      "YouBot": {
        "visits": 7834,
        "share": 0.0,
        "brands": 12
      },
      "Perplexity-User": {
        "visits": 3492,
        "share": 0.0,
        "brands": 13
      },
      "bytespider": {
        "visits": 2748,
        "share": 0.0,
        "brands": 1
      },
      "DeepSeekBot": {
        "visits": 2051,
        "share": 0.0,
        "brands": 19
      },
      "Claude": {
        "visits": 1661,
        "share": 0.0,
        "brands": 8
      },
      "MistralAI-User": {
        "visits": 1431,
        "share": 0.0,
        "brands": 43
      },
      "Googlebot": {
        "visits": 911,
        "share": 0.0,
        "brands": 1
      },
      "Perplexity": {
        "visits": 752,
        "share": 0.0,
        "brands": 6
      },
      "Bing": {
        "visits": 682,
        "share": 0.0,
        "brands": 3
      },
      "Amazon": {
        "visits": 645,
        "share": 0.0,
        "brands": 3
      },
      "Meta AI": {
        "visits": 503,
        "share": 0.0,
        "brands": 3
      },
      "cohere-ai": {
        "visits": 225,
        "share": 0.0,
        "brands": 13
      },
      "Claude-Web": {
        "visits": 203,
        "share": 0.0,
        "brands": 7
      },
      "Diffbot": {
        "visits": 177,
        "share": 0.0,
        "brands": 10
      },
      "facebookexternalhit": {
        "visits": 171,
        "share": 0.0,
        "brands": 1
      },
      "Google-Agent": {
        "visits": 88,
        "share": 0.0,
        "brands": 5
      },
      "AI2Bot": {
        "visits": 82,
        "share": 0.0,
        "brands": 4
      },
      "anthropic-ai": {
        "visits": 82,
        "share": 0.0,
        "brands": 6
      },
      "Timpibot": {
        "visits": 37,
        "share": 0.0,
        "brands": 2
      },
      "MistralBot": {
        "visits": 21,
        "share": 0.0,
        "brands": 7
      },
      "Mistral": {
        "visits": 4,
        "share": 0.0,
        "brands": 1
      },
      "Omgili": {
        "visits": 3,
        "share": 0.0,
        "brands": 2
      },
      "Google-Extended": {
        "visits": 2,
        "share": 0.0,
        "brands": 2
      }
    },
    "homepage_preference": {
      "GPTBot": 1.759882102960634,
      "ClaudeBot": 1.082980678227529
    },
    "weekend_patterns": {
      "GPTBot": {
        "weekday_avg": 2674,
        "weekend_avg": 2019,
        "ratio": 0.75
      },
      "ClaudeBot": {
        "weekday_avg": 2596,
        "weekend_avg": 10664,
        "ratio": 4.11
      },
      "OAI-SearchBot": {
        "weekday_avg": 3040,
        "weekend_avg": 1585,
        "ratio": 0.52
      },
      "Claude-SearchBot": {
        "weekday_avg": 83,
        "weekend_avg": 166,
        "ratio": 2.0
      }
    },
    "revisit_histogram": {
      "1": 63.5,
      "10+": 2.1,
      "2": 13.9,
      "3-5": 15.1,
      "6-10": 5.3
    },
    "depth_distribution": {
      "GPTBot": {
        "Depth 0": 1.8,
        "Depth 1": 9.6,
        "Depth 2": 46.5,
        "Depth 3": 33.2,
        "Depth 4": 7.2,
        "Depth 5+": 1.8
      },
      "ClaudeBot": {
        "Depth 0": 1.1,
        "Depth 1": 75.8,
        "Depth 2": 9.7,
        "Depth 3": 9.3,
        "Depth 4": 3.3,
        "Depth 5+": 1.0
      },
      "OAI-SearchBot": {
        "Depth 0": 0.8,
        "Depth 1": 6.2,
        "Depth 2": 79.9,
        "Depth 3": 7.8,
        "Depth 4": 3.3,
        "Depth 5+": 2.0
      },
      "Claude-SearchBot": {
        "Depth 0": 4.3,
        "Depth 1": 14.0,
        "Depth 2": 49.3,
        "Depth 3": 20.7,
        "Depth 4": 11.6,
        "Depth 5+": 0.2
      }
    },
    "coverage": {
      "GPTBot": {
        "brands_visited": 122,
        "total_brands": 144,
        "coverage": 84.7,
        "avg_visits_per_brand": 7118
      },
      "OAI-SearchBot": {
        "brands_visited": 101,
        "total_brands": 144,
        "coverage": 70.1,
        "avg_visits_per_brand": 8925
      },
      "ClaudeBot": {
        "brands_visited": 86,
        "total_brands": 144,
        "coverage": 59.7,
        "avg_visits_per_brand": 20193
      },
      "Claude-SearchBot": {
        "brands_visited": 12,
        "total_brands": 144,
        "coverage": 8.3,
        "avg_visits_per_brand": 934
      }
    },
    "velocity": {
      "GPTBot": 55.7,
      "ClaudeBot": 83.5,
      "OAI-SearchBot": 45.3,
      "Claude-SearchBot": 14.3
    },
    "bytespider_behavior": {
      "Deep page": {
        "visits": 7866156,
        "pct": 98.0,
        "brands": 64
      },
      "Homepage": {
        "visits": 154519,
        "pct": 1.9,
        "brands": 72
      },
      "Robots.txt": {
        "visits": 8359,
        "pct": 0.1,
        "brands": 14
      },
      "Favicon": {
        "visits": 982,
        "pct": 0.0,
        "brands": 4
      },
      "Sitemap": {
        "visits": 380,
        "pct": 0.0,
        "brands": 7
      }
    },
    "multi_crawler_coverage": {
      "1 crawler": {
        "brands": 35,
        "pct": 25.9
      },
      "2 crawlers": {
        "brands": 26,
        "pct": 19.3
      },
      "3 crawlers": {
        "brands": 62,
        "pct": 45.9
      },
      "4 crawlers": {
        "brands": 12,
        "pct": 8.9
      }
    },
    "hour_of_day": {
      "GPTBot": {
        "peak_hour_utc": 16,
        "peak_visits": 50231,
        "peak_pct": 5.8,
        "trough_hour_utc": 2,
        "hourly_data": {
          "0": 28714,
          "1": 30064,
          "2": 24870,
          "3": 28411,
          "4": 36415,
          "5": 35715,
          "6": 33191,
          "7": 35844,
          "8": 39684,
          "9": 41407,
          "10": 36645,
          "11": 29344,
          "12": 31051,
          "13": 41521,
          "14": 42508,
          "15": 46424,
          "16": 50231,
          "17": 43245,
          "18": 30016,
          "19": 33818,
          "20": 33217,
          "21": 33589,
          "22": 40393,
          "23": 42050
        }
      },
      "ClaudeBot": {
        "peak_hour_utc": 3,
        "peak_visits": 415927,
        "peak_pct": 24.0,
        "trough_hour_utc": 6,
        "hourly_data": {
          "0": 36513,
          "1": 41040,
          "2": 35204,
          "3": 415927,
          "4": 38214,
          "5": 38041,
          "6": 32735,
          "7": 39656,
          "8": 41110,
          "9": 48843,
          "10": 321325,
          "11": 41411,
          "12": 49411,
          "13": 41306,
          "14": 55495,
          "15": 56387,
          "16": 51404,
          "17": 64219,
          "18": 44344,
          "19": 41037,
          "20": 47573,
          "21": 75603,
          "22": 40402,
          "23": 39391
        }
      },
      "OAI-SearchBot": {
        "peak_hour_utc": 8,
        "peak_visits": 299176,
        "peak_pct": 33.2,
        "trough_hour_utc": 17,
        "hourly_data": {
          "0": 21595,
          "1": 20184,
          "2": 20064,
          "3": 24532,
          "4": 25166,
          "5": 23159,
          "6": 24456,
          "7": 28248,
          "8": 299176,
          "9": 23555,
          "10": 21196,
          "11": 21161,
          "12": 22649,
          "13": 16680,
          "14": 17070,
          "15": 17379,
          "16": 17616,
          "17": 13843,
          "18": 15060,
          "19": 151068,
          "20": 16230,
          "21": 22450,
          "22": 20601,
          "23": 18248
        }
      },
      "Claude-SearchBot": {
        "peak_hour_utc": 8,
        "peak_visits": 857,
        "peak_pct": 7.6,
        "trough_hour_utc": 2,
        "hourly_data": {
          "0": 612,
          "1": 333,
          "2": 201,
          "3": 362,
          "4": 645,
          "5": 732,
          "6": 323,
          "7": 579,
          "8": 857,
          "9": 432,
          "10": 465,
          "11": 694,
          "12": 473,
          "13": 361,
          "14": 418,
          "15": 547,
          "16": 274,
          "17": 223,
          "18": 373,
          "19": 437,
          "20": 543,
          "21": 535,
          "22": 384,
          "23": 401
        }
      }
    },
    "session_data": {
      "OAI-SearchBot": {
        "total_sessions": 24920,
        "total_visits": 901386,
        "avg_pages_per_session": 36.2
      },
      "ClaudeBot": {
        "total_sessions": 24927,
        "total_visits": 1736591,
        "avg_pages_per_session": 69.7
      },
      "GPTBot": {
        "total_sessions": 18669,
        "total_visits": 868367,
        "avg_pages_per_session": 46.5
      },
      "Claude-SearchBot": {
        "total_sessions": 836,
        "total_visits": 11204,
        "avg_pages_per_session": 13.4
      }
    },
    "day_of_week": {
      "GPTBot": {
        "Sun": 12.5,
        "Mon": 11.6,
        "Tue": 15.3,
        "Wed": 15.1,
        "Thu": 15.8,
        "Fri": 19.2,
        "Sat": 10.5
      },
      "ClaudeBot": {
        "Sun": 48.0,
        "Mon": 7.1,
        "Tue": 8.1,
        "Wed": 5.4,
        "Thu": 6.9,
        "Fri": 10.4,
        "Sat": 14.0
      },
      "OAI-SearchBot": {
        "Sun": 7.3,
        "Mon": 7.5,
        "Tue": 20.8,
        "Wed": 37.2,
        "Thu": 9.3,
        "Fri": 8.8,
        "Sat": 9.1
      },
      "Claude-SearchBot": {
        "Sun": 23.2,
        "Mon": 6.8,
        "Tue": 3.9,
        "Wed": 18.9,
        "Thu": 23.4,
        "Fri": 2.5,
        "Sat": 21.2
      }
    }
  },
  "crawler_types": {
    "training": {
      "description": "Collect data to train future AI models. No direct traffic benefit.",
      "crawlers": [
        "GPTBot",
        "ClaudeBot",
        "Bytespider"
      ]
    },
    "search": {
      "description": "Index content for real-time AI search. Users find your content via AI.",
      "crawlers": [
        "OAI-SearchBot",
        "Claude-SearchBot"
      ]
    },
    "browsing": {
      "description": "User-initiated browsing. AI fetches pages in real-time when users ask questions.",
      "crawlers": [
        "Claude-Web",
        "Claude-User",
        "ChatGPT-User"
      ]
    },
    "other": {
      "description": "Link previews, feed fetching, or ambiguous purpose.",
      "crawlers": [
        "Meta-ExternalFetcher",
        "Amazonbot"
      ]
    }
  }
}