"""Summarize results.json from scan.py for the 544 domains that were real websites
in the 1,000-site test. Run with: python3 analyze.py"""
import json
from collections import Counter

R = json.load(open("results.json"))
sites = [r for r in R if not r["group"].startswith("excluded")]
parsed = [r for r in sites if "allowed" in r]
n = len(parsed)
print(f"{len(sites)} websites, {n} with a robots.txt we could read")
print("no robots.txt read:", Counter((r.get("robots_status") or r.get("robots_error")) for r in sites if "allowed" not in r).most_common(6))

print("\nHomepage disallowed, and named in its own group:")
agents = list(parsed[0]["allowed"])
for a in agents:
    blocked = sum(not r["allowed"][a] for r in parsed)
    named = sum(a in r["named"] for r in parsed)
    print(f"  {a:20} blocked {blocked:4} ({blocked/n:5.1%})   named {named:4} ({named/n:5.1%})")

all_blocked = sum(all(not v for v in r["allowed"].values()) for r in parsed)
print(f"\nBlocks every agent on the list: {all_blocked}")
ai_only = [r for r in parsed if r["allowed"]["Googlebot"] and not r["allowed"]["GPTBot"]]
print(f"Allows Googlebot but blocks GPTBot: {len(ai_only)} ({len(ai_only)/n:.1%})")
train_not_search = [r for r in parsed if not r["allowed"]["GPTBot"] and r["allowed"]["OAI-SearchBot"]]
print(f"Blocks GPTBot but allows OAI-SearchBot: {len(train_not_search)}")
user_ok = [r for r in parsed if not r["allowed"]["GPTBot"] and r["allowed"]["ChatGPT-User"]]
print(f"Blocks GPTBot but allows ChatGPT-User: {len(user_ok)}")
cs = [r for r in parsed if r["content_signal"]]
print(f"\nSites with Content-Signal lines: {len(cs)} ({len(cs)/n:.1%})")
for r in cs:
    print(f"  {r['domain']}: {sorted(set(r['content_signal']))}")
for key in ["llms_txt", "ai_txt", "tdmrep"]:
    k = sum(r.get(key) is True for r in sites)
    print(f"{key}: {k} of {len(sites)} websites ({k/len(sites):.1%})")