"""RSL adoption on real sites: License lines in robots.txt, RSL links on the homepage, and the license files they point to.
Same 1,000 Tranco domains and the same research user agent as the October 12, 2026 robots.txt scan, plus the
supporters named on rslcollective.org. Run with: uv run --with httpx python scan.py"""
import asyncio, json, re
import xml.etree.ElementTree as ET
from urllib.parse import urljoin
import httpx

UA = "DoccraftResearchBot/1.0 (https://doccraft.dev; hello@doccraft.dev) httpx/0.28"
NS = "https://rslstandard.org/rsl"
SUPPORTERS = {  # organization on rslcollective.org -> the main domains checked
    "Reddit": ["reddit.com"], "People Inc.": ["people.com", "allrecipes.com", "investopedia.com"],
    "Yahoo": ["yahoo.com"], "Internet Brands": ["webmd.com", "nolo.com"],
    "Ziff Davis": ["pcmag.com", "mashable.com", "ign.com", "cnet.com", "zdnet.com"],
    "Vox Media": ["vox.com", "theverge.com", "nymag.com"], "wikiHow": ["wikihow.com"],
    "Stack Overflow": ["stackoverflow.com"], "O'Reilly Media": ["oreilly.com"], "Medium": ["medium.com"],
    "Quizlet": ["quizlet.com"], "Fast Company": ["fastcompany.com"], "Inc.": ["inc.com"], "Note Inc.": ["note.com"],
    "Slate": ["slate.com"], "BuzzFeed": ["buzzfeed.com"], "The Daily Beast": ["thedailybeast.com"],
    "MIT Press": ["mitpress.mit.edu"], "Ranker": ["ranker.com"], "Evolve Media": ["evolvemedia.com"],
    "Raptive": ["raptive.com"], "VerticalScope": ["verticalscope.com"], "Man of Many": ["manofmany.com"],
    "USA TODAY Co.": ["usatoday.com"],
}


def license_lines(text):
    """License directives, with the User-agent group they sit in (None = global)."""
    out, agents, in_rules = [], None, False
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if ":" not in line:
            continue
        key, val = (s.strip() for s in line.split(":", 1))
        k = key.lower()
        if k == "user-agent":
            agents = [val] if (agents is None or in_rules) else agents + [val]
            in_rules = False
        elif k == "license":
            out.append({"url": val, "group": agents})
        elif k in ("allow", "disallow", "crawl-delay", "content-signal"):
            in_rules = True
    return out


def parse_rsl(body):
    """Terms from an RSL document: for each <content>, its url, permits, prohibits and payment types."""
    root = ET.fromstring(body)
    if root.tag != f"{{{NS}}}rsl":
        return None
    q = lambda e, n: e.findall(f"{{{NS}}}{n}")
    terms = []
    for c in q(root, "content"):
        for lic in q(c, "license"):
            terms.append({
                "content_url": c.get("url"),
                "permits": {p.get("type"): (p.text or "").split() for p in q(lic, "permits")},
                "prohibits": {p.get("type"): (p.text or "").split() for p in q(lic, "prohibits")},
                "payment": [{"type": p.get("type"), "custom": [x.text for x in q(p, "custom")],
                             "standard": [x.text for x in q(p, "standard")],
                             "amount": [(x.get("currency"), x.text) for x in q(p, "amount")]} for p in q(lic, "payment")],
            })
    return terms


async def fetch(client, url):
    try:
        r = await client.get(url, timeout=20)
        return {"status": r.status_code, "final": str(r.url), "ctype": r.headers.get("content-type", ""),
                "link": r.headers.get("link", ""), "text": r.text}
    except Exception as e:
        return {"error": type(e).__name__}


async def check_license(client, url):
    r = await fetch(client, url)
    out = {k: r.get(k) for k in ("status", "final", "ctype", "error")}
    body = r.get("text") or ""
    try:
        out["terms"] = parse_rsl(body) if body.lstrip().startswith("<") else None
    except ET.ParseError:
        out["terms"] = None
    out["is_rsl"] = out["terms"] is not None
    out["looks_blocked"] = (r.get("status") in (403, 429, 503)) or bool(re.search(r"Attention Required|you have been blocked|Just a moment|captcha", body[:3000], re.I))
    return out


async def check(client, sem, domain, source):
    async with sem:
        row = {"domain": domain, "source": source}
        rob = await fetch(client, f"https://{domain}/robots.txt")
        row["robots_status"] = rob.get("status")
        row["license_lines"] = license_lines(rob.get("text", "")) if rob.get("status") == 200 and "html" not in rob.get("ctype", "") else []
        home = await fetch(client, f"https://{domain}/")
        row["home_status"] = home.get("status")
        row["home_html"] = "html" in (home.get("ctype") or "")
        html = home.get("text") or ""
        links = re.findall(r'<link[^>]+rel=["\']?license["\']?[^>]*>', html, re.I)
        row["html_license"] = [urljoin(home.get("final") or f"https://{domain}/", m.group(1)) for l in links if "rsl" in l.lower() and (m := re.search(r'href=["\']([^"\']+)', l))]
        row["header_license"] = [h for h in re.findall(r'<([^>]+)>\s*;[^,]*rel="?license"?[^,]*', home.get("link", "")) if "rsl" in home.get("link", "").lower()]
        urls = {urljoin(f"https://{domain}/", x["url"]) for x in row["license_lines"]} | set(row["html_license"]) | set(row["header_license"])
        row["licenses"] = {u: await check_license(client, u) for u in sorted(urls)}
        return row


async def main():
    tranco = [r["domain"] for r in json.load(open("ai-crawler-rules-2026-results.json"))]
    supporters = [(d, f"supporter: {org}") for org, ds in SUPPORTERS.items() for d in ds]
    jobs = [(d, "tranco") for d in tranco] + [(d, s) for d, s in supporters if d not in tranco]
    sem = asyncio.Semaphore(24)
    async with httpx.AsyncClient(headers={"User-Agent": UA}, follow_redirects=True, http2=False) as client:
        rows = await asyncio.gather(*(check(client, sem, d, s) for d, s in jobs))
    json.dump(rows, open("results.json", "w"), indent=1)
    found = [r for r in rows if r["licenses"]]
    print(f"domains checked: {len(rows)}   with an RSL signal: {len(found)}")
    for r in found:
        print(r["domain"], r["source"], {u: (v["status"], v["is_rsl"], v["looks_blocked"]) for u, v in r["licenses"].items()})

asyncio.run(main())