"""Check a site for RSL: License lines in robots.txt (with the User-agent group each one belongs to, per RFC 9309)
and the terms in the license file. Run with: uv run --with httpx python rsl_check.py theguardian.com bild.de"""
import sys
import xml.etree.ElementTree as ET
from urllib.parse import urljoin
import httpx

UA = "DoccraftResearchBot/1.0 (https://doccraft.dev; hello@doccraft.dev) httpx/0.28"
NS = "{https://rslstandard.org/rsl}"


def license_lines(text):
    """(license URL, user agents of its group or None for global). A group ends only at the next User-agent line."""
    out, agents, in_rules = [], None, False
    for raw in text.splitlines():
        line = raw.split("#", 1)[0].strip()
        if ":" not in line:
            continue
        key, val = (s.strip() for s in line.split(":", 1))
        if key.lower() == "user-agent":
            agents = [val] if agents is None or in_rules else agents + [val]
            in_rules = False
        elif key.lower() == "license":
            out.append((val, agents))
        elif key.lower() in ("allow", "disallow"):
            in_rules = True
    return out


def terms(body):
    root = ET.fromstring(body)
    for lic in root.iter(f"{NS}license"):
        for tag in ("permits", "prohibits"):
            for e in lic.findall(f"{NS}{tag}"):
                yield f"{tag} {e.get('type')}: {e.text.strip()}"
        for p in lic.findall(f"{NS}payment"):
            yield f"payment: {p.get('type')}"


with httpx.Client(headers={"User-Agent": UA}, follow_redirects=True, timeout=20) as client:
    for domain in sys.argv[1:]:
        robots = client.get(f"https://{domain}/robots.txt")
        print(domain)
        for url, group in license_lines(robots.text):
            scope = "all crawlers" if group is None else (f"User-agent: {group[0]}" if len(group) == 1 else f"a group of {len(group)} user agents")
            r = client.get(urljoin(f"https://{domain}/", url))
            print(f"  License: {url}  (for {scope})")
            if r.status_code != 200 or not r.text.lstrip().startswith("<rsl"):
                print(f"    license file: HTTP {r.status_code}, not readable as RSL")
                continue
            for t in terms(r.text):
                print(f"    {t}")