"""What each tool sent to the test site, with default settings. robots.txt disallowed the page every tool was asked for.
Reads logs/<tool>.jsonl from run_tool.sh. Requests to /browsermcp/ came from a browser extension on the test machine, not a tool."""
import json, pathlib, re

ORDER = ["curl", "python-requests", "httpx", "curl_cffi-default", "curl_cffi-impersonate-chrome", "scrapy-runspider",
         "scrapy-robotstxt-obey", "playwright-chromium", "playwright-chrome-channel", "puppeteer", "agent-browser",
         "chrome-devtools-mcp", "browser-use", "lightpanda", "lightpanda-obey-robots", "obscura", "obscura-obey-robots",
         "crawl4ai", "crawl4ai-check-robots", "scrapling-fetcher"]


def short(ua):
    if not ua:
        return "(no User-Agent header)"
    m = re.search(r"(HeadlessChrome|Chrome)/(\d+)", ua)
    os_ = "Windows" if "Windows" in ua else "Linux" if "Linux" in ua else "Mac" if "Mac OS" in ua else ""
    return f"{m.group(1)}/{m.group(2)}, {os_}" if m and ua.startswith("Mozilla") else ua


print(f"{'tool':<30}{'user agent':<44}{'robots.txt':<12}{'page'}")
for name in ORDER:
    rows = [json.loads(l) for l in pathlib.Path(f"logs/{name}.jsonl").read_text().splitlines()]
    rows = [r for r in rows if not r["path"].startswith("/browsermcp/")]
    paths = [r["path"] for r in rows]
    page = next((r for r in rows if r["path"].startswith("/private/")), None)
    first = page or rows[0]
    ua = {k.lower(): v for k, v in first["headers"].items()}.get("user-agent")
    print(f"{name:<30}{short(ua)[:42]:<44}{'fetched' if '/robots.txt' in paths else '-':<12}{'fetched' if page else 'skipped'}")