"""Summarize the runs: 10 one-fact tasks through Chrome DevTools MCP 1.12.0, with and without --no-javascript-evaluation,
2 runs each, Claude Opus 5.5. Each answer is checked against what the page showed on October 29, 2026."""
import json, re, statistics as S

EXPECT = {
    "allbirds-price": r"\$50", "trueclassic-28L": r"sold out", "pydocs-taskgroup": r"3\.11",
    "mdn-abort": r"abort\(\)", "stripe-amount": r"required.*smallest|smallest.*required",
    "gh-playwright-release": r"1\.64\.1.*(14 Oct|Oct\w* 14)", "pypi-httpx": r"0\.28\.1.*Dec\w* 6, 2024",
    "hf-top-downloads": r"all-MiniLM-L6-v2", "wiki-follow-link": r"1994", "github-search-repo": r"44[.,][78]",
}
WRONG = {"gh-playwright-release": r"Oct\w* 13|13 Oct"}  # an answer that gives October 13, even with a hedge, is wrong
PRICE = {"input": 4.0, "cache_write": 5.0, "cache_read": 0.20, "output": 20.0}
print(f"{'':<27}{'correct':>9}{'input tokens':>14}{'tool calls':>12}{'time':>8}{'cost':>8}")
print(f"{'':<27}{'':>9}{'(all runs)':>14}{'(all runs)':>12}{'(median)':>8}{'(all)':>8}")
for name, setup in (("JavaScript on (default)", "devtools-mcp"), ("--no-javascript-evaluation", "devtools-mcp-nojs")):
    runs = [d for d in json.load(open("agent-runs.json")) if d["setup"] == setup]
    ok = sum(bool(re.search(EXPECT[d["task"]], d.get("final") or "", re.I | re.S))
             and not re.search(WRONG.get(d["task"], r"(?!)"), d.get("final") or "", re.I) for d in runs)
    cost = sum(sum(d["totals"][k] * p for k, p in PRICE.items()) for d in runs) / 1e6
    print(f"{name:<27}{ok:>4} / {len(runs)}{sum(d['totals']['input_all'] for d in runs):>14,}"
          f"{sum(len(d['tool_calls']) for d in runs):>12}{S.median(d['secs'] for d in runs):>7.0f}s{'$%.2f' % cost:>8}")