"""Summarize the API-discovery runs: 8 tasks x 2 runs per setup, Claude Opus 5.5 at medium effort.
A run works when its script ran and printed the data (grade.py), or, on Hacker News, when the answer says
the page loads its stories without an API request."""
import glob, json, pathlib, statistics as S

PRICE = {"input": 4.0, "cache_write": 5.0, "cache_read": 0.20, "output": 20.0}  # USD per million tokens
SETUPS = {"Playwright MCP": "agent-r?/playwright-mcp", "rea MCP": "agent-r?/rea-mcp", "rea MCP, capped": "agent-cap-r?/rea-mcp"}
print(f"{'':<17}{'works':>7}{'tokens in':>12}{'cost':>8}{'time':>7}{'failed calls':>14}{'largest result':>16}")
for name, pattern in SETUPS.items():
    runs = []
    for f in sorted(glob.glob(f"{pattern}/*.json")):
        d = json.load(open(f))
        g = json.loads((pathlib.Path("scripts") / pathlib.Path(f).parent / f"{d['task']}.result.json").read_text())
        final = (d.get("final") or "").lower()
        d["ungraded"] = "ungraded" in g
        d["works"] = (" no " in final or "doesn't" in final) if d["task"] == "hn-front" else \
            g.get("exit") == 0 and len(g.get("stdout", "").strip().splitlines()) >= 2
        runs.append(d)
    calls = [c for d in runs for c in d["tool_calls"]]
    cost = sum(sum(d["totals"][k] * p for k, p in PRICE.items()) for d in runs) / 1e6
    graded = [d for d in runs if not d["ungraded"]]
    print(f"{name:<17}{sum(d['works'] for d in graded):>3} / {len(graded)}{S.median(d['totals']['input_all'] for d in runs):>12,.0f}"
          f"{'$%.2f' % cost:>8}{S.median(d['secs'] for d in runs):>6.0f}s{sum(c['error'] for c in calls):>6} of {len(calls):<5}"
          f"{max(c['chars'] for c in calls):>15,}")
print("\ntokens in and time: median run. cost: all 16 runs, Opus 5.5 prices, prompt caching.")