Spaces:
Running
Running
Download src/miniweb/evaluation/xray.py from sparklabuofu/MiniWeb-demo: direct link, hf CLI and curl.
- Browser
- Download file 20 kB
-
https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/src/miniweb/evaluation/xray.py
- Command line
-
hf download hf://spaces/sparklabuofu/MiniWeb-demo/src/miniweb/evaluation/xray.py
-
curl -L -o xray.py https://huggingface.co/spaces/sparklabuofu/MiniWeb-demo/resolve/main/src/miniweb/evaluation/xray.py
20 kB
| """MiniWeb X-Ray β standalone eval-results inspector (agentlab-xray style). | |
| Browses src/miniweb/evaluation/results/<run>/<agent__task>/ directories and shows, per task: | |
| - instruction, expected vs agent answer, pass/fail, timing | |
| - the FILLED verifier (verify_report.json check tree: spec + pass/fail + reason) | |
| - the audit note (note.md, stored IN the result dir β moved out of data/ so | |
| Railway annotation pulls can never wipe it) | |
| - step-through screenshots + the actions taken at each step (history.json) | |
| - a human feedback box, saved to feedback.md in the result dir | |
| Usage: python src/miniweb/evaluation/xray.py [--port 8125] | |
| """ | |
| from helpers.paths import REPO_ROOT | |
| import argparse | |
| import glob | |
| import html | |
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| from flask import Flask, abort, redirect, request, send_from_directory | |
| ROOT = REPO_ROOT | |
| RESULTS = Path(__file__).resolve().parent / "results" # run outputs, next to this package | |
| app = Flask(__name__) | |
| VERDICT_COLORS = {"VERIFIER_OK": "#16a34a", "VERIFIER_SUSPECT": "#d97706", | |
| "TASK_BROKEN": "#dc2626", "RECORDING_GAP": "#2563eb"} | |
| # ββ helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def runs(): | |
| return sorted((d.name for d in RESULTS.iterdir() if d.is_dir()), | |
| key=lambda n: (RESULTS / n).stat().st_mtime, reverse=True) | |
| def safe_dir(run, name): | |
| """Resolve a task dir inside a run, refusing path escapes.""" | |
| d = (RESULTS / run / name).resolve() | |
| if not str(d).startswith(str((RESULTS / run).resolve())) or not d.is_dir(): | |
| abort(404) | |
| return d | |
| def read_json(p): | |
| try: | |
| return json.loads(Path(p).read_text()) | |
| except (OSError, json.JSONDecodeError): | |
| return None | |
| def note_verdict(note): | |
| m = re.search(r"\*\*Verdict:\*\*\s*([A-Z_]+)", note or "") | |
| return m.group(1) if m else None | |
| def md_lite(text): | |
| """Tiny markdown renderer: headings, bold, code. Escapes everything else.""" | |
| out = html.escape(text or "") | |
| out = re.sub(r"^# (.*)$", r'<span class="h">\1</span>', out, flags=re.M) | |
| out = re.sub(r"\*\*([^*]+)\*\*", r"<b>\1</b>", out) | |
| out = re.sub(r"`([^`]+)`", r"<code>\1</code>", out) | |
| return out | |
| def badge(text, color): | |
| return (f'<span style="background:{color}1a;color:{color};border:1px solid {color}55;' | |
| f'padding:.05rem .45rem;border-radius:.3rem;font-size:.72rem;font-weight:600">{text}</span>') | |
| def _cval(v): | |
| """Human form of one constraint value: open -> any; dict -> value + mode.""" | |
| if isinstance(v, dict): | |
| if v.get("open"): | |
| return '<i style="color:#94a3b8">any</i>' | |
| val = html.escape(json.dumps(v.get("value"))) if "value" in v else html.escape(json.dumps(v)) | |
| mode = v.get("mode") | |
| return f'<code>{val}</code>' + (f' <span class="adv" style="background:#e0e7ff;color:#4338ca">{html.escape(str(mode))}</span>' if mode else "") | |
| return f'<code>{html.escape(json.dumps(v))}</code>' | |
| def render_constraints(ctype, spec): | |
| """Readable constraint table for a check's spec (what the gate REQUIRES).""" | |
| rows = [] | |
| if ctype == "request_made": | |
| m, u = spec.get("method"), spec.get("url") | |
| if m or u: | |
| rows.append(f'<tr><td>request</td><td><code>{html.escape(str(m or "ANY"))} {html.escape(str(u or "(any url)"))}</code></td></tr>') | |
| if spec.get("status") not in (None, "", {}): | |
| rows.append(f'<tr><td>status</td><td>{_cval(spec["status"])}</td></tr>') | |
| bf = spec.get("body_fields") | |
| if isinstance(bf, dict) and bf: | |
| for k, v in bf.items(): | |
| rows.append(f'<tr><td>body.{html.escape(k)}</td><td>{_cval(v)}</td></tr>') | |
| elif ctype == "page_visited": | |
| rows.append(f'<tr><td>visit url</td><td>{_cval(spec.get("url", {"open": True}))}</td></tr>') | |
| elif ctype == "action_included": | |
| rows.append(f'<tr><td>action</td><td><code>{html.escape(str(spec.get("action") or "any"))}</code></td></tr>') | |
| for k in ("target", "value"): | |
| if k in spec: | |
| rows.append(f'<tr><td>{k}</td><td>{_cval(spec[k])}</td></tr>') | |
| elif ctype in ("qa_answer", "answer_matches", "reasoning_contains"): | |
| if spec.get("expected"): | |
| rows.append(f'<tr><td>expected</td><td><code>{html.escape(str(spec["expected"]))}</code></td></tr>') | |
| if not rows: | |
| return "" | |
| return ('<table class="constraints"><tr><th colspan="2">gate constraints</th></tr>' | |
| + "".join(rows) + "</table>") | |
| def render_check(node): | |
| """Recursive HTML for a verify_report check tree node.""" | |
| if not isinstance(node, dict): | |
| return "" | |
| if node.get("op"): # AND/OR group | |
| inner = "".join(render_check(c) for c in node.get("checks") or []) | |
| ok = node.get("passed") | |
| mark = "β" if ok else "β" | |
| col = "#16a34a" if ok else "#dc2626" | |
| adv = ' <span class="adv">advisory</span>' if node.get("advisory") else "" | |
| return (f'<div class="grp"><div class="grph" style="color:{col}">{mark} ' | |
| f'{node["op"]}{adv}</div>{inner}</div>') | |
| ok = node.get("passed") | |
| mark = "β" if ok else "β" | |
| col = "#16a34a" if ok else "#dc2626" | |
| adv = ' <span class="adv">advisory</span>' if node.get("advisory") else "" | |
| label = html.escape(node.get("label") or node.get("type") or "check") | |
| reason = html.escape(node.get("reason") or "") | |
| spec = node.get("spec") or {} | |
| spec_pre = html.escape(json.dumps(spec, indent=1)) | |
| return (f'<div class="chk"><span style="color:{col};font-weight:700">{mark}</span> ' | |
| f'<b>{html.escape(node.get("type") or "")}</b> {label}{adv}' | |
| f'{render_constraints(node.get("type"), spec)}' | |
| f'<div class="reason">{reason}</div>' | |
| f'<details><summary>raw spec</summary><pre>{spec_pre}</pre></details></div>') | |
| STYLE = """<style> | |
| body{font-family:system-ui,sans-serif;margin:0;background:#f8fafc;color:#0f172a;font-size:14px} | |
| a{color:#2563eb;text-decoration:none} a:hover{text-decoration:underline} | |
| .top{background:#0f172a;color:#e2e8f0;padding:.5rem 1rem;display:flex;gap:1rem;align-items:center} | |
| .top select{font-size:.85rem;padding:.15rem} | |
| .wrap{display:flex;min-height:calc(100vh - 42px)} | |
| .side{width:330px;border-right:1px solid #e2e8f0;background:#fff;overflow-y:auto;max-height:calc(100vh - 42px);position:sticky;top:0} | |
| .side .row{display:block;padding:.35rem .6rem;border-bottom:1px solid #f1f5f9;font-size:.78rem;color:#0f172a} | |
| .side .row:hover{background:#f1f5f9} .side .row.active{background:#eef2ff} | |
| .main{flex:1;padding:1rem 1.4rem;max-width:1100px} | |
| .card{background:#fff;border:1px solid #e2e8f0;border-radius:.5rem;padding:.8rem 1rem;margin-bottom:1rem} | |
| .card h3{margin:.1rem 0 .6rem;font-size:.95rem} | |
| .grp{border-left:3px solid #e2e8f0;margin:.3rem 0 .3rem .4rem;padding-left:.6rem} | |
| .grph{font-weight:700;font-size:.78rem} | |
| .chk{margin:.35rem 0;font-size:.8rem} | |
| .chk .reason{color:#64748b;margin-left:1.15rem;font-size:.75rem} | |
| .chk details{margin-left:1.15rem} .chk summary{cursor:pointer;font-size:.7rem;color:#94a3b8} | |
| .chk pre{background:#f8fafc;border:1px solid #e2e8f0;padding:.4rem;font-size:.68rem;overflow-x:auto} | |
| .adv{background:#fef9c3;color:#a16207;font-size:.65rem;padding:0 .3rem;border-radius:.2rem} | |
| .constraints{border-collapse:collapse;margin:.25rem 0 .1rem 1.15rem;font-size:.72rem} | |
| .constraints th{text-align:left;color:#94a3b8;font-weight:600;font-size:.65rem;text-transform:uppercase;padding:.1rem .5rem .1rem 0} | |
| .constraints td{border-top:1px solid #f1f5f9;padding:.15rem .6rem .15rem 0;color:#475569} | |
| .constraints td:first-child{color:#64748b;font-weight:600;white-space:nowrap} | |
| .constraints code{background:#f1f5f9;padding:0 .3rem;border-radius:.2rem} | |
| .note{white-space:pre-wrap;word-break:break-word;font-size:.8rem;line-height:1.55} | |
| .note .h{font-weight:700;font-size:.9rem} .note code{background:#eef2ff;padding:0 .25rem;border-radius:.2rem} | |
| .mono{font-family:ui-monospace,monospace;font-size:.78rem;background:#f8fafc;border:1px solid #e2e8f0;padding:.5rem;border-radius:.3rem;white-space:pre-wrap;word-break:break-word} | |
| .shot{max-width:100%;border:1px solid #e2e8f0;border-radius:.4rem} | |
| .steps{display:flex;gap:.4rem;flex-wrap:wrap;margin-bottom:.5rem} | |
| .steps a{border:1px solid #cbd5e1;border-radius:.3rem;padding:.1rem .45rem;font-size:.75rem} | |
| .steps a.cur{background:#0f172a;color:#fff;border-color:#0f172a} | |
| textarea{width:100%;min-height:110px;font-family:inherit;font-size:.85rem;padding:.5rem;border:1px solid #cbd5e1;border-radius:.4rem;box-sizing:border-box} | |
| button{background:#0f172a;color:#fff;border:0;border-radius:.35rem;padding:.4rem .9rem;font-size:.8rem;cursor:pointer} | |
| .meta{display:flex;gap:1.2rem;flex-wrap:wrap;font-size:.78rem;color:#475569} | |
| </style>""" | |
| def page(body, title="MiniWeb X-Ray"): | |
| return f"<!doctype html><html><head><meta charset='utf-8'><title>{title}</title>{STYLE}</head><body>{body}</body></html>" | |
| def sidebar(run, active=None, flt=""): | |
| items = [] | |
| for d in sorted((RESULTS / run).iterdir()): | |
| if not d.is_dir(): | |
| continue | |
| res = read_json(d / "result.json") or {} | |
| note = (d / "note.md").read_text() if (d / "note.md").exists() else "" | |
| verdict = note_verdict(note) | |
| fb = (d / "feedback.md").exists() | |
| passed = res.get("passed") | |
| pf = badge("PASS", "#16a34a") if passed else (badge("FAIL", "#dc2626") if passed is False else badge("β", "#94a3b8")) | |
| vb = badge(verdict.replace("VERIFIER_", ""), VERDICT_COLORS[verdict]) if verdict in VERDICT_COLORS else "" | |
| fbi = " π¬" if fb else "" | |
| tid = res.get("task_id") or d.name.split("__", 1)[-1] | |
| if flt and flt.lower() not in (tid + " " + (verdict or "") + (" fail" if passed is False else " pass")).lower(): | |
| continue | |
| cls = "row active" if d.name == active else "row" | |
| items.append(f'<a class="{cls}" href="/task/{run}/{d.name}">{pf} {vb}{fbi} {html.escape(tid)}</a>') | |
| search = (f'<form method="get" action="/run/{run}" style="padding:.4rem .6rem">' | |
| f'<input name="f" value="{html.escape(flt)}" placeholder="filter: text / fail / SUSPECT" ' | |
| f'style="width:100%;box-sizing:border-box;font-size:.78rem;padding:.25rem"></form>') | |
| return f'<div class="side">{search}{"".join(items)}</div>' | |
| def topbar(run): | |
| opts = "".join(f'<option value="{r}" {"selected" if r == run else ""}>{r}</option>' for r in runs()) | |
| return (f'<div class="top"><b>MiniWeb X-Ray</b>' | |
| f'<form method="get" action="/pickrun">run: <select name="run" onchange="this.form.submit()">{opts}</select></form>' | |
| f'<span style="font-size:.75rem;color:#94a3b8">verifier + note + feedback inspector</span></div>') | |
| # ββ routes βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def index(): | |
| rs = runs() | |
| if not rs: | |
| return page("<div class='main'>no runs under src/miniweb/evaluation/results/</div>") | |
| return redirect(f"/run/{rs[0]}") | |
| def pickrun(): | |
| return redirect(f"/run/{request.args.get('run', '')}") | |
| def run_view(run): | |
| if run not in runs(): | |
| abort(404) | |
| flt = request.args.get("f", "") | |
| res = [read_json(p) for p in glob.glob(str(RESULTS / run / "*/result.json"))] | |
| res = [r for r in res if r] | |
| p = sum(1 for r in res if r.get("passed")) | |
| body = (topbar(run) + '<div class="wrap">' + sidebar(run, flt=flt) + | |
| f'<div class="main"><div class="card"><h3>{run}</h3>' | |
| f'<div class="meta"><span>{len(res)} tasks</span>' | |
| f'<span>pass {p} / fail {len(res) - p}</span>' | |
| f'<span>rate {100 * p / max(1, len(res)):.1f}%</span></div>' | |
| f'<p style="font-size:.8rem;color:#64748b">Pick a task on the left. ' | |
| f'π¬ = has human feedback. Badges: verdict from the audit note.</p></div></div></div>') | |
| return page(body) | |
| def task_view(run, name): | |
| d = safe_dir(run, name) | |
| res = read_json(d / "result.json") or {} | |
| rep = read_json(d / "verify_report.json") or {} | |
| note = (d / "note.md").read_text() if (d / "note.md").exists() else None | |
| feedback = (d / "feedback.md").read_text() if (d / "feedback.md").exists() else "" | |
| saved = request.args.get("saved") | |
| passed = res.get("passed") | |
| pf = badge("PASS", "#16a34a") if passed else (badge("FAIL", "#dc2626") if passed is False else badge("no result", "#94a3b8")) | |
| head = (f'<div class="card"><h3>{html.escape(res.get("task_id") or name)} {pf}</h3>' | |
| f'<div class="meta"><span>agent: {html.escape(str(res.get("model") or "?"))}</span>' | |
| f'<span>steps: {res.get("steps", "?")}</span>' | |
| f'<span>time: {round(res.get("elapsed_s", 0))}s</span>' | |
| f'<span>site: {html.escape(str(res.get("site") or ""))}</span></div>' | |
| f'<p style="margin:.5rem 0 0;font-size:.85rem"><b>Instruction:</b> {html.escape(res.get("instruction") or "")}</p>' | |
| f'<p style="margin:.3rem 0 0;font-size:.8rem"><b>Expected:</b> <code>{html.escape(str(res.get("expected_answer") or "(none)"))}</code></p>' | |
| f'<details style="margin-top:.3rem"><summary style="cursor:pointer;font-size:.78rem">agent answer</summary>' | |
| f'<div class="mono">{html.escape(res.get("agent_answer") or "(none)")}</div></details></div>') | |
| # filled verifier | |
| macros_html = "" | |
| for m, tree in (rep.get("macros") or {}).items(): | |
| ok = (rep.get("by_macro") or {}).get(m) | |
| mb = badge("β", "#16a34a") if ok else badge("β", "#dc2626") | |
| macros_html += f'<div style="margin:.5rem 0"><b style="font-size:.85rem">{mb} {html.escape(m)}</b>{render_check(tree)}</div>' | |
| verifier_card = f'<div class="card"><h3>Filled verifier (verify_report.json)</h3>{macros_html or "<i>no report</i>"}</div>' | |
| # audit note | |
| note_card = "" | |
| if note: | |
| v = note_verdict(note) | |
| vb = badge(v, VERDICT_COLORS.get(v, "#64748b")) if v else "" | |
| note_card = f'<div class="card"><h3>Audit note {vb}</h3><div class="note">{md_lite(note)}</div></div>' | |
| # steps: screenshots + actions | |
| shots = sorted(glob.glob(str(d / "screenshots" / "step_*.png")), | |
| key=lambda p: int(re.search(r"(\d+)", Path(p).name).group(1))) | |
| hist = read_json(d / "history.json") or {} | |
| steps = hist.get("history") or [] | |
| cur = max(0, min(int(request.args.get("step", 1)) - 1, len(shots) - 1)) if shots else 0 | |
| steps_card = "" | |
| if shots: | |
| # per-step data inlined so Prev/Next (and arrow keys) swap in place β no page reload | |
| step_data = [] | |
| for i in range(len(shots)): | |
| s = steps[min(i + 1, len(steps) - 1)] if steps else {} | |
| mo = (s.get("model_output") or {}) | |
| step_data.append({"img": f"/shot/{run}/{name}/{Path(shots[i]).name}", | |
| "acts": json.dumps(mo.get("action") or [], indent=1)[:1500], | |
| "think": (mo.get("thinking") or "")[:4000], | |
| "eval": (mo.get("evaluation_previous_goal") or "")[:1000], | |
| "memory": (mo.get("memory") or "")[:1500], | |
| "goal": (mo.get("next_goal") or "")[:1000]}) | |
| steps_card = (f'<div class="card" id="steps"><h3>Steps ({len(shots)} screenshots)</h3>' | |
| f'<div style="display:flex;gap:.5rem;align-items:center;margin-bottom:.5rem">' | |
| f'<button type="button" onclick="xstep(-1)">← Prev</button>' | |
| f'<span id="step-counter" style="font-size:.8rem;color:#475569;min-width:70px;text-align:center"></span>' | |
| f'<button type="button" onclick="xstep(1)">Next →</button>' | |
| f'<span style="font-size:.72rem;color:#94a3b8">(or β/β keys)</span></div>' | |
| f'<img class="shot" id="step-img" src="">' | |
| f'<div id="step-reason" style="margin-top:.5rem;font-size:.78rem;line-height:1.5"></div>' | |
| f'<details open style="margin-top:.4rem"><summary style="cursor:pointer;font-size:.78rem">actions at this step</summary>' | |
| f'<pre class="mono" id="step-acts"></pre></details>' | |
| f'<details style="margin-top:.3rem"><summary style="cursor:pointer;font-size:.78rem">full thinking</summary>' | |
| f'<div class="mono" id="step-think"></div></details>' | |
| f'<script>' | |
| f'var XSTEPS={json.dumps(step_data)};var XCUR={cur};' | |
| f'function esc(t){{var d=document.createElement("div");d.textContent=t||"";return d.innerHTML;}}' | |
| f'function xrender(){{var s=XSTEPS[XCUR];' | |
| f'document.getElementById("step-img").src=s.img;' | |
| f'document.getElementById("step-acts").textContent=s.acts;' | |
| f'document.getElementById("step-think").textContent=s.think||"(none)";' | |
| f'var evc=/success/i.test(s.eval)?"#16a34a":(/fail/i.test(s.eval)?"#dc2626":"#64748b");' | |
| f'document.getElementById("step-reason").innerHTML=' | |
| f'(s.eval?"<div><b style=\'color:"+evc+"\'>eval:</b> "+esc(s.eval)+"</div>":"")+' | |
| f'(s.goal?"<div><b style=\'color:#4338ca\'>next goal:</b> "+esc(s.goal)+"</div>":"")+' | |
| f'(s.memory?"<div style=\'color:#64748b\'><b>memory:</b> "+esc(s.memory)+"</div>":"");' | |
| f'document.getElementById("step-counter").textContent=(XCUR+1)+" / "+XSTEPS.length;}}' | |
| f'function xstep(d){{XCUR=Math.min(Math.max(XCUR+d,0),XSTEPS.length-1);xrender();}}' | |
| f'document.addEventListener("keydown",function(e){{' | |
| f'if(e.target.tagName==="TEXTAREA"||e.target.tagName==="INPUT")return;' | |
| f'if(e.key==="ArrowLeft")xstep(-1);else if(e.key==="ArrowRight")xstep(1);}});' | |
| f'xrender();' | |
| f'</script></div>') | |
| # human feedback | |
| saved_tag = '<span style="color:#16a34a;font-size:.78rem">saved β</span>' if saved else "" | |
| fb_card = (f'<div class="card"><h3>Human feedback</h3>' | |
| f'<form method="post" action="/feedback/{run}/{name}">' | |
| f'<textarea name="text" placeholder="your notes on this task/verifier/run...">{html.escape(feedback)}</textarea>' | |
| f'<div style="margin-top:.4rem;display:flex;gap:.6rem;align-items:center">' | |
| f'<button type="submit">Save feedback.md</button>{saved_tag}' | |
| f'</div></form></div>') | |
| body = (topbar(run) + '<div class="wrap">' + sidebar(run, active=name) + | |
| f'<div class="main">{head}{verifier_card}{note_card}{steps_card}{fb_card}</div></div>') | |
| return page(body, title=res.get("task_id") or name) | |
| def save_feedback(run, name): | |
| d = safe_dir(run, name) | |
| text = (request.form.get("text") or "").strip() | |
| f = d / "feedback.md" | |
| if text: | |
| f.write_text(text + "\n") | |
| elif f.exists(): | |
| f.unlink() # empty submit clears feedback | |
| return redirect(f"/task/{run}/{name}?saved=1") | |
| def shot(run, name, fname): | |
| d = safe_dir(run, name) | |
| return send_from_directory(d / "screenshots", fname) | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--port", type=int, default=8125) | |
| args = ap.parse_args() | |
| print(f"MiniWeb X-Ray β http://localhost:{args.port}") | |
| app.run(host="0.0.0.0", port=args.port, debug=False) | |