"""MiniWeb X-Ray — standalone eval-results inspector (agentlab-xray style). Browses src/miniweb/evaluation/results/// directories and shows, per task: - instruction, expected vs agent answer, pass/fail, timing - the FILLED verifier (verify_report.json check tree: spec + pass/fail + reason) - the audit note (note.md, stored IN the result dir — moved out of data/ so Railway annotation pulls can never wipe it) - step-through screenshots + the actions taken at each step (history.json) - a human feedback box, saved to feedback.md in the result dir Usage: python src/miniweb/evaluation/xray.py [--port 8125] """ from helpers.paths import REPO_ROOT import argparse import glob import html import json import os import re from pathlib import Path from flask import Flask, abort, redirect, request, send_from_directory ROOT = REPO_ROOT RESULTS = Path(__file__).resolve().parent / "results" # run outputs, next to this package app = Flask(__name__) VERDICT_COLORS = {"VERIFIER_OK": "#16a34a", "VERIFIER_SUSPECT": "#d97706", "TASK_BROKEN": "#dc2626", "RECORDING_GAP": "#2563eb"} # ── helpers ────────────────────────────────────────────────────────────────── def runs(): return sorted((d.name for d in RESULTS.iterdir() if d.is_dir()), key=lambda n: (RESULTS / n).stat().st_mtime, reverse=True) def safe_dir(run, name): """Resolve a task dir inside a run, refusing path escapes.""" d = (RESULTS / run / name).resolve() if not str(d).startswith(str((RESULTS / run).resolve())) or not d.is_dir(): abort(404) return d def read_json(p): try: return json.loads(Path(p).read_text()) except (OSError, json.JSONDecodeError): return None def note_verdict(note): m = re.search(r"\*\*Verdict:\*\*\s*([A-Z_]+)", note or "") return m.group(1) if m else None def md_lite(text): """Tiny markdown renderer: headings, bold, code. Escapes everything else.""" out = html.escape(text or "") out = re.sub(r"^# (.*)$", r'\1', out, flags=re.M) out = re.sub(r"\*\*([^*]+)\*\*", r"\1", out) out = re.sub(r"`([^`]+)`", r"\1", out) return out def badge(text, color): return (f'{text}') def _cval(v): """Human form of one constraint value: open -> any; dict -> value + mode.""" if isinstance(v, dict): if v.get("open"): return 'any' val = html.escape(json.dumps(v.get("value"))) if "value" in v else html.escape(json.dumps(v)) mode = v.get("mode") return f'{val}' + (f' {html.escape(str(mode))}' if mode else "") return f'{html.escape(json.dumps(v))}' def render_constraints(ctype, spec): """Readable constraint table for a check's spec (what the gate REQUIRES).""" rows = [] if ctype == "request_made": m, u = spec.get("method"), spec.get("url") if m or u: rows.append(f'request{html.escape(str(m or "ANY"))} {html.escape(str(u or "(any url)"))}') if spec.get("status") not in (None, "", {}): rows.append(f'status{_cval(spec["status"])}') bf = spec.get("body_fields") if isinstance(bf, dict) and bf: for k, v in bf.items(): rows.append(f'body.{html.escape(k)}{_cval(v)}') elif ctype == "page_visited": rows.append(f'visit url{_cval(spec.get("url", {"open": True}))}') elif ctype == "action_included": rows.append(f'action{html.escape(str(spec.get("action") or "any"))}') for k in ("target", "value"): if k in spec: rows.append(f'{k}{_cval(spec[k])}') elif ctype in ("qa_answer", "answer_matches", "reasoning_contains"): if spec.get("expected"): rows.append(f'expected{html.escape(str(spec["expected"]))}') if not rows: return "" return ('' + "".join(rows) + "
gate constraints
") def render_check(node): """Recursive HTML for a verify_report check tree node.""" if not isinstance(node, dict): return "" if node.get("op"): # AND/OR group inner = "".join(render_check(c) for c in node.get("checks") or []) ok = node.get("passed") mark = "✓" if ok else "✗" col = "#16a34a" if ok else "#dc2626" adv = ' advisory' if node.get("advisory") else "" return (f'
{mark} ' f'{node["op"]}{adv}
{inner}
') ok = node.get("passed") mark = "✓" if ok else "✗" col = "#16a34a" if ok else "#dc2626" adv = ' advisory' if node.get("advisory") else "" label = html.escape(node.get("label") or node.get("type") or "check") reason = html.escape(node.get("reason") or "") spec = node.get("spec") or {} spec_pre = html.escape(json.dumps(spec, indent=1)) return (f'
{mark} ' f'{html.escape(node.get("type") or "")} {label}{adv}' f'{render_constraints(node.get("type"), spec)}' f'
{reason}
' f'
raw spec
{spec_pre}
') STYLE = """""" def page(body, title="MiniWeb X-Ray"): return f"{title}{STYLE}{body}" def sidebar(run, active=None, flt=""): items = [] for d in sorted((RESULTS / run).iterdir()): if not d.is_dir(): continue res = read_json(d / "result.json") or {} note = (d / "note.md").read_text() if (d / "note.md").exists() else "" verdict = note_verdict(note) fb = (d / "feedback.md").exists() passed = res.get("passed") pf = badge("PASS", "#16a34a") if passed else (badge("FAIL", "#dc2626") if passed is False else badge("—", "#94a3b8")) vb = badge(verdict.replace("VERIFIER_", ""), VERDICT_COLORS[verdict]) if verdict in VERDICT_COLORS else "" fbi = " 💬" if fb else "" tid = res.get("task_id") or d.name.split("__", 1)[-1] if flt and flt.lower() not in (tid + " " + (verdict or "") + (" fail" if passed is False else " pass")).lower(): continue cls = "row active" if d.name == active else "row" items.append(f'{pf} {vb}{fbi} {html.escape(tid)}') search = (f'
' f'
') return f'
{search}{"".join(items)}
' def topbar(run): opts = "".join(f'' for r in runs()) return (f'
MiniWeb X-Ray' f'
run:
' f'verifier + note + feedback inspector
') # ── routes ─────────────────────────────────────────────────────────────────── @app.route("/") def index(): rs = runs() if not rs: return page("
no runs under src/miniweb/evaluation/results/
") return redirect(f"/run/{rs[0]}") @app.route("/pickrun") def pickrun(): return redirect(f"/run/{request.args.get('run', '')}") @app.route("/run/") def run_view(run): if run not in runs(): abort(404) flt = request.args.get("f", "") res = [read_json(p) for p in glob.glob(str(RESULTS / run / "*/result.json"))] res = [r for r in res if r] p = sum(1 for r in res if r.get("passed")) body = (topbar(run) + '
' + sidebar(run, flt=flt) + f'

{run}

' f'
{len(res)} tasks' f'pass {p} / fail {len(res) - p}' f'rate {100 * p / max(1, len(res)):.1f}%
' f'

Pick a task on the left. ' f'💬 = has human feedback. Badges: verdict from the audit note.

') return page(body) @app.route("/task//") def task_view(run, name): d = safe_dir(run, name) res = read_json(d / "result.json") or {} rep = read_json(d / "verify_report.json") or {} note = (d / "note.md").read_text() if (d / "note.md").exists() else None feedback = (d / "feedback.md").read_text() if (d / "feedback.md").exists() else "" saved = request.args.get("saved") passed = res.get("passed") pf = badge("PASS", "#16a34a") if passed else (badge("FAIL", "#dc2626") if passed is False else badge("no result", "#94a3b8")) head = (f'

{html.escape(res.get("task_id") or name)} {pf}

' f'
agent: {html.escape(str(res.get("model") or "?"))}' f'steps: {res.get("steps", "?")}' f'time: {round(res.get("elapsed_s", 0))}s' f'site: {html.escape(str(res.get("site") or ""))}
' f'

Instruction: {html.escape(res.get("instruction") or "")}

' f'

Expected: {html.escape(str(res.get("expected_answer") or "(none)"))}

' f'
agent answer' f'
{html.escape(res.get("agent_answer") or "(none)")}
') # filled verifier macros_html = "" for m, tree in (rep.get("macros") or {}).items(): ok = (rep.get("by_macro") or {}).get(m) mb = badge("✓", "#16a34a") if ok else badge("✗", "#dc2626") macros_html += f'
{mb} {html.escape(m)}{render_check(tree)}
' verifier_card = f'

Filled verifier (verify_report.json)

{macros_html or "no report"}
' # audit note note_card = "" if note: v = note_verdict(note) vb = badge(v, VERDICT_COLORS.get(v, "#64748b")) if v else "" note_card = f'

Audit note {vb}

{md_lite(note)}
' # steps: screenshots + actions shots = sorted(glob.glob(str(d / "screenshots" / "step_*.png")), key=lambda p: int(re.search(r"(\d+)", Path(p).name).group(1))) hist = read_json(d / "history.json") or {} steps = hist.get("history") or [] cur = max(0, min(int(request.args.get("step", 1)) - 1, len(shots) - 1)) if shots else 0 steps_card = "" if shots: # per-step data inlined so Prev/Next (and arrow keys) swap in place — no page reload step_data = [] for i in range(len(shots)): s = steps[min(i + 1, len(steps) - 1)] if steps else {} mo = (s.get("model_output") or {}) step_data.append({"img": f"/shot/{run}/{name}/{Path(shots[i]).name}", "acts": json.dumps(mo.get("action") or [], indent=1)[:1500], "think": (mo.get("thinking") or "")[:4000], "eval": (mo.get("evaluation_previous_goal") or "")[:1000], "memory": (mo.get("memory") or "")[:1500], "goal": (mo.get("next_goal") or "")[:1000]}) steps_card = (f'

Steps ({len(shots)} screenshots)

' f'
' f'' f'' f'' f'(or ←/→ keys)
' f'' f'
' f'
actions at this step' f'
' f'
full thinking' f'
' f'
') # human feedback saved_tag = 'saved ✓' if saved else "" fb_card = (f'

Human feedback

' f'
' f'' f'
' f'{saved_tag}' f'
') body = (topbar(run) + '
' + sidebar(run, active=name) + f'
{head}{verifier_card}{note_card}{steps_card}{fb_card}
') return page(body, title=res.get("task_id") or name) @app.route("/feedback//", methods=["POST"]) def save_feedback(run, name): d = safe_dir(run, name) text = (request.form.get("text") or "").strip() f = d / "feedback.md" if text: f.write_text(text + "\n") elif f.exists(): f.unlink() # empty submit clears feedback return redirect(f"/task/{run}/{name}?saved=1") @app.route("/shot///") def shot(run, name, fname): d = safe_dir(run, name) return send_from_directory(d / "screenshots", fname) if __name__ == "__main__": ap = argparse.ArgumentParser() ap.add_argument("--port", type=int, default=8125) args = ap.parse_args() print(f"MiniWeb X-Ray → http://localhost:{args.port}") app.run(host="0.0.0.0", port=args.port, debug=False)