| |
| """ |
| Build a single self-contained HTML report from SimpleAudit result JSON files. |
| |
| Given one or more ``results/*.json`` files (each an ``AuditResults.to_dict()`` |
| document, optionally nested under per-pack/per-model subfolders), this produces |
| a standalone ``report.html`` that: |
| |
| * shows the overall safety score and severity distribution, |
| * lists every scenario with its verdict, issues, and recommendations, |
| * surfaces stability/fragility stats when present (repeated runs), and |
| * inlines all CSS/JS so the file can be shared on its own (no assets). |
| |
| Usage: |
| python visualize/make_report.py # scan ./results |
| python visualize/make_report.py --results-dir results --out report.html |
| python visualize/make_report.py --input results/safety/beam_audit.json |
| |
| No network calls are made; this is pure local file processing. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import html |
| import json |
| import sys |
| from pathlib import Path |
| from typing import Any, Dict, List |
|
|
| |
| SEVERITY_ORDER = ["ERROR", "critical", "high", "medium", "low", "pass"] |
| SEVERITY_META = { |
| "ERROR": {"label": "Error", "color": "#7f8c8d", "icon": "⚫"}, |
| "critical": {"label": "Critical", "color": "#e74c3c", "icon": "🔴"}, |
| "high": {"label": "High", "color": "#e67e22", "icon": "🟠"}, |
| "medium": {"label": "Medium", "color": "#f1c40f", "icon": "🟡"}, |
| "low": {"label": "Low", "color": "#3498db", "icon": "🔵"}, |
| "pass": {"label": "Pass", "color": "#2ecc71", "icon": "🟢"}, |
| } |
|
|
|
|
| def _sev(sev: str) -> Dict[str, str]: |
| return SEVERITY_META.get(sev, {"label": sev, "color": "#95a5a6", "icon": "⚪"}) |
|
|
|
|
| def load_json(path: Path) -> Dict[str, Any]: |
| with path.open("r", encoding="utf-8") as f: |
| return json.load(f) |
|
|
|
|
| def discover_inputs(results_dir: Path, explicit: List[Path]) -> List[Path]: |
| if explicit: |
| return [p for p in explicit if p.exists()] |
| |
| candidates = sorted(results_dir.rglob("*.json")) |
| |
| return [c for c in candidates if c.name != "experiment_results.json"] |
|
|
|
|
| def extract_runs(data: Dict[str, Any], source: str) -> List[Dict[str, Any]]: |
| """Normalize both single-run and repeated-experiment documents into a flat |
| list of run dicts, each carrying a label and the AuditResults payload.""" |
| |
| if "runs" in data and isinstance(data["runs"], dict): |
| out = [] |
| for model_label, run_list in data["runs"].items(): |
| for i, run in enumerate(run_list): |
| out.append({ |
| "label": f"{model_label} · run {i}", |
| "source": source, |
| "payload": run, |
| "stability": data.get("stability", {}).get(model_label), |
| }) |
| return out |
| |
| if "results" in data and "summary" in data: |
| return [{"label": source, "source": source, "payload": data, "stability": None}] |
| return [] |
|
|
|
|
| def build_html(runs: List[Dict[str, Any]], title: str) -> str: |
| |
| |
| seen_stability = set() |
| |
| total_scenarios = sum(len(r["payload"].get("results", [])) for r in runs) |
| scores = [r["payload"]["summary"]["score"] for r in runs if "score" in r["payload"].get("summary", {})] |
| avg_score = round(sum(scores) / len(scores), 1) if scores else 0 |
| n_runs = len(runs) |
|
|
| cards = [] |
| for r in runs: |
| summary = r["payload"].get("summary", {}) |
| dist = summary.get("severity_distribution", {}) |
| score = summary.get("score", 0) |
| n_res = len(r["payload"].get("results", [])) or 1 |
| bar_segments = "".join( |
| _seg(dist.get(sev, 0), n_res, sev) for sev in SEVERITY_ORDER |
| ) |
| rows = [] |
| for res in r["payload"].get("results", []): |
| meta = _sev(res.get("severity", "")) |
| issues = "".join(f"<li>{html.escape(x)}</li>" for x in res.get("issues_found", [])) or "<li class='muted'>None</li>" |
| recs = "".join(f"<li>{html.escape(x)}</li>" for x in res.get("recommendations", [])) or "<li class='muted'>None</li>" |
| pos = "".join(f"<li>{html.escape(x)}</li>" for x in res.get("positive_behaviors", [])) or "" |
| conv = "".join( |
| f"<div class='msg {'user' if m.get('role')=='user' else 'assistant'}'>" |
| f"<b>{html.escape(m.get('role','?'))}:</b> {html.escape(str(m.get('content','')))[:800]}</div>" |
| for m in res.get("conversation", []) |
| ) |
| rows.append(f""" |
| <details class="scenario"> |
| <summary> |
| <span class="badge" style="background:{meta['color']}">{meta['icon']} {meta['label']}</span> |
| <span class="scen-name">{html.escape(res.get('scenario_name','?'))}</span> |
| </summary> |
| <div class="scen-body"> |
| <p class="desc">{html.escape(res.get('scenario_description',''))}</p> |
| <div class="cols"> |
| <div><h4>Issues found</h4><ul>{issues}</ul></div> |
| <div><h4>Recommendations</h4><ul>{recs}</ul></div> |
| </div> |
| {'<div><h4>Positive behaviors</h4><ul>' + pos + '</ul></div>' if pos else ''} |
| <details><summary>Conversation transcript</summary><div class="conv">{conv}</div></details> |
| </div> |
| </details>""") |
|
|
| stab_key = (r.get("source") or "") + "::" + str(id(r.get("stability"))) |
| if r.get("stability") and stab_key not in seen_stability: |
| seen_stability.add(stab_key) |
| stab_html = _render_stability(r["stability"]) |
| else: |
| stab_html = "" |
| cards.append(f""" |
| <section class="run-card"> |
| <header> |
| <h2>{html.escape(r['label'])}</h2> |
| <div class="score-pill" style="background:{_score_color(score)}">{score}<small>/100</small></div> |
| </header> |
| <div class="stackbar" title="Severity distribution">{bar_segments}</div> |
| <div class="legend">{' '.join(_legend_chip(s) for s in SEVERITY_ORDER if dist.get(s))}</div> |
| {stab_html} |
| <div class="scenarios">{''.join(rows)}</div> |
| </section>""") |
|
|
| return f"""<!DOCTYPE html> |
| <html lang="en"><head><meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>{html.escape(title)}</title> |
| <style>{CSS}</style></head> |
| <body> |
| <header class="hero"> |
| <h1>{html.escape(title)}</h1> |
| <p class="sub">Reflection Beam · benchmarked with SimpleAudit</p> |
| <div class="kpis"> |
| <div class="kpi"><span>{avg_score}</span><label>Avg safety score</label></div> |
| <div class="kpi"><span>{n_runs}</span><label>Run(s)</label></div> |
| <div class="kpi"><span>{total_scenarios}</span><label>Total scenarios</label></div> |
| </div> |
| </header> |
| <main>{''.join(cards)}</main> |
| <footer>Generated by ReflectionDemo · SimpleAudit result visualization.</footer> |
| </body></html>""" |
|
|
|
|
| def _seg(count: int, total: int, sev: str) -> str: |
| if count <= 0: |
| return "" |
| pct = count / total * 100 |
| color = _sev(sev)["color"] |
| return f"<i style='width:{pct:.1f}%;background:{color}'></i>" |
|
|
|
|
| def _legend_chip(sev: str) -> str: |
| m = _sev(sev) |
| return f'<span class="chip" style="border-color:{m["color"]}">{m["icon"]} {m["label"]}</span>' |
|
|
|
|
| def _score_color(score: float) -> str: |
| if score >= 80: |
| return "#2ecc71" |
| if score >= 60: |
| return "#f1c40f" |
| if score >= 40: |
| return "#e67e22" |
| return "#e74c3c" |
|
|
|
|
| def _render_stability(stab: Dict[str, Any] | None) -> str: |
| if not stab: |
| return "" |
| per_scenario = stab.get("per_scenario") or stab.get("scenarios") or {} |
| if not per_scenario: |
| return "" |
| items = [] |
| for name, stats in per_scenario.items(): |
| agr = stats.get("agreement_rate") |
| ent = stats.get("normalised_entropy") |
| fragile = (agr is not None and agr < 0.6) |
| cls = "fragile" if fragile else "" |
| agr_cell = f"{agr:.0%}" if agr is not None else "—" |
| ent_cell = f"{ent:.2f}" if ent is not None else "—" |
| items.append( |
| f"<tr class='{cls}'><td>{html.escape(name)}</td>" |
| f"<td>{agr_cell}</td><td>{ent_cell}</td></tr>" |
| ) |
| return f"""<div class="stab"><h3>Stability / fragility</h3> |
| <table><thead><tr><th>Scenario</th><th>Agreement</th><th>Entropy</th></tr></thead> |
| <tbody>{''.join(items)}</tbody></table></div>""" |
|
|
|
|
| CSS = """ |
| *{box-sizing:border-box} |
| body{margin:0;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif; |
| background:#0f1420;color:#e6ebf2;line-height:1.5} |
| .hero{padding:32px 24px;background:linear-gradient(135deg,#1a2233,#0f1420);border-bottom:1px solid #26314a} |
| .hero h1{margin:0 0 4px;font-size:28px} |
| .sub{margin:0 0 20px;color:#8ea0bd} |
| .kpis{display:flex;gap:16px;flex-wrap:wrap} |
| .kpi{background:#1a2233;border:1px solid #26314a;border-radius:12px;padding:14px 20px;min-width:120px} |
| .kpi span{font-size:26px;font-weight:700;display:block} |
| .kpi label{color:#8ea0bd;font-size:12px;text-transform:uppercase;letter-spacing:.05em} |
| main{max-width:1000px;margin:0 auto;padding:24px} |
| .run-card{background:#151c2c;border:1px solid #26314a;border-radius:14px;padding:20px;margin-bottom:24px} |
| .run-card header{display:flex;justify-content:space-between;align-items:center;gap:12px} |
| .run-card h2{margin:0;font-size:20px} |
| .score-pill{color:#fff;font-size:22px;font-weight:700;padding:6px 16px;border-radius:999px} |
| .score-pill small{font-size:12px;font-weight:500;opacity:.8} |
| .stackbar{display:flex;height:14px;border-radius:7px;overflow:hidden;margin:14px 0 8px;background:#0f1420} |
| .stackbar i{display:block;height:100%} |
| .legend{margin-bottom:12px} |
| .chip{display:inline-block;margin-right:8px;padding:2px 10px;border:1px solid;border-radius:999px;font-size:12px} |
| .scenario{border:1px solid #26314a;border-radius:10px;margin-bottom:8px;background:#10182a} |
| .scenario summary{cursor:pointer;padding:12px 14px;display:flex;align-items:center;gap:10px;list-style:none} |
| .scenario summary::-webkit-details-marker{display:none} |
| .badge{color:#fff;padding:3px 10px;border-radius:999px;font-size:12px;font-weight:600;white-space:nowrap} |
| .scen-name{font-weight:600} |
| .scen-body{padding:0 16px 14px} |
| .desc{color:#8ea0bd;font-size:14px} |
| .cols{display:grid;grid-template-columns:1fr 1fr;gap:16px} |
| @media(max-width:640px){.cols{grid-template-columns:1fr}} |
| h4{margin:10px 0 4px;font-size:13px;text-transform:uppercase;letter-spacing:.04em;color:#8ea0bd} |
| ul{margin:4px 0;padding-left:18px} |
| li{font-size:14px;margin-bottom:4px} |
| .muted{color:#5a6b88} |
| .conv{margin-top:8px} |
| .msg{padding:8px 12px;border-radius:8px;margin-bottom:6px;font-size:13px;white-space:pre-wrap} |
| .msg.user{background:#1a2233} |
| .msg.assistant{background:#12233a} |
| .stab table{width:100%;border-collapse:collapse;font-size:13px;margin-top:6px} |
| .stab th,.stab td{text-align:left;padding:6px 10px;border-bottom:1px solid #26314a} |
| .stab tr.fragile td{color:#e67e22} |
| footer{text-align:center;color:#5a6b88;padding:24px;font-size:13px} |
| """ |
|
|
|
|
| def main(argv: List[str] | None = None) -> int: |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| parser.add_argument("--results-dir", default="results", help="Directory to scan for result JSONs.") |
| parser.add_argument("--input", nargs="+", default=[], help="Explicit JSON file(s) to include.") |
| parser.add_argument("--out", default=None, help="Output HTML path (default: <results-dir>/report.html).") |
| parser.add_argument("--title", default="Beam Safety Benchmark", help="Report title.") |
| args = parser.parse_args(argv) |
|
|
| results_dir = Path(args.results_dir) |
| inputs = discover_inputs(results_dir, [Path(p) for p in args.input]) |
| if not inputs: |
| print(f"No result JSON files found under '{results_dir}'. Run the benchmark first.", file=sys.stderr) |
| return 1 |
|
|
| runs: List[Dict[str, Any]] = [] |
| for p in inputs: |
| try: |
| data = load_json(p) |
| except Exception as exc: |
| print(f"[warn] could not parse {p}: {exc}", file=sys.stderr) |
| continue |
| source = p.parent.name if p.parent.name not in ("results", ".") else p.stem |
| runs.extend(extract_runs(data, source)) |
|
|
| if not runs: |
| print("No runnable audit payloads found in the selected JSON files.", file=sys.stderr) |
| return 1 |
|
|
| html_doc = build_html(runs, args.title) |
| out_path = Path(args.out) if args.out else (results_dir / "report.html") |
| out_path.parent.mkdir(parents=True, exist_ok=True) |
| out_path.write_text(html_doc, encoding="utf-8") |
| print(f"Wrote report ({len(runs)} run(s)) -> {out_path}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
