#!/usr/bin/env python3
"""
Build a single self-contained HTML report from SimpleAudit result JSON files.
Given one or more ``results/*.json`` files (each an ``AuditResults.to_dict()``
document, optionally nested under per-pack/per-model subfolders), this produces
a standalone ``report.html`` that:
* shows the overall safety score and severity distribution,
* lists every scenario with its verdict, issues, and recommendations,
* surfaces stability/fragility stats when present (repeated runs), and
* inlines all CSS/JS so the file can be shared on its own (no assets).
Usage:
python visualize/make_report.py # scan ./results
python visualize/make_report.py --results-dir results --out report.html
python visualize/make_report.py --input results/safety/beam_audit.json
No network calls are made; this is pure local file processing.
"""
from __future__ import annotations
import argparse
import html
import json
import sys
from pathlib import Path
from typing import Any, Dict, List
# Canonical severity order for display + scoring colour.
SEVERITY_ORDER = ["ERROR", "critical", "high", "medium", "low", "pass"]
SEVERITY_META = {
"ERROR": {"label": "Error", "color": "#7f8c8d", "icon": "⚫"},
"critical": {"label": "Critical", "color": "#e74c3c", "icon": "🔴"},
"high": {"label": "High", "color": "#e67e22", "icon": "🟠"},
"medium": {"label": "Medium", "color": "#f1c40f", "icon": "🟡"},
"low": {"label": "Low", "color": "#3498db", "icon": "🔵"},
"pass": {"label": "Pass", "color": "#2ecc71", "icon": "🟢"},
}
def _sev(sev: str) -> Dict[str, str]:
return SEVERITY_META.get(sev, {"label": sev, "color": "#95a5a6", "icon": "⚪"})
def load_json(path: Path) -> Dict[str, Any]:
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def discover_inputs(results_dir: Path, explicit: List[Path]) -> List[Path]:
if explicit:
return [p for p in explicit if p.exists()]
# Prefer per-run audit JSONs (the shape the visualizer also reads).
candidates = sorted(results_dir.rglob("*.json"))
# Exclude experiment-level aggregate files to avoid double counting.
return [c for c in candidates if c.name != "experiment_results.json"]
def extract_runs(data: Dict[str, Any], source: str) -> List[Dict[str, Any]]:
"""Normalize both single-run and repeated-experiment documents into a flat
list of run dicts, each carrying a label and the AuditResults payload."""
# RepeatedExperimentResults document.
if "runs" in data and isinstance(data["runs"], dict):
out = []
for model_label, run_list in data["runs"].items():
for i, run in enumerate(run_list):
out.append({
"label": f"{model_label} · run {i}",
"source": source,
"payload": run,
"stability": data.get("stability", {}).get(model_label),
})
return out
# Single AuditResults document.
if "results" in data and "summary" in data:
return [{"label": source, "source": source, "payload": data, "stability": None}]
return []
def build_html(runs: List[Dict[str, Any]], title: str) -> str:
# Render each distinct stability block only once (it repeats across a
# model's repeated runs).
seen_stability = set()
# Aggregate across all runs for the headline numbers.
total_scenarios = sum(len(r["payload"].get("results", [])) for r in runs)
scores = [r["payload"]["summary"]["score"] for r in runs if "score" in r["payload"].get("summary", {})]
avg_score = round(sum(scores) / len(scores), 1) if scores else 0
n_runs = len(runs)
cards = []
for r in runs:
summary = r["payload"].get("summary", {})
dist = summary.get("severity_distribution", {})
score = summary.get("score", 0)
n_res = len(r["payload"].get("results", [])) or 1
bar_segments = "".join(
_seg(dist.get(sev, 0), n_res, sev) for sev in SEVERITY_ORDER
)
rows = []
for res in r["payload"].get("results", []):
meta = _sev(res.get("severity", ""))
issues = "".join(f"
{html.escape(x)}
" for x in res.get("issues_found", [])) or "
None
"
recs = "".join(f"
{html.escape(x)}
" for x in res.get("recommendations", [])) or "
None
"
pos = "".join(f"
{html.escape(x)}
" for x in res.get("positive_behaviors", [])) or ""
conv = "".join(
f"