Blame for autojev/report.py

"""Replay recorded work, training loss and evaluation measurements as a report or film."""

import argparse
from collections.abc import Mapping, Sequence
from datetime import datetime
import hashlib
import html
import importlib
import json
import math
from pathlib import Path
import subprocess
import textwrap
import time
from typing import Callable, Literal, TypedDict, cast

import matplotlib

matplotlib.use("Agg")
from matplotlib import pyplot as plt
from matplotlib.axes import Axes
from matplotlib.backends.backend_agg import FigureCanvasAgg
from matplotlib.figure import Figure
from matplotlib.ticker import PercentFormatter
import numpy as np

from autojev.types import JSONValue


type Event = dict[str, JSONValue]
type Panel = Literal["full", "quick"]
type Cohort = Literal["public", "guard"]
GREEN = "#286b4d"
GRAY = "#87938f"
INK = "#273431"
PALE = "#c7dccb"
RAW = "#597d99"
MARGIN = 0.010
RUN_EVENTS = {"training_started", "training_step", "evaluation", "quick_evaluation", "training_paused", "training_finished", "training_failed", "run_restarted"}
QUICK_NOTE = "Quick diagnostic: 1,000 public / 200 guard / 700 temperature-fit rows · never selects checkpoints"


class Series(TypedDict):
    run: str
    panel: Panel
    cohort: Cohort
    events: list[Event]
    steps: list[Event]
    evaluations: list[Event]
    reference: dict[str, float]
    total_steps: int


class HistoryRow(TypedDict):
    run: str
    step: int
    status: str
    detail: str
    recovery: str


def number(value: JSONValue | object, default: float = 0.0) -> float:
    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else default


def timestamp(event: Event) -> float:
    value = event.get("timestamp")
    if not isinstance(value, str):
        raise ValueError("Every report event needs a UTC timestamp")
    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
    if parsed.tzinfo is None:
        raise ValueError("Report timestamps must have a time zone")
    return parsed.timestamp()


def load_events(path: Path) -> list[Event]:
    return parse_events(path.read_text())


def parse_events(content: str) -> list[Event]:
    lines = content.splitlines()
    # A live writer may not have finished its final line yet.
    if content and not content.endswith("\n"):
        lines = lines[:-1]
    events: list[Event] = []
    for line in lines:
        if line.strip():
            event = cast(Event, json.loads(line))
            if not isinstance(event.get("kind"), str):
                raise ValueError("Every report event needs a kind")
            timestamp(event)
            events.append(event)
    if not events:
        raise ValueError("No complete recorded events are available")
    return sorted(events, key=timestamp)


def metric_values(event: Event, key: str) -> dict[str, float]:
    value = event.get(key)
    if not isinstance(value, dict):
        return {}
    result: dict[str, float] = {}
    for name in ("accuracy", "ece", "brier", "nll"):
        if name in value:
            metric = value[name]
            if not isinstance(metric, (int, float)) or isinstance(metric, bool) or not math.isfinite(metric):
                raise ValueError(f"Invalid recorded {key}/{name}")
            result[name] = float(metric)
    return result


def cohort_event(event: Event, cohort: Cohort) -> Event | None:
    if cohort == "public":
        return event
    if not all(isinstance(event.get(f"guard_{key}"), dict) for key in ("raw", "fitted", "jev")):
        return None
    return {**event, **{key: event[f"guard_{key}"] for key in ("raw", "fitted", "jev")}}


def prepare(events: Sequence[Event], run: str | None, panel: Panel = "full", cohort: Cohort = "public") -> Series:
    candidates = [str(event["run"]) for event in events if isinstance(event.get("run"), str) and event["kind"] in ("training_started", "evaluation", "quick_evaluation", "run_restarted", "final_comparison")]
    selected = run if run is not None else (candidates[-1] if candidates else "not started")
    if run is not None and run not in candidates:
        raise ValueError(f"No recorded training run named {run}")
    relevant = [event for event in events if event.get("run") == selected]
    steps = [event for event in relevant if event["kind"] == "training_step"]
    kind = "quick_evaluation" if panel == "quick" else "evaluation"
    evaluations: list[Event] = []
    for event in relevant:
        if event["kind"] != kind:
            continue
        if panel == "quick" and (event.get("diagnostic") is not True or event.get("selection_candidate") is not False
                                 or tuple(event.get(key) for key in ("development_rows", "guard_rows", "temperature_rows")) != (1000, 200, 700)):
            raise ValueError("Quick panels require the declared 1000/200/700 diagnostic-only measurements")
        selected_event = cohort_event(event, cohort)
        if selected_event is not None:
            evaluations.append(selected_event)
    reference: dict[str, float] = {}
    for event in evaluations:
        current = metric_values(event, "jev")
        if reference and current != reference:
            raise ValueError("A fixed development comparison cannot change its Jev baseline mid-run")
        reference = current
    total = max([int(number(event.get("total_steps"))) for event in relevant] + [int(number(event.get("step"))) for event in steps] + [1])
    return {"run": selected, "panel": panel, "cohort": cohort, "events": list(events), "steps": steps, "evaluations": evaluations, "reference": reference, "total_steps": total}


def research_history(events: Sequence[Event]) -> list[HistoryRow]:
    """Lifecycle summaries remain visible after individual work items scroll away."""
    runs: dict[str, HistoryRow] = {}
    for event in events:
        run, kind = event.get("run"), str(event["kind"])
        if not isinstance(run, str) or kind not in RUN_EVENTS | {"pilot_recovery_verified"}:
            continue
        row = runs.setdefault(run, {"run": run, "step": 0, "status": "recorded", "detail": "", "recovery": ""})
        row["step"] = max(row["step"], int(number(event.get("step", event.get("steps")))))
        if kind == "run_restarted":
            previous = event.get("previous_run", "previous attempt")
            row["status"] = "restart planned"
            row["detail"] = f"After {previous}: {event.get('reason', 'separate recorded run')}"
        elif kind == "training_started":
            row["status"] = "running"
        elif kind in ("training_paused", "training_finished"):
            row["status"] = kind.removeprefix("training_")
        elif kind == "training_failed":
            phase = str(event.get("phase", ""))
            row["status"] = "restart failed" if phase.startswith("resume") else "training failed"
            row["detail"] = str(event.get("error", event.get("reason", "Failure recorded")))
            if event.get("updates_after_resume") == 0:
                row["detail"] += "; no additional optimizer update"
        elif kind == "pilot_recovery_verified" and event.get("logits_bitwise_equal") is True:
            row["recovery"] = f"Recovery matched {int(number(event.get('rows'))):,} development rows exactly."
    return list(runs.values())


def segments(events: Sequence[Event]) -> list[list[Event]]:
    """Show an explicit curve break when a resumed attempt repeats earlier steps."""
    result: list[list[Event]] = []
    previous = -1.0
    for event in events:
        step = number(event.get("step"))
        if not result or step <= previous:
            result.append([])
        result[-1].append(event)
        previous = step
    return result


def style_axis(axis: Axes, title: str) -> None:
    axis.set_title(title, loc="left", fontsize=12, fontweight="bold", color=INK, pad=12)
    axis.spines[["top", "right"]].set_visible(False)
    axis.spines[["left", "bottom"]].set_color("#d2d9d5")
    axis.tick_params(colors="#61726a", labelsize=9)
    axis.grid(axis="y", color="#edf1ee", linewidth=0.8)
    axis.set_axisbelow(True)


def trailing_mean(values: Sequence[float], window: int = 10) -> list[float]:
    return [sum(values[max(0, i - window + 1):i + 1]) / min(i + 1, window) for i in range(len(values))]


def milestone(event: Event) -> str:
    kind = str(event["kind"])
    task = event.get("task")
    if kind in ("task_started", "task_finished", "agent_task"):
        status = str(event.get("status", "started" if kind == "task_started" else "completed"))
        return f"{event.get('agent', 'Team')}: {str(task or kind).replace('_', ' ')} ({status})"
    if kind == "data_task_partitioned":
        return f"Partitioned {task} by source family"
    if kind.startswith("data_"):
        return f"{kind.removeprefix('data_').replace('_', ' ')}: {task}" if task else kind.replace("_", " ")
    if kind in {"evaluation", "quick_evaluation"}:
        values = metric_values(event, "fitted")
        prefix = "Quick diagnostic · " if kind == "quick_evaluation" else ""
        return f"{prefix}Step {int(number(event.get('step')))}: accuracy {values.get('accuracy', 0):.2%}; ECE {values.get('ece', 0):.4f}"
    if kind == "training_step":
        return f"Optimizer step {int(number(event.get('step')))}"
    if kind == "training_started":
        return f"Started {event.get('run', 'training')} from step {int(number(event.get('step')))}"
    if kind == "training_failed":
        phase = str(event.get("phase", "")).replace("_", " ")
        return f"{event.get('run', 'Training')}: {phase or 'training'} failed at step {int(number(event.get('step')))}"
    if kind == "run_restarted":
        return f"{event.get('previous_run', 'Previous run')} → {event.get('run', 'new run')}: {event.get('reason', 'restart recorded')}"
    if kind == "pilot_recovery_verified":
        return f"{event.get('run', 'Pilot')}: recovery checked on {int(number(event.get('rows'))):,} development rows"
    if kind in ("training_finished", "training_paused"):
        return f"{event.get('run', 'Training')}: {kind.removeprefix('training_')} at step {int(number(event.get('steps')))}"
    if kind == "public_evaluation":
        accuracy = metric_values(event, "metrics").get("accuracy")
        return f"Known suite · update {int(number(event.get('step')))} · accuracy {accuracy:.2%}" if accuracy is not None else "Known-suite evaluation"
    if kind == "final_comparison":
        return f"Locked test: {int(number(event.get('count'))):,} paired examples"
    if kind == "repeat_cancelled" and event.get("started") is False:
        return f"Cancelled {event.get('run', 'repeat')} before training; one full seed"
    return str(task or kind).replace("_", " ")


def active_series(events: Sequence[Event], cutoff: float, panel: Panel = "full", cohort: Cohort = "public") -> Series:
    """Follow recorded run starts; a final comparison names the frozen winning run."""
    visible = [event for event in events if timestamp(event) <= cutoff]
    transitions = [event for event in visible if event["kind"] in ("training_started", "run_restarted", "final_comparison") and isinstance(event.get("run"), str)]
    return prepare(events, str(transitions[-1]["run"]), panel, cohort) if transitions else prepare(visible, None, panel, cohort)


def draw(series: Series, cutoff: float, *, width: int = 1920, height: int = 1080, all_runs: bool = False) -> Figure:
    figure = plt.figure(figsize=(width / 120, height / 120), dpi=120, facecolor="white")
    grid = figure.add_gridspec(3, 3, width_ratios=[1.05, 1.35, 1.2], left=0.045, right=0.97, top=0.83, bottom=0.11, hspace=0.70, wspace=0.35)
    activity = figure.add_subplot(grid[:, 0])
    loss_axis = figure.add_subplot(grid[:, 1])
    metric_axes = [figure.add_subplot(grid[index, 2]) for index in range(3)]
    visible = [event for event in series["events"] if timestamp(event) <= cutoff]
    steps = [event for event in series["steps"] if timestamp(event) <= cutoff]
    evaluations = [event for event in series["evaluations"] if timestamp(event) <= cutoff]
    final_events = [event for event in visible if event["kind"] == "final_comparison" and event.get("run") == series["run"]] if series["panel"] == "full" and series["cohort"] == "public" else []
    final = final_events[-1] if final_events else None
    final_effects = cast(dict[str, JSONValue], final.get("effects", {})) if final else {}
    first = timestamp(series["events"][0])
    elapsed = max(0, int(cutoff - first))
    heading = "AUTOJEV / LOCKED TEST" if final else "AUTOJEV / SUPERVISED LEARNING"
    if all_runs and not final:
        heading = "AUTOJEV / RESEARCH" if series["run"] == "not started" else f"AUTOJEV / {series['run'].upper()}"
    if series["panel"] == "quick":
        heading = f"AUTOJEV / QUICK {series['cohort'].upper()} DIAGNOSTIC"
    elif series["cohort"] == "guard":
        heading = "AUTOJEV / FULL SYNTHETIC GUARD"
    figure.text(0.045, 0.93, heading, color=GREEN, fontsize=22, weight="bold")
    identity = f"{series['run']} · selected update {final.get('selected_step', 'unrecorded')} · T={number(final.get('temperature')):.4f}" if final else series["run"]
    figure.text(0.045, 0.885, f"{identity}  ·  {elapsed // 3600:02d}:{elapsed // 60 % 60:02d}:{elapsed % 60:02d} actual elapsed", color="#61726a", fontsize=11)
    if visible:
        figure.text(0.97, 0.925, str(visible[-1]["timestamp"])[:19].replace("T", " ") + " UTC", ha="right", fontsize=10, color=GRAY)

    activity.set_axis_off()
    activity.set_title("Recorded work", loc="left", fontsize=13, fontweight="bold", color=INK, pad=12)
    milestones = [event for event in visible if event["kind"] not in ("training_step", "resume_saved")]
    history = research_history(visible)
    show_history = any("failed" in row["status"] or row["detail"] for row in history)
    recent = milestones[-3:] if show_history else milestones[-7:]
    cursor = 0.95
    for event in recent:
        label = textwrap.fill(milestone(event), width=36, max_lines=3, placeholder="…")
        color = "#a45835" if event["kind"] == "training_failed" else GREEN
        activity.text(0, cursor, "●", color=color, fontsize=10, va="top", transform=activity.transAxes)
        activity.text(0.07, cursor, label, color=INK, fontsize=10, va="top", linespacing=1.4, transform=activity.transAxes)
        cursor -= 0.052 + 0.035 * len(label.splitlines())
    if not recent:
        activity.text(0, 0.9, "Waiting for recorded work", color=GRAY, transform=activity.transAxes)
    if show_history:
        activity.text(0, 0.41, "Research history", fontweight="bold", color=INK, fontsize=10, transform=activity.transAxes)
        cursor = 0.36
        for row in history[-3:]:
            color = "#a45835" if "failed" in row["status"] else GREEN
            activity.text(0, cursor, f"{row['run']} · update {row['step']} · {row['status']}", fontsize=9, color=color, va="top", transform=activity.transAxes)
            detail = row["recovery"] or row["detail"]
            if detail:
                activity.text(0, cursor - 0.035, textwrap.fill(detail, width=47, max_lines=2, placeholder="…"), fontsize=8, color=GRAY, va="top", transform=activity.transAxes)
            cursor -= 0.11
    repeat_cancelled = any(event["kind"] == "repeat_cancelled" for event in visible)
    if repeat_cancelled:
        activity.text(0, 0.095, "One full seed; planned repeat cancelled.\nSeed-to-seed robustness was not measured.",
                      color=GRAY, fontsize=8.5, linespacing=1.5, transform=activity.transAxes)
    activity.text(0, -0.05, "Research attempts remain in the record.\nCurves show the selected run only.", color=GRAY, fontsize=8.5, linespacing=1.5, transform=activity.transAxes)

    style_axis(loss_axis, "Training cross-entropy ↓")
    loss_axis.set_xlim(0, series["total_steps"])
    maximum = max([number(event.get("loss")) for event in series["steps"]] + [1.0])
    loss_axis.set_ylim(0, maximum * 1.10)
    loss_axis.set_xlabel("Optimizer update", color="#61726a", fontsize=10)
    for index, segment in enumerate(segments(steps)):
        x = [number(event.get("step")) for event in segment]
        y = [number(event.get("loss")) for event in segment]
        marker = "o" if len(segment) == 1 else None
        loss_axis.plot(x, y, color=PALE, linewidth=1.5, marker=marker, markersize=3.5, label="Recorded loss" if index == 0 else None)
        loss_axis.plot(x, trailing_mean(y), color=GREEN, linewidth=2.1, marker=marker, markersize=3.5, label="Trailing 10-update mean" if index == 0 else None)
    if steps:
        loss_axis.legend(frameon=False, fontsize=8, loc="upper right")
        loss_axis.text(0.04, 0.05, f"Latest loss  {number(steps[-1].get('loss')):.4f}", color=GREEN, fontsize=12, weight="bold", transform=loss_axis.transAxes)
    else:
        loss_axis.text(0.5, 0.5, "Training measurements\nwill appear when recorded", ha="center", va="center", color=GRAY, fontsize=11, transform=loss_axis.transAxes)
    if final and isinstance(final.get("selected_step"), int):
        loss_axis.axvline(number(final["selected_step"]), color=GRAY, linestyle=":", linewidth=1)
        loss_axis.text(0.04, 0.095, f"Frozen checkpoint: update {final['selected_step']}", color=GRAY, fontsize=9, transform=loss_axis.transAxes)

    for axis, name, title in zip(metric_axes, ("accuracy", "ece", "brier"), ("Overall accuracy ↑", "Calibration error ↓", "Brier probability score ↓"), strict=True):
        style_axis(axis, title)
        if final:
            effect = cast(dict[str, JSONValue], final_effects[name])
            local, reference_value = number(effect["local"]), number(effect["reference"])
            axis.bar([0, 1], [local, reference_value], width=0.55, color=[GREEN, GRAY])
            axis.set_xticks([0, 1], ["Local", "Jev"])
            axis.set_xlim(-0.7, 1.7)
            if name == "accuracy":
                axis.set_ylim(0, 1)
                axis.yaxis.set_major_formatter(PercentFormatter(1, decimals=0))
            else:
                axis.set_ylim(0, max(local, reference_value + MARGIN, MARGIN) * 1.2)
                axis.axhline(reference_value + MARGIN, color="#a0b09e", linestyle=":", linewidth=1)
            for position, value in enumerate((local, reference_value)):
                axis.annotate(f"{value:.2%}" if name == "accuracy" else f"{value:.4f}", (position, value), xytext=(0, 5), textcoords="offset points", ha="center", fontsize=9, color=INK)
            interval = cast(list[JSONValue], effect["difference_ci95"])
            scale, unit = (100, " pp") if name == "accuracy" else (1, "")
            difference = number(effect["difference"]) * scale
            low, high = number(interval[0]) * scale, number(interval[1]) * scale
            axis.text(0.5, -0.34, f"Δ {difference:+.3f}{unit} · paired 95% [{low:+.3f}, {high:+.3f}]", ha="center", fontsize=8, color=GRAY, transform=axis.transAxes)
            continue
        axis.set_xlim(0, series["total_steps"])
        probability_modes = ("fitted", "raw") if name != "accuracy" else ("fitted",)
        all_values = [metric_values(event, mode)[name] for event in series["evaluations"] for mode in probability_modes if name in metric_values(event, mode)]
        reference = series["reference"].get(name)
        if reference is not None:
            all_values.append(reference)
        if name == "accuracy":
            axis.set_ylim(max(0, min(all_values, default=0.5) - 0.045), min(1, max(all_values, default=0.95) + 0.035))
            axis.yaxis.set_major_formatter(PercentFormatter(1, decimals=0))
        else:
            axis.set_ylim(0, max(all_values + [MARGIN], default=0.1) * 1.2 + MARGIN)
        if reference is not None and evaluations:
            axis.axhline(reference, color=GRAY, linewidth=1.2, linestyle="--", label="Jev")
            if name != "accuracy":
                axis.axhspan(0, reference + MARGIN, color=PALE, alpha=0.25)
                axis.axhline(reference + MARGIN, color="#a0b09e", linewidth=0.9, linestyle=":", label="Jev + 0.010 margin")
        for index, segment in enumerate(segments(evaluations)):
            x = [number(event.get("step")) for event in segment]
            y = [metric_values(event, "fitted")[name] for event in segment]
            if name != "accuracy":
                raw = [metric_values(event, "raw")[name] for event in segment]
                axis.plot(x, raw, color=RAW, linewidth=1.4, linestyle=":", marker="x", markersize=4, label="Raw (diagnostic)" if index == 0 else None)
            label = "Local" if name == "accuracy" else "Fitted"
            axis.plot(x, y, color=GREEN, linewidth=1.8, marker="o", markersize=3.5, label=label if index == 0 else None)
        if evaluations:
            axis.legend(frameon=True, facecolor="white", edgecolor="none", framealpha=0.96, fancybox=False,
                        fontsize=7, loc="upper right", ncol=1 if name == "accuracy" else 2)
        else:
            axis.text(0.5, 0.5, "No evaluation recorded", ha="center", va="center", color=GRAY, fontsize=9, transform=axis.transAxes)
    if final:
        status = "met" if final.get("empirical_target_met") is True else "not met"
        supported = "established" if final.get("statistically_supported") is True else "not established"
        footer = f"Locked test: {int(number(final.get('count'))):,} examples · observed accuracy/calibration target {status} · paired statistical support {supported}"
        if repeat_cancelled:
            footer += " · one training seed"
    else:
        metric_axes[-1].set_xlabel(f"Optimizer update · {'quick ' if series['panel'] == 'quick' else ''}{'synthetic guard' if series['cohort'] == 'guard' else 'fixed development fold'}", color="#61726a", fontsize=9)
        footer = QUICK_NOTE + f" · plotted: {series['cohort']} · Jev uses the same subset" if series["panel"] == "quick" else "Measured checkpoints only · raw probabilities: diagnostic · selection/final comparison: fitted (separate temperature fold) · hard reference labels"
    figure.text(0.045, 0.035, footer, color=GRAY, fontsize=9)
    return figure


def dashboard(series: Series, destination: Path, *, watch: bool = False) -> None:
    # Publish only the fields the replay needs; usage/accounting fields are never embedded.
    keep = {"timestamp", "kind", "run", "task", "agent", "status", "step", "steps", "total_steps", "loss", "fitted", "raw", "jev", "eligible", "selected_step", "temperature", "count", "effects", "empirical_target_met", "statistically_supported", "previous_run", "reason", "error", "phase", "from_base", "recipe_changed", "updates_after_resume", "rows", "logits_bitwise_equal", "calibration_margin", "freeze_sha256", "metrics", "started", "guard_raw", "guard_fitted", "guard_jev", "diagnostic", "selection_candidate", "development_rows", "guard_rows", "temperature_rows"}
    events = [{key: value for key, value in event.items() if key in keep} for event in series["events"]]
    for event in events:
        if "metrics" in event:
            event["metrics"] = cast(JSONValue, metric_values(event, "metrics"))
    payload = json.dumps({"events": events, "run": series["run"], "panel": series["panel"], "cohort": series["cohort"], "total_steps": series["total_steps"], "watch": watch}, ensure_ascii=False, allow_nan=False).replace("<", "\\u003c")
    content = DASHBOARD.replace("__PAYLOAD__", payload).replace("__TITLE__", html.escape(series["run"]))
    destination.write_text(content)
    if watch:
        temporary = destination.with_name("live.json.tmp")
        temporary.write_text(payload)
        temporary.replace(destination.with_name("live.json"))


DASHBOARD = r'''<!doctype html>
<html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>AutoJev · __TITLE__</title>
<style>
*{box-sizing:border-box}body{margin:0;background:#f6f8f6;color:#273431;font:15px system-ui,sans-serif}main{max-width:1500px;margin:35px auto;padding:28px;background:white;border:1px solid #e2e8e2;border-radius:18px}h1{color:#286b4d;font-size:26px;margin:0 0 9px}p,.muted{color:#718078}.controls{display:flex;gap:14px;align-items:center;margin:24px 0}button,select{background:white;border:1px solid #b9c9bd;border-radius:7px;padding:9px;color:#273431}input{flex:1;accent-color:#286b4d}.panels{display:grid;grid-template-columns:1fr 1.5fr 1.3fr;gap:28px}h2{font-size:16px}canvas{width:100%;height:185px;display:block}#loss{height:535px}#tasks{padding:0;list-style:none}#tasks li{border-left:2px solid #bed4c4;padding:0 0 18px 13px;line-height:1.5}.metric{border-bottom:1px solid #eef2ef;padding-bottom:14px}.stat{color:#286b4d;font-weight:650}footer{font-size:12px;margin-top:24px;color:#718078;line-height:1.6}.legend{font-size:12px;color:#718078}#table{width:100%;border-collapse:collapse;font-size:13px;margin-top:18px}td,th{text-align:right;border-bottom:1px solid #eef2ef;padding:8px}td:first-child,th:first-child{text-align:left}details{margin-top:25px}@media(max-width:850px){main{margin:0;border:0;padding:22px}.panels{grid-template-columns:1fr}#loss{height:280px}.controls{flex-wrap:wrap}input{min-width:180px}}
</style><main><h1>AUTOJEV / SUPERVISED LEARNING</h1><div class="muted" id="status"></div>
<style>#history{margin-top:28px;border-top:1px solid #e2e8e2;padding-top:14px}#history h3{font-size:14px}#history-list p{font-size:12px;margin:5px 0 15px;line-height:1.5}.failed{color:#a45835}</style>
<div class="controls"><button id="play">Play</button><input id="time" type="range" min="0" step="1"><label>Run <select id="run"></select></label><label>Panel <select id="panel"><option value="full">Full evaluations</option><option value="quick">Quick diagnostic</option></select></label><label>Cohort <select id="cohort"><option value="public">Public</option><option value="guard">Synthetic guard</option></select></label><label>Probabilities <select id="mode"><option value="fitted">Temperature fitted</option><option value="raw">Raw</option></select></label></div>
<p class="legend" id="panel-note"></p>
<div class="panels"><section><h2>Recorded work</h2><ul id="tasks"></ul><section id="history" hidden><h3>Research history</h3><div id="history-list"></div><p class="legend">Each run has its own curve. The setup pilot is not part of the full run.</p></section></section><section><h2>Training cross-entropy ↓</h2><canvas id="loss"></canvas><div class="legend">Light: recorded loss · dark: trailing 10-update mean</div></section><section>
<div class="metric"><h2>Overall accuracy ↑ <span class="stat" id="accuracy-value"></span></h2><canvas id="accuracy"></canvas></div>
<div class="metric"><h2>Calibration error ↓ <span class="stat" id="ece-value"></span></h2><canvas id="ece"></canvas></div>
<div class="metric"><h2>Brier probability score ↓ <span class="stat" id="brier-value"></span></h2><canvas id="brier"></canvas></div>
<div class="legend">Green: local · dashed: Jev on the same selected cohort · shaded: Jev + 0.010 margin</div></section></div>
<details><summary>Measured checkpoints</summary><table id="table"><thead><tr><th>Step</th><th>Accuracy</th><th>ECE</th><th>Brier</th><th>Temperature</th><th>Eligible</th></tr></thead><tbody></tbody></table></details>
<section id="final" hidden><h2>Locked test comparison</h2><p id="final-status"></p><table id="final-table"><thead><tr><th>Metric</th><th>Local</th><th>Jev</th><th>Difference</th><th>Paired 95% interval</th></tr></thead><tbody></tbody></table></section>
<footer id="report-note">Every point comes from a recorded evaluation on the fixed development fold. Temperature is fitted on a separate fold and does not change accuracy. Raw curves remain available. Repeated steps after a restart create an explicit curve break. Research attempts are separate runs; no best-so-far substitution or synthetic evaluation points are used.</footer></main>
<script>
let data=__PAYLOAD__,events=data.events;const slider=document.querySelector('#time'),selector=document.querySelector('#run'),mode=document.querySelector('#mode'),panel=document.querySelector('#panel'),cohort=document.querySelector('#cohort');panel.value=data.panel||'full';cohort.value=data.cohort||'public';const fullNote=document.querySelector('#report-note').textContent;
function refreshRuns(selected){const runs=[...new Set(events.filter(e=>e.run&&['training_started','evaluation','quick_evaluation','run_restarted','final_comparison'].includes(e.kind)).map(e=>e.run))];if(!runs.length)runs.push('not started');selector.replaceChildren();for(const run of runs){const option=document.createElement('option');option.value=run;option.textContent=run;selector.append(option)}selector.value=runs.includes(selected)?selected:data.run}refreshRuns(data.run);slider.max=events.length-1;slider.value=events.length-1;
function label(e){
if(e.kind==='training_failed')return `${e.run}: ${(e.phase||'training').replaceAll('_',' ')} failed at step ${e.step}`;
if(e.kind==='run_restarted')return `${e.previous_run||'Previous run'} → ${e.run}: ${e.reason||'restart recorded'}`;
if(e.kind==='pilot_recovery_verified')return `${e.run}: recovery checked on ${e.rows.toLocaleString()} development rows`;
if(e.kind==='public_evaluation')return Number.isFinite(e.metrics?.accuracy)?`Known suite · update ${e.step} · accuracy ${(e.metrics.accuracy*100).toFixed(2)}%`:'Known-suite evaluation';
if(['evaluation','quick_evaluation'].includes(e.kind))return `${e.kind==='quick_evaluation'?'Quick diagnostic · ':''}Step ${e.step}: accuracy ${(e.fitted.accuracy*100).toFixed(2)}%; ECE ${e.fitted.ece.toFixed(4)}`;
if(e.kind==='data_task_partitioned')return `Partitioned ${e.task} by source family`;
if(e.kind.startsWith('data_')&&e.task)return `${e.kind.slice(5).replaceAll('_',' ')}: ${e.task}`;
return `${e.agent?e.agent+': ':''}${String(e.task||e.kind).replaceAll('_',' ')}${e.status?' ('+e.status+')':''}`}
function historyRows(visible){const result=new Map();for(const e of visible){if(!e.run||!['training_started','training_step','evaluation','quick_evaluation','training_paused','training_finished','training_failed','run_restarted','pilot_recovery_verified'].includes(e.kind))continue;if(!result.has(e.run))result.set(e.run,{run:e.run,step:0,status:'recorded',detail:'',recovery:''});const row=result.get(e.run);row.step=Math.max(row.step,e.step||e.steps||0);if(e.kind==='run_restarted'){row.status='restart planned';row.detail=`After ${e.previous_run||'previous attempt'}: ${e.reason||'separate recorded run'}`}else if(e.kind==='training_started')row.status='running';else if(['training_paused','training_finished'].includes(e.kind))row.status=e.kind.slice(9);else if(e.kind==='training_failed'){row.status=(e.phase||'').startsWith('resume')?'restart failed':'training failed';row.detail=e.error||e.reason||'Failure recorded';if(e.updates_after_resume===0)row.detail+='; no additional optimizer update'}else if(e.kind==='pilot_recovery_verified'&&e.logits_bitwise_equal)row.recovery=`Recovery matched ${e.rows.toLocaleString()} development rows exactly.`}return [...result.values()]}
function split(points){const groups=[];for(const p of points){if(!groups.length||p[0]<=groups[groups.length-1].at(-1)[0])groups.push([]);groups.at(-1).push(p)}return groups}
function chart(id,groups,range,reference,margin,xmax,percent=false){const c=document.getElementById(id),dpr=devicePixelRatio||1,w=c.clientWidth,h=c.clientHeight;c.width=w*dpr;c.height=h*dpr;const ctx=c.getContext('2d');ctx.scale(dpr,dpr);const l=46,r=w-14,t=12,b=h-31;const x=v=>l+v/xmax*(r-l),y=v=>b-(v-range[0])/(range[1]-range[0])*(b-t);ctx.font='11px system-ui';ctx.fillStyle='#718078';ctx.strokeStyle='#edf1ee';for(let i=0;i<=3;i++){const v=range[0]+i/3*(range[1]-range[0]),yy=y(v);ctx.beginPath();ctx.moveTo(l,yy);ctx.lineTo(r,yy);ctx.stroke();ctx.fillText(percent?(100*v).toFixed(0)+'%':v.toFixed(2),2,yy+4)}if(reference!==null){if(margin){ctx.fillStyle='#c7dccb55';ctx.fillRect(l,y(reference+margin),r-l,b-y(reference+margin));ctx.strokeStyle='#a0b09e';ctx.setLineDash([2,3]);ctx.beginPath();ctx.moveTo(l,y(reference+margin));ctx.lineTo(r,y(reference+margin));ctx.stroke()}ctx.strokeStyle='#87938f';ctx.setLineDash([5,4]);ctx.beginPath();ctx.moveTo(l,y(reference));ctx.lineTo(r,y(reference));ctx.stroke();ctx.setLineDash([])}for(const group of groups){ctx.strokeStyle=group.color;ctx.fillStyle=group.color;ctx.lineWidth=group.width||2;for(const part of split(group.points)){ctx.beginPath();part.forEach((p,i)=>i?ctx.lineTo(x(p[0]),y(p[1])):ctx.moveTo(x(p[0]),y(p[1])));ctx.stroke();if(group.dots||part.length===1)for(const p of part){ctx.beginPath();ctx.arc(x(p[0]),y(p[1]),3,0,Math.PI*2);ctx.fill()}}}ctx.fillStyle='#718078';ctx.fillText('0',l,b+21);ctx.fillText(String(xmax),r-20,b+21);if(groups.every(g=>!g.points.length)){ctx.fillStyle='#87938f';ctx.fillText('No measurements recorded',l+10,(t+b)/2)}}
function panelEvents(items){const kind=panel.value==='quick'?'quick_evaluation':'evaluation';return items.filter(e=>e.kind===kind).map(e=>{if(panel.value==='quick'&&(e.diagnostic!==true||e.selection_candidate!==false||e.development_rows!==1000||e.guard_rows!==200||e.temperature_rows!==700))throw Error('Quick panel requires the declared 1000/200/700 diagnostic-only measurements');return cohort.value==='public'?e:e.guard_raw&&e.guard_fitted&&e.guard_jev?{...e,raw:e.guard_raw,fitted:e.guard_fitted,jev:e.guard_jev}:null}).filter(Boolean)}
function render(){const position=+slider.value,visible=events.slice(0,position+1),run=selector.value,all=events.filter(e=>e.run===run),current=visible.filter(e=>e.run===run),steps=current.filter(e=>e.kind==='training_step'),evals=panelEvents(current),allEvals=panelEvents(all),xmax=Math.max(1,...all.map(e=>e.total_steps||e.step||0));const last=visible.at(-1);document.querySelector('#status').textContent=`${run} · ${last.timestamp.replace('T',' ').slice(0,19)} UTC · ${position+1}/${events.length} recorded events`;
const quick=panel.value==='quick';document.querySelector('h1').textContent=quick?`AUTOJEV / QUICK ${cohort.value.toUpperCase()} DIAGNOSTIC`:cohort.value==='guard'?'AUTOJEV / FULL SYNTHETIC GUARD':'AUTOJEV / SUPERVISED LEARNING';document.querySelector('#panel-note').textContent=quick?`Quick diagnostic: 1,000 public / 200 guard / 700 temperature-fit rows · plotted: ${cohort.value} · never selects checkpoints`:`Full evaluations · plotted: ${cohort.value}${cohort.value==='guard'?' · separate synthetic guard cohort':''}`;document.querySelector('#report-note').textContent=quick?'Every point is a recorded quick evaluation. Jev is evaluated on the same fixed subset. Temperature is fitted separately on the 700-row diagnostic fold. These measurements never select checkpoints and are never connected to full-evaluation curves.':fullNote;
const history=historyRows(visible),showHistory=history.some(row=>row.status.includes('failed')||row.detail);document.querySelector('#history').hidden=!showHistory;const historyList=document.querySelector('#history-list');historyList.replaceChildren();for(const row of history){const title=document.createElement('strong');title.textContent=`${row.run} · update ${row.step} · ${row.status}`;if(row.status.includes('failed'))title.className='failed';const detail=document.createElement('p');detail.textContent=[row.detail,row.recovery].filter(Boolean).join(' ');historyList.append(title,detail)}
const tasks=document.querySelector('#tasks');tasks.replaceChildren();for(const event of visible.filter(e=>!['training_step','resume_saved'].includes(e.kind)).slice(showHistory?-3:-7)){const li=document.createElement('li');li.textContent=label(event);if(event.kind==='training_failed')li.className='failed';tasks.append(li)}const losses=steps.map(e=>[e.step,e.loss]),smoothed=[];for(const part of split(losses))for(let i=0;i<part.length;i++){const tail=part.slice(Math.max(0,i-9),i+1);smoothed.push([part[i][0],tail.reduce((s,p)=>s+p[1],0)/tail.length])}chart('loss',[{points:losses,color:'#c7dccb',width:1.5},{points:smoothed,color:'#286b4d'}],[0,1.1*Math.max(1,...all.filter(e=>e.kind==='training_step').map(e=>e.loss))],null,0,xmax);
for(const metric of ['accuracy','ece','brier']){const reference=evals.length&&Number.isFinite(evals[0].jev?.[metric])?evals[0].jev[metric]:null,values=allEvals.map(e=>e[mode.value][metric]);if(reference!==null)values.push(reference);const range=metric==='accuracy'?[Math.max(0,(values.length?Math.min(...values):.5)-.045),Math.min(1,(values.length?Math.max(...values):.95)+.035)]:[0,1.2*Math.max(.01,...values)+.01];chart(metric,[{points:evals.map(e=>[e.step,e[mode.value][metric]]),color:'#286b4d',dots:true}],range,reference,metric==='accuracy'?0:.01,xmax,metric==='accuracy');document.getElementById(metric+'-value').textContent=evals.length?(metric==='accuracy'?(evals.at(-1)[mode.value][metric]*100).toFixed(2)+'%':evals.at(-1)[mode.value][metric].toFixed(4)):''}const body=document.querySelector('#table tbody');body.replaceChildren();for(const e of evals){const tr=document.createElement('tr');for(const value of [e.step,(e[mode.value].accuracy*100).toFixed(2)+'%',e[mode.value].ece.toFixed(4),e[mode.value].brier.toFixed(4),e.temperature.toFixed(4),panel.value==='quick'?'diagnostic only':e.eligible?'yes':'no']){const td=document.createElement('td');td.textContent=String(value);tr.append(td)}body.append(tr)}
const final=panel.value==='full'&&cohort.value==='public'?current.filter(e=>e.kind==='final_comparison').at(-1):null;document.querySelector('#final').hidden=!final;if(final){document.querySelector('#final-status').textContent=`Frozen ${final.run}, update ${final.selected_step??'unrecorded'}, temperature ${Number(final.temperature).toFixed(4)}. ${final.count.toLocaleString()} fixed examples. Observed target ${final.empirical_target_met?'met':'not met'}; paired statistical support ${final.statistically_supported?'established':'not established'}. This table always uses the deployed temperature-fitted probabilities; the raw/fitted control affects only development curves.`;const tbody=document.querySelector('#final-table tbody');tbody.replaceChildren();for(const metric of ['accuracy','ece','brier']){const e=final.effects[metric],s=metric==='accuracy'?100:1,unit=metric==='accuracy'?' pp':'',suffix=metric==='accuracy'?'%':'';const tr=document.createElement('tr');for(const value of [metric,(e.local*s).toFixed(4)+suffix,(e.reference*s).toFixed(4)+suffix,(e.difference*s).toFixed(4)+unit,e.difference_ci95.map(x=>(x*s).toFixed(4)).join(' to ')+unit]){const td=document.createElement('td');td.textContent=String(value);tr.append(td)}tbody.append(tr)}}}
let timer=null;document.querySelector('#play').onclick=()=>{if(timer){clearInterval(timer);timer=null;document.querySelector('#play').textContent='Play'}else{if(+slider.value>=+slider.max)slider.value=0;document.querySelector('#play').textContent='Pause';timer=setInterval(()=>{slider.value=+slider.value+1;render();if(+slider.value>=+slider.max){clearInterval(timer);timer=null;document.querySelector('#play').textContent='Play'}},Math.max(60,60000/events.length))}};slider.oninput=render;selector.onchange=render;mode.onchange=render;panel.onchange=render;cohort.onchange=render;addEventListener('resize',render);render();
async function refreshLive(){try{const response=await fetch('live.json',{cache:'no-store'});if(!response.ok)return;const updated=await response.json();if(JSON.stringify(updated)===JSON.stringify(data))return;const selected=selector.value,position=+slider.value,follow=position===+slider.max&&!timer;data=updated;events=data.events;refreshRuns(selected);slider.max=events.length-1;slider.value=follow?slider.max:Math.min(position,+slider.max);render()}catch(error){document.querySelector('#status').title='Live refresh unavailable; showing the last recorded snapshot'}}
if(data.watch&&location.protocol.startsWith('http'))setInterval(refreshLive,10000);
</script></html>'''


def film(series: Series, output: Path, *, duration: int, fps: int, width: int, height: int, all_runs: bool = False) -> None:
    executable = cast(Callable[[], str], getattr(importlib.import_module("imageio_ffmpeg"), "get_ffmpeg_exe"))()
    times = [timestamp(event) for event in series["events"]]
    selected_times = [timestamp(event) for event in series["events"] if
                      (event["kind"] == "training_started" if all_runs else event.get("run") == series["run"])]
    beginning, end = times[0], times[-1]
    training_start = selected_times[0] if selected_times else beginning
    frames = duration * fps
    command = [executable, "-y", "-f", "rawvideo", "-vcodec", "rawvideo", "-s", f"{width}x{height}",
               "-pix_fmt", "rgb24", "-r", str(fps), "-i", "-", "-an", "-vcodec", "libx264", "-pix_fmt", "yuv420p",
               "-preset", "veryfast", "-crf", "20", "-movflags", "+faststart", str(output)]
    with output.with_suffix(".ffmpeg.log").open("w") as log, subprocess.Popen(command, stdin=subprocess.PIPE, stderr=log) as writer:
        assert writer.stdin is not None
        try:
            for index in range(frames):
                fraction = index / max(1, frames - 1)
                if fraction < 0.15 and training_start > beginning:
                    cutoff = beginning + fraction / 0.15 * (training_start - beginning)
                else:
                    progress = min(1, max(0, (fraction - 0.15) / 0.80)) if training_start > beginning else min(1, fraction / 0.95)
                    cutoff = training_start + progress * (end - training_start)
                current = active_series(series["events"], cutoff, series["panel"], series["cohort"]) if all_runs else series
                figure = draw(current, cutoff, width=width, height=height, all_runs=all_runs)
                canvas = FigureCanvasAgg(figure)
                cast(Callable[[], None], canvas.draw)()
                frame = np.asarray(cast(Callable[[], memoryview], canvas.buffer_rgba)())[:, :, :3].copy()
                writer.stdin.write(frame.tobytes())
                plt.close(figure)
                if index % (fps * 5) == 0:
                    print(f"Rendered {index}/{frames} frames", flush=True)
        finally:
            writer.stdin.close()
        status = writer.wait()
        if status:
            raise RuntimeError(f"Video encoder exited {status}; see {output.with_suffix('.ffmpeg.log')}")


class Arguments(argparse.Namespace):
    events: Path
    output: Path
    run: str | None
    panel: Panel
    cohort: Cohort
    video: bool
    all_runs: bool
    watch: bool
    duration: int
    fps: int
    width: int
    height: int


def write_report(args: Arguments, source_bytes: bytes) -> None:
    events = parse_events(source_bytes.decode())
    series = active_series(events, timestamp(events[-1]), args.panel, args.cohort) if args.all_runs else prepare(events, args.run, args.panel, args.cohort)
    args.output.mkdir(parents=True, exist_ok=True)
    dashboard(series, args.output / "index.html", watch=args.watch)
    figure = draw(series, timestamp(events[-1]), width=args.width, height=args.height, all_runs=args.all_runs)
    figure.savefig(args.output / "training.png", dpi=120)
    svg = args.output / "training.svg"
    figure.savefig(svg)
    svg.write_text("\n".join(line.rstrip(" \t") for line in svg.read_text().splitlines()) + "\n")
    plt.close(figure)
    provenance = {"events": str(args.events), "events_sha256": hashlib.sha256(source_bytes).hexdigest(), "events_bytes": len(source_bytes),
                  "renderer_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "run": series["run"],
                  "panel": series["panel"], "cohort": series["cohort"],
                  "event_count": len(events), "last_event": events[-1]["timestamp"],
                  "video": args.video, "duration_seconds": args.duration if args.video else None,
                  "frames_per_second": args.fps if args.video else None,
                  "all_runs": args.all_runs,
                  "replay": "15% research, 80% chronological run history, 5% final hold; each run has separate curves" if args.all_runs else
                            "15% research, 80% selected-run time, 5% final hold; only recorded events are revealed"}
    if args.video:
        film(series, args.output / "autojev.mp4", duration=args.duration, fps=args.fps, width=args.width, height=args.height, all_runs=args.all_runs)
    (args.output / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")
    print(f"Report: {args.output / 'index.html'}", flush=True)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--events", type=Path, default=Path("progress/events.jsonl"))
    parser.add_argument("--output", type=Path, default=Path("progress/report"))
    parser.add_argument("--run", help="Show one recorded training run; defaults to the latest")
    parser.add_argument("--panel", choices=("full", "quick"), default="full", help="Keep full selection evaluations and quick diagnostics in separate plots")
    parser.add_argument("--cohort", choices=("public", "guard"), default="public", help="Use this cohort's own measurements and matching Jev reference")
    parser.add_argument("--video", action="store_true")
    parser.add_argument("--all-runs", action="store_true", help="Video follows recorded run starts and returns to the frozen selected run at final comparison")
    parser.add_argument("--watch", action="store_true", help="Refresh the report every 10 seconds when recorded events change")
    parser.add_argument("--duration", type=int, default=60)
    parser.add_argument("--fps", type=int, default=20)
    parser.add_argument("--width", type=int, default=1920)
    parser.add_argument("--height", type=int, default=1080)
    args = parser.parse_args(namespace=Arguments())
    if min(args.duration, args.fps) < 1 or args.width < 960 or args.height < 540 or args.width % 2 or args.height % 2:
        parser.error("Use positive duration/fps and even dimensions of at least 960×540")
    if args.watch and args.video:
        parser.error("Use --watch for the live dashboard or --video for a fixed replay")
    if args.all_runs and (not args.video or args.run is not None):
        parser.error("Use --all-runs with --video and without --run")
    previous: bytes | None = None
    while True:
        source_bytes = args.events.read_bytes()
        if source_bytes != previous:
            write_report(args, source_bytes)
            previous = source_bytes
        if not args.watch:
            break
        time.sleep(10)


if __name__ == "__main__":
    main()