scripts/make_results.py

#!/usr/bin/env python3
"""Generate RESULTS.md from the evaluation artifacts.

Every number in the document comes from a JSON file written by another
script, so the documented results and the measured results cannot drift.
Missing inputs are skipped with a note rather than guessed at.
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]


def read(path: Path):
    try:
        return json.loads(path.read_text())
    except (OSError, json.JSONDecodeError):
        return None


def training_section(run: Path, has_probe: bool = True) -> list[str]:
    history = read(run / "history.json")
    if not history:
        return []
    out = ["## Training curve", "",
           "Frozen validation split, never sampled during training.", "",
           "`conv field R²` scores `supervised_field()` -- the conv head the "
           "field loss trains. It is **not** the field the controller "
           "descends. Under `--min-plus-field` those are different objects: "
           "the read-out is the min-plus recurrence" +
           (", scored as `read-out R²` in the probe table above" if has_probe
            else "") +
           ", and nothing at inference reads the conv "
           "head at all. It stays in the loss as an auxiliary task on the "
           "shared trunk, so a negative value here means that auxiliary head "
           "has stopped tracking the trunk the action head and the recurrence "
           "are shaping -- it does not mean the planner is wrong, and `reach` "
           "is where that would show up.", "",
           "| step | loss | acc | choice | boolean | score | conv field R² | maze action | snake action |",
           "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
    for row in history:
        out.append(
            f"| {row['step']} | {row['loss']:.4f} | {row['acc']:.4f} | "
            f"{row.get('acc_choice', float('nan')):.4f} | "
            f"{row.get('acc_boolean', float('nan')):.4f} | "
            f"{row.get('acc_score', float('nan')):.4f} | "
            f"{row.get('field_r2', float('nan')):.4f} | "
            f"{row.get('maze_action', float('nan')):.4f} | "
            f"{row.get('snake_action', float('nan')):.4f} |")
    return out + [""]


def eval_section(data) -> list[str]:
    if not data:
        return []
    out = ["## Question-level accuracy", "",
           "`uniform` is the accuracy of guessing uniformly over the same "
           "candidates. A model that matches its control has learned nothing, "
           "which is the failure NanoJev's own log records for its maze model.", "",
           "`boards` is how many distinct boards the row's questions came "
           "from, and it is there so that `n` can be trusted. Questions that "
           "share a board are not independent tests of whether the model can "
           "read a board -- they share walls, goal and distance field, so "
           "they succeed or fail together -- and a row reporting `n` alone "
           "cannot tell you whether that is happening. Here it is not: the "
           "generator draws a fresh board per state, so `boards` tracks `n` "
           "on every split and `n` is the honest denominator. A row where "
           "the two diverge is one whose precision should be read off "
           "`boards`.", ""]
    for split, block in data.items():
        out += [f"### {split} — {block['questions']} questions, ECE {block['ece']:.4f}", "",
                "| group | n | boards | accuracy | uniform control | TVD | Brier |",
                "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"]
        for name, row in block["groups"].items():
            boards = row.get("boards")
            out.append(f"| `{name}` | {row['n']} | "
                       f"{'--' if boards is None else boards} | "
                       f"**{row['accuracy']:.4f}** | "
                       f"{row['uniform_control']:.4f} | {row['tvd']:.4f} | "
                       f"{row['brier']:.4f} |")
        out.append("")
    return out


def probe_section(data) -> list[str]:
    if not data:
        return []
    out = ["## Does the planner actually compute distances?", "",
           "Ridge probe from the planner's per-cell features to true BFS "
           "distance, on held-out mazes. `diameter` is the longest true "
           "shortest-path in those boards; `T` is the iteration budget.", "",
           "`descent` is the fraction of cells whose lowest-predicted "
           "neighbour lies on a true shortest path. `reach` is the fraction "
           "from which greedy descent actually arrives at the goal, and it "
           "is the one that predicts solve rate: a field can be right 70% of "
           "the time per step and still trap most walks in a spurious basin, "
           "so `descent` is an upper bound on nothing in particular. R² "
           "scores absolute values; only `reach` scores the behaviour.", "",
           "`read-out R²` scores `predict_field()`, the field this "
           "controller actually descends -- the min-plus recurrence when "
           "the run enabled it, the conv head otherwise. The training "
           "curve's `conv field R²` scores a different head; see the "
           "note there.", "",
           "| size | cells | probe R² | probe MAE (cells) | read-out R² | descent | reach | diameter | T |",
           "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
    for size, row in data.items():
        out.append(f"| {size} | {row['cells']} | **{row['probe_r2']:.4f}** | "
                   f"{row['probe_mae']:.2f} | {row['head_r2']:.4f} | "
                   f"{row.get('descent_acc', float('nan')):.4f} | "
                   f"**{row.get('reach_rate', float('nan')):.4f}** | "
                   f"{row['diameter']} | {row['iterations']} |")
    return out + [""]


# Reading order, not alphabetical order. Sorting by filename puts
# `maze_random_memory` three rows below the `maze_model_memory` it is the
# control for, and `model+memory` above the plain `model` it is scaffolding
# over. Anything not listed here still renders, at the end, so adding a play
# config can never silently drop it from the table.
PLAY_ORDER = ["maze_model", "maze_model_sampled", "maze_model_field",
              "maze_model_memory", "maze_random_memory", "maze_reference",
              "maze_random", "snake_model", "snake_reference", "snake_random"]


def play_section(summaries: dict) -> list[str]:
    maze, snake = [], []
    ordered = sorted(summaries.items(),
                     key=lambda kv: (PLAY_ORDER.index(kv[0])
                                     if kv[0] in PLAY_ORDER else len(PLAY_ORDER),
                                     kv[0]))
    for label, data in ordered:
        block = data.get("summary", data) if data else {}
        if "maze" in block:
            m = block["maze"]
            steps = m.get("mean_steps_when_solved")
            dead = m.get("deadlock_rate")
            maze.append(f"| `{label}` | {m['solve_rate']:.2f} | "
                        f"{'--' if steps is None else format(steps, '.1f')} | "
                        f"{m['mean_efficiency']:.3f} | {m['mean_collisions']:.1f} | "
                        f"{'--' if dead is None else format(dead, '.2f')} | "
                        f"{m.get('mean_distinct_cells', float('nan')):.1f} |")
        if "snake" in block:
            s = block["snake"]
            snake.append(f"| `{label}` | {s['mean_food']:.2f} | {s['max_food']} | "
                         f"{s['mean_steps']:.1f} | {s['survival_rate']:.2f} |")
    out = []
    if maze:
        out += ["## Closed-loop play", "",
                "Read `model` and `model-field` as answering different "
                "questions. `model` is the network's action head alone -- no "
                "search, no memory, no visited set -- and is the "
                "apples-to-apples comparison with NanoJev. `model-field` "
                "descends a min-plus recurrence whose fixed point is a "
                "shortest-path distance by construction, so it arrives from "
                "anywhere; for a maze it does so at initialisation, before "
                "any training, because uniform cost is already the right "
                "answer. Its solve rate is a property of the architecture, "
                "not a measurement of what this run learned.", "",
                "### Maze", "",
                "`deadlock` is the fraction of episodes in which the "
                "controller spent a tenth of its step budget walking into the "
                "same wall, and `cells` is how many distinct squares it stood "
                "on. They separate two failures a solve rate of 0.00 reports "
                "identically: standing still and touring the board. Deadlock "
                "should now read 0.00 for every controller, because only legal "
                "moves are offered as candidates -- it is kept as a regression "
                "guard, not a finding. The failure that remains is cycling: "
                "the planner never sees the agent, so a greedy policy that "
                "steps A->B->A has no state with which to notice, and `cells` "
                "is what exposes it.", "",
                "| controller | solve rate | mean steps | efficiency | collisions | deadlock | cells |",
                "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"] + maze + [""]
    if snake:
        out += ["### Snake", "",
                "| controller | mean food | max food | mean steps | survival |",
                "| --- | ---: | ---: | ---: | ---: |"] + snake + [""]
    return out


def nanojev_section() -> list[str]:
    """The numbers NanoJev reports for itself, quoted from its own repo.

    Transcribed from docs/DEVELOPMENT_RESULTS.md in TianyuCodings/NanoJev so
    the comparison is against what its authors measured, not a rerun of ours.
    The point of the table is the control column: where a reported accuracy
    equals the always-true rate, the number is reporting the class balance.
    """
    return [
        "## Baseline: what NanoJev reports for itself", "",
        "From `docs/DEVELOPMENT_RESULTS.md` in the NanoJev repository.", "",
        "| NanoJev task | reported acc | always-true control | above control |",
        "| --- | ---: | ---: | :---: |",
        "| `scaled_maze` (test) | 0.5625 | 0.5625 | no |",
        "| `scaled_maze` (ood) | 0.5625 | 0.5625 | no |",
        "| `snake_one_step_safety_v4` | 0.9220 | 0.9220 | no |",
        "",
        "Closed-loop play, same source, 128-step limit:", "",
        "| maze | Jev | NanoJev | reference |",
        "| --- | ---: | ---: | ---: |",
        "| all three mazes | 0/128 | 0/128 | solved 1/16, 1/24, 1/96 |",
        "",
        "> \"The learned direct-action models and Jev do not solve any of these",
        "> three mazes within the shared 128-step limit.\"", "",
        "Every accuracy in this file is therefore printed next to its",
        "constant-prediction control, and gameplay is reported as solve rate",
        "rather than per-question accuracy.", "",
    ]


def main() -> None:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--run", default="runs/jevon-final")
    p.add_argument("--eval", default="runs/jevon-final/eval.json")
    p.add_argument("--probe", default="runs/jevon-final/probe.json")
    p.add_argument("--play-dir", default="runs/jevon-final/play")
    p.add_argument("--out", default="RESULTS.md")
    args = p.parse_args()

    summaries = {}
    play_dir = Path(args.play_dir)
    if play_dir.is_dir():
        for path in sorted(play_dir.glob("*_summary.json")):
            summaries[path.stem.replace("_summary", "")] = read(path)

    lines = ["# Results", "",
             "Generated by `scripts/make_results.py` from the JSON written by "
             "`train.py`, `evaluate.py`, `probe.py` and `play.py`. Do not edit "
             "by hand.", ""]
    lines += nanojev_section()
    probe = read(Path(args.probe))
    lines += probe_section(probe)
    lines += eval_section(read(Path(args.eval)))
    lines += play_section(summaries)
    lines += training_section(Path(args.run), has_probe=bool(probe))

    Path(args.out).write_text("\n".join(lines))
    print(f"wrote {args.out} ({len(lines)} lines)")


if __name__ == "__main__":
    main()