#!/usr/bin/env python3 """Generate RESULTS.md from the evaluation artifacts. Every number in the document comes from a JSON file written by another script, so the documented results and the measured results cannot drift. Missing inputs are skipped with a note rather than guessed at. """ from __future__ import annotations import argparse import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] def read(path: Path): try: return json.loads(path.read_text()) except (OSError, json.JSONDecodeError): return None def training_section(run: Path, has_probe: bool = True) -> list[str]: history = read(run / "history.json") if not history: return [] out = ["## Training curve", "", "Frozen validation split, never sampled during training.", "", "`conv field R²` scores `supervised_field()` -- the conv head the " "field loss trains. It is **not** the field the controller " "descends. Under `--min-plus-field` those are different objects: " "the read-out is the min-plus recurrence" + (", scored as `read-out R²` in the probe table above" if has_probe else "") + ", and nothing at inference reads the conv " "head at all. It stays in the loss as an auxiliary task on the " "shared trunk, so a negative value here means that auxiliary head " "has stopped tracking the trunk the action head and the recurrence " "are shaping -- it does not mean the planner is wrong, and `reach` " "is where that would show up.", "", "| step | loss | acc | choice | boolean | score | conv field R² | maze action | snake action |", "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"] for row in history: out.append( f"| {row['step']} | {row['loss']:.4f} | {row['acc']:.4f} | " f"{row.get('acc_choice', float('nan')):.4f} | " f"{row.get('acc_boolean', float('nan')):.4f} | " f"{row.get('acc_score', float('nan')):.4f} | " f"{row.get('field_r2', float('nan')):.4f} | " f"{row.get('maze_action', float('nan')):.4f} | " f"{row.get('snake_action', float('nan')):.4f} |") return out + [""] def eval_section(data) -> list[str]: if not data: return [] out = ["## Question-level accuracy", "", "`uniform` is the accuracy of guessing uniformly over the same " "candidates. A model that matches its control has learned nothing, " "which is the failure NanoJev's own log records for its maze model.", "", "`boards` is how many distinct boards the row's questions came " "from, and it is there so that `n` can be trusted. Questions that " "share a board are not independent tests of whether the model can " "read a board -- they share walls, goal and distance field, so " "they succeed or fail together -- and a row reporting `n` alone " "cannot tell you whether that is happening. Here it is not: the " "generator draws a fresh board per state, so `boards` tracks `n` " "on every split and `n` is the honest denominator. A row where " "the two diverge is one whose precision should be read off " "`boards`.", ""] for split, block in data.items(): out += [f"### {split} — {block['questions']} questions, ECE {block['ece']:.4f}", "", "| group | n | boards | accuracy | uniform control | TVD | Brier |", "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"] for name, row in block["groups"].items(): boards = row.get("boards") out.append(f"| `{name}` | {row['n']} | " f"{'--' if boards is None else boards} | " f"**{row['accuracy']:.4f}** | " f"{row['uniform_control']:.4f} | {row['tvd']:.4f} | " f"{row['brier']:.4f} |") out.append("") return out def probe_section(data) -> list[str]: if not data: return [] out = ["## Does the planner actually compute distances?", "", "Ridge probe from the planner's per-cell features to true BFS " "distance, on held-out mazes. `diameter` is the longest true " "shortest-path in those boards; `T` is the iteration budget.", "", "`descent` is the fraction of cells whose lowest-predicted " "neighbour lies on a true shortest path. `reach` is the fraction " "from which greedy descent actually arrives at the goal, and it " "is the one that predicts solve rate: a field can be right 70% of " "the time per step and still trap most walks in a spurious basin, " "so `descent` is an upper bound on nothing in particular. R² " "scores absolute values; only `reach` scores the behaviour.", "", "`read-out R²` scores `predict_field()`, the field this " "controller actually descends -- the min-plus recurrence when " "the run enabled it, the conv head otherwise. The training " "curve's `conv field R²` scores a different head; see the " "note there.", "", "| size | cells | probe R² | probe MAE (cells) | read-out R² | descent | reach | diameter | T |", "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"] for size, row in data.items(): out.append(f"| {size} | {row['cells']} | **{row['probe_r2']:.4f}** | " f"{row['probe_mae']:.2f} | {row['head_r2']:.4f} | " f"{row.get('descent_acc', float('nan')):.4f} | " f"**{row.get('reach_rate', float('nan')):.4f}** | " f"{row['diameter']} | {row['iterations']} |") return out + [""] # Reading order, not alphabetical order. Sorting by filename puts # `maze_random_memory` three rows below the `maze_model_memory` it is the # control for, and `model+memory` above the plain `model` it is scaffolding # over. Anything not listed here still renders, at the end, so adding a play # config can never silently drop it from the table. PLAY_ORDER = ["maze_model", "maze_model_sampled", "maze_model_field", "maze_model_memory", "maze_random_memory", "maze_reference", "maze_random", "snake_model", "snake_reference", "snake_random"] def play_section(summaries: dict) -> list[str]: maze, snake = [], [] ordered = sorted(summaries.items(), key=lambda kv: (PLAY_ORDER.index(kv[0]) if kv[0] in PLAY_ORDER else len(PLAY_ORDER), kv[0])) for label, data in ordered: block = data.get("summary", data) if data else {} if "maze" in block: m = block["maze"] steps = m.get("mean_steps_when_solved") dead = m.get("deadlock_rate") maze.append(f"| `{label}` | {m['solve_rate']:.2f} | " f"{'--' if steps is None else format(steps, '.1f')} | " f"{m['mean_efficiency']:.3f} | {m['mean_collisions']:.1f} | " f"{'--' if dead is None else format(dead, '.2f')} | " f"{m.get('mean_distinct_cells', float('nan')):.1f} |") if "snake" in block: s = block["snake"] snake.append(f"| `{label}` | {s['mean_food']:.2f} | {s['max_food']} | " f"{s['mean_steps']:.1f} | {s['survival_rate']:.2f} |") out = [] if maze: out += ["## Closed-loop play", "", "Read `model` and `model-field` as answering different " "questions. `model` is the network's action head alone -- no " "search, no memory, no visited set -- and is the " "apples-to-apples comparison with NanoJev. `model-field` " "descends a min-plus recurrence whose fixed point is a " "shortest-path distance by construction, so it arrives from " "anywhere; for a maze it does so at initialisation, before " "any training, because uniform cost is already the right " "answer. Its solve rate is a property of the architecture, " "not a measurement of what this run learned.", "", "### Maze", "", "`deadlock` is the fraction of episodes in which the " "controller spent a tenth of its step budget walking into the " "same wall, and `cells` is how many distinct squares it stood " "on. They separate two failures a solve rate of 0.00 reports " "identically: standing still and touring the board. Deadlock " "should now read 0.00 for every controller, because only legal " "moves are offered as candidates -- it is kept as a regression " "guard, not a finding. The failure that remains is cycling: " "the planner never sees the agent, so a greedy policy that " "steps A->B->A has no state with which to notice, and `cells` " "is what exposes it.", "", "| controller | solve rate | mean steps | efficiency | collisions | deadlock | cells |", "| --- | ---: | ---: | ---: | ---: | ---: | ---: |"] + maze + [""] if snake: out += ["### Snake", "", "| controller | mean food | max food | mean steps | survival |", "| --- | ---: | ---: | ---: | ---: |"] + snake + [""] return out def nanojev_section() -> list[str]: """The numbers NanoJev reports for itself, quoted from its own repo. Transcribed from docs/DEVELOPMENT_RESULTS.md in TianyuCodings/NanoJev so the comparison is against what its authors measured, not a rerun of ours. The point of the table is the control column: where a reported accuracy equals the always-true rate, the number is reporting the class balance. """ return [ "## Baseline: what NanoJev reports for itself", "", "From `docs/DEVELOPMENT_RESULTS.md` in the NanoJev repository.", "", "| NanoJev task | reported acc | always-true control | above control |", "| --- | ---: | ---: | :---: |", "| `scaled_maze` (test) | 0.5625 | 0.5625 | no |", "| `scaled_maze` (ood) | 0.5625 | 0.5625 | no |", "| `snake_one_step_safety_v4` | 0.9220 | 0.9220 | no |", "", "Closed-loop play, same source, 128-step limit:", "", "| maze | Jev | NanoJev | reference |", "| --- | ---: | ---: | ---: |", "| all three mazes | 0/128 | 0/128 | solved 1/16, 1/24, 1/96 |", "", "> \"The learned direct-action models and Jev do not solve any of these", "> three mazes within the shared 128-step limit.\"", "", "Every accuracy in this file is therefore printed next to its", "constant-prediction control, and gameplay is reported as solve rate", "rather than per-question accuracy.", "", ] def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--run", default="runs/jevon-final") p.add_argument("--eval", default="runs/jevon-final/eval.json") p.add_argument("--probe", default="runs/jevon-final/probe.json") p.add_argument("--play-dir", default="runs/jevon-final/play") p.add_argument("--out", default="RESULTS.md") args = p.parse_args() summaries = {} play_dir = Path(args.play_dir) if play_dir.is_dir(): for path in sorted(play_dir.glob("*_summary.json")): summaries[path.stem.replace("_summary", "")] = read(path) lines = ["# Results", "", "Generated by `scripts/make_results.py` from the JSON written by " "`train.py`, `evaluate.py`, `probe.py` and `play.py`. Do not edit " "by hand.", ""] lines += nanojev_section() probe = read(Path(args.probe)) lines += probe_section(probe) lines += eval_section(read(Path(args.eval))) lines += play_section(summaries) lines += training_section(Path(args.run), has_probe=bool(probe)) Path(args.out).write_text("\n".join(lines)) print(f"wrote {args.out} ({len(lines)} lines)") if __name__ == "__main__": main()