WitneyWW's picture
1116 TODO: pushT/rope val cases + val inference at epochs 14/15 (infer_val.sh), README
fc95770 verified
Raw History Blame Contribute Delete
4.63 kB
"""Aggregate the per-case results of eval/run_test_case.py into a markdown table, optionally
paired against the reference numbers shipped in eval/cases/<task>_reference_metrics.json.
python eval/summarize.py --task rope [--root DIR] [--reference]
python eval/summarize.py --task rope --split val --root runs/<run>/val_epoch_014 \
--ref-root runs/nano_action_causal_rope_v1/val_epoch_014 # paired vs another run on the same val cases
short = 8 observed -> 8 predicted frames (metrics on the 8 predicted frames);
long = mean over the 15 autoregressive windows (120 generated frames).
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
METRICS = ["psnr", "ssim", "lpips"]
def load(root: Path, man: dict) -> dict:
out = {}
for c in man["cases"]:
d = root / c["output_subdir"]
if not (d / "complete.json").is_file():
continue
w = json.loads((d / "window_eval.json").read_text())["results"][0]
r = {"category": c["category"], "short": {s: {m: w["image"][s][m] for m in METRICS} for s in STREAMS}}
rr = d / "reference_rollout.json"
if c["rollout_windows"] and rr.is_file():
wins = json.loads(rr.read_text())[0]["windows"]
r["long_mean"] = {s: {m: sum(x["image"][s][m] for x in wins) / len(wins) for m in METRICS} for s in STREAMS}
out[c["output_subdir"]] = r
return out
def mean(xs):
return sum(xs) / len(xs) if xs else float("nan")
def table(res: dict, hz: str, title: str) -> list[str]:
rows = [r for r in res.values() if hz in r]
lines = [f"### {title} (n={len(rows)})", "", "| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ |", "|---|---:|---:|---:|"]
for s in STREAMS:
v = [mean([r[hz][s][m] for r in rows]) for m in METRICS]
lines.append(f"| {s} | {v[0]:.2f} | {v[1]:.4f} | {v[2]:.4f} |")
return lines + [""]
def main():
ap = argparse.ArgumentParser()
tasks = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
ap.add_argument("--task", required=True, choices=tasks)
ap.add_argument("--root", default=None, help="default: $ACTS_RUNS_ROOT/<run_name>/test_<ckpt stem of the manifest>")
ap.add_argument("--split", default="test", choices=["test", "val"])
ap.add_argument("--reference", action="store_true",
help="also print paired deltas vs the shipped reference metrics (test split only)")
ap.add_argument("--ref-root", default=None,
help="also print paired deltas vs another result dir on the same cases (e.g. the parent run's "
"val_epoch_014); use this for the val split")
a = ap.parse_args()
man = json.loads((REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json").read_text())
runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
root = Path(a.root or runs / man["run_name"] / f"{a.split}_{Path(man['checkpoint']).stem}")
res = load(root, man)
print(f"# {root.parent.name} {root.name} ({len(res)}/{len(man['cases'])} cases complete) -- {root}\n")
print("\n".join(table(res, "short", "short horizon: 8 -> 8 frames")))
print("\n".join(table(res, "long_mean", "long horizon: mean over 15 chained windows")))
refs = []
if a.reference:
if a.split != "test":
raise SystemExit("--reference is the shipped TEST reference; for val use --ref-root")
ref_file = REPO / "eval" / "cases" / f"{a.task}_reference_metrics.json"
if not ref_file.is_file():
raise SystemExit(f"no reference metrics for {a.task}: {ref_file}")
refs.append(("reference", {r["output_subdir"]: r for r in json.loads(ref_file.read_text())["cases"]}))
if a.ref_root:
rr = Path(a.ref_root)
refs.append((f"{rr.parent.name}/{rr.name}", load(rr, man)))
for name, ref in refs:
print(f"### paired difference vs {name} (this run − {name}), PSNR dB, n = {len(ref)} reference cases\n")
print("| horizon | stream | n | mean Δ | max |Δ| |\n|---|---|---:|---:|---:|")
for hz in ("short", "long_mean"):
for s in STREAMS:
d = [res[k][hz][s]["psnr"] - ref[k][hz][s]["psnr"] for k in res if hz in res[k] and hz in ref.get(k, {})]
if d:
print(f"| {hz} | {s} | {len(d)} | {mean(d):+.3f} | {max(abs(x) for x in d):.3f} |")
print()
if __name__ == "__main__":
main()