File size: 4,634 Bytes
b1b6f13
 
 
 
fc95770
 
b1b6f13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fc95770
 
 
 
 
 
b1b6f13
fc95770
b1b6f13
fc95770
b1b6f13
 
 
 
fc95770
b1b6f13
fc95770
 
b1b6f13
 
 
fc95770
 
 
 
 
 
b1b6f13
 
 
 
 
 
fc95770
b1b6f13
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
"""Aggregate the per-case results of eval/run_test_case.py into a markdown table, optionally
paired against the reference numbers shipped in eval/cases/<task>_reference_metrics.json.

    python eval/summarize.py --task rope [--root DIR] [--reference]
    python eval/summarize.py --task rope --split val --root runs/<run>/val_epoch_014 \
        --ref-root runs/nano_action_causal_rope_v1/val_epoch_014        # paired vs another run on the same val cases

short = 8 observed -> 8 predicted frames (metrics on the 8 predicted frames);
long  = mean over the 15 autoregressive windows (120 generated frames).
"""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
METRICS = ["psnr", "ssim", "lpips"]


def load(root: Path, man: dict) -> dict:
    out = {}
    for c in man["cases"]:
        d = root / c["output_subdir"]
        if not (d / "complete.json").is_file():
            continue
        w = json.loads((d / "window_eval.json").read_text())["results"][0]
        r = {"category": c["category"], "short": {s: {m: w["image"][s][m] for m in METRICS} for s in STREAMS}}
        rr = d / "reference_rollout.json"
        if c["rollout_windows"] and rr.is_file():
            wins = json.loads(rr.read_text())[0]["windows"]
            r["long_mean"] = {s: {m: sum(x["image"][s][m] for x in wins) / len(wins) for m in METRICS} for s in STREAMS}
        out[c["output_subdir"]] = r
    return out


def mean(xs):
    return sum(xs) / len(xs) if xs else float("nan")


def table(res: dict, hz: str, title: str) -> list[str]:
    rows = [r for r in res.values() if hz in r]
    lines = [f"### {title} (n={len(rows)})", "", "| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ |", "|---|---:|---:|---:|"]
    for s in STREAMS:
        v = [mean([r[hz][s][m] for r in rows]) for m in METRICS]
        lines.append(f"| {s} | {v[0]:.2f} | {v[1]:.4f} | {v[2]:.4f} |")
    return lines + [""]


def main():
    ap = argparse.ArgumentParser()
    tasks = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
    ap.add_argument("--task", required=True, choices=tasks)
    ap.add_argument("--root", default=None, help="default: $ACTS_RUNS_ROOT/<run_name>/test_<ckpt stem of the manifest>")
    ap.add_argument("--split", default="test", choices=["test", "val"])
    ap.add_argument("--reference", action="store_true",
                    help="also print paired deltas vs the shipped reference metrics (test split only)")
    ap.add_argument("--ref-root", default=None,
                    help="also print paired deltas vs another result dir on the same cases (e.g. the parent run's "
                         "val_epoch_014); use this for the val split")
    a = ap.parse_args()
    man = json.loads((REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json").read_text())
    runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
    root = Path(a.root or runs / man["run_name"] / f"{a.split}_{Path(man['checkpoint']).stem}")
    res = load(root, man)
    print(f"# {root.parent.name} {root.name} ({len(res)}/{len(man['cases'])} cases complete) -- {root}\n")
    print("\n".join(table(res, "short", "short horizon: 8 -> 8 frames")))
    print("\n".join(table(res, "long_mean", "long horizon: mean over 15 chained windows")))
    refs = []
    if a.reference:
        if a.split != "test":
            raise SystemExit("--reference is the shipped TEST reference; for val use --ref-root")
        ref_file = REPO / "eval" / "cases" / f"{a.task}_reference_metrics.json"
        if not ref_file.is_file():
            raise SystemExit(f"no reference metrics for {a.task}: {ref_file}")
        refs.append(("reference", {r["output_subdir"]: r for r in json.loads(ref_file.read_text())["cases"]}))
    if a.ref_root:
        rr = Path(a.ref_root)
        refs.append((f"{rr.parent.name}/{rr.name}", load(rr, man)))
    for name, ref in refs:
        print(f"### paired difference vs {name} (this run − {name}), PSNR dB, n = {len(ref)} reference cases\n")
        print("| horizon | stream | n | mean Δ | max |Δ| |\n|---|---|---:|---:|---:|")
        for hz in ("short", "long_mean"):
            for s in STREAMS:
                d = [res[k][hz][s]["psnr"] - ref[k][hz][s]["psnr"] for k in res if hz in res[k] and hz in ref.get(k, {})]
                if d:
                    print(f"| {hz} | {s} | {len(d)} | {mean(d):+.3f} | {max(abs(x) for x in d):.3f} |")
        print()


if __name__ == "__main__":
    main()