File size: 8,466 Bytes
b1b6f13
 
 
 
 
 
fc95770
b1b6f13
fc95770
 
 
b1b6f13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fc95770
 
b1b6f13
fc95770
 
 
 
 
 
 
 
b1b6f13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fc95770
 
 
b1b6f13
 
fc95770
b1b6f13
 
 
 
 
fc95770
 
b1b6f13
 
 
fc95770
b1b6f13
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render
GT/prediction videos for the five image streams.

    python eval/run_test_case.py --task rope --case 0            # one case
    python eval/run_test_case.py --task rope --case all          # every case, sequentially
    python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt
    python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt

The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the
exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py).
Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json,
strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py.
Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT)."""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
from pathlib import Path

import imageio.v2 as imageio
import numpy as np
from PIL import Image, ImageDraw

REPO = Path(__file__).resolve().parents[1]
STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
# data tree from the split file of the case manifest
TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope",
                 "mixed_pusht_wrist_holdout": "mixed_pushT"}


def default_out_root(ckpt: Path, split: str = "test") -> Path:
    """$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014"""
    runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
    return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}"


def case_file(task: str, split: str) -> Path:
    f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json"
    if not f.is_file():
        raise SystemExit(f"no {split} cases for task {task}: {f}")
    return f


def env(name: str) -> str:
    v = os.environ.get(name)
    if not v:
        raise SystemExit(f"${name} is not set -- run `source env.sh` first")
    return v


def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool):
    out = out_root / case["output_subdir"]
    if skip_done and (out / "complete.json").is_file():
        print("[skip] done:", out, flush=True)
        return
    out.mkdir(parents=True, exist_ok=True)
    (out / "case.json").write_text(json.dumps(case, indent=2))
    print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True)
    data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT")
    tree = TREE_OF_SPLIT[Path(man["split_file"]).stem]
    overrides = [
        f"data.latent_root={data}/{tree}_latents",
        f"data.action_root={data}/{tree}_actions",
        f"data.split_file={REPO / man['split_file']}",
        f"model.nano_checkpoint={ckroot}/Cosmos3-Nano",
        f"out_dir={out_root}",
    ]
    subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout",
                    "--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1",
                    "--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]),
                    "--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)),
                    "--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO)

    w = json.loads((out / "window_eval.json").read_text())["results"][0]
    if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])):
        raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, "
                           f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)")

    # Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once,
    # then append each rollout window's 8 future frames.
    stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}"
    if case["rollout_windows"]:
        rolls = json.loads((out / "reference_rollout.json").read_text())
        assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout"
        assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early"
    for mode in (("short", "long") if case["rollout_windows"] else ("short",)):
        video_path = out / f"{mode}_gt_pred.mp4"
        count = 0
        with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer:
            for k in range(1 if mode == "short" else case["rollout_windows"]):
                suffix = "" if mode == "short" else f"_roll{k}"
                strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS]
                h = (strips[0].shape[0] - 4) // 2
                wd = strips[0].shape[1] // 16 - 2
                for t in range(0 if k == 0 else 8, 16):
                    canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black")
                    draw = ImageDraw.Draw(canvas)
                    for col, (stream, strip) in enumerate(zip(STREAMS, strips)):
                        x = t * (wd + 2)
                        canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32))
                        canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64))
                        draw.text((col * wd + 4, 16), "GT: " + stream, fill="white")
                        draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white")
                    label = "observed history" if count < 8 else "predicted future"
                    draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white")
                    writer.append_data(np.asarray(canvas))
                    count += 1
        print("[video]", video_path, "frames=", count, flush=True)
    (out / "complete.json").write_text(json.dumps(dict(
        short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0,
        rollout_windows=case["rollout_windows"])))
    print("[case done]", out, flush=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)")
    ap.add_argument("--split", default="test", choices=["test", "val"],
                    help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)")
    ap.add_argument("--case", required=True, help="case number, comma list, or 'all'")
    ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases")
    ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>")
    ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)")
    ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)")
    ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json")
    a = ap.parse_args()

    man = json.loads(case_file(a.task, a.split).read_text())
    assert man["split"] == a.split, (man["split"], a.split)
    ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"])
    if not ckpt.is_file():
        raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)")
    out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split)
    out_root.mkdir(parents=True, exist_ok=True)
    ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")]
    steps = a.steps or man.get("denoising_steps_per_latent_block", 35)
    for i in ids:
        case = dict(man["cases"][i])
        if a.short_only:
            case["rollout_windows"] = 0
        run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo)


if __name__ == "__main__":
    main()