acts-sensoronly-ablation / eval /run_test_case.py
WitneyWW's picture
1116 TODO: pushT/rope val cases + val inference at epochs 14/15 (infer_val.sh), README
fc95770 verified
Raw History Blame Contribute Delete
8.47 kB
"""Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render
GT/prediction videos for the five image streams.
python eval/run_test_case.py --task rope --case 0 # one case
python eval/run_test_case.py --task rope --case all # every case, sequentially
python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt
python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt
The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the
exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py).
Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json,
strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py.
Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT)."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
import imageio.v2 as imageio
import numpy as np
from PIL import Image, ImageDraw
REPO = Path(__file__).resolve().parents[1]
STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
# data tree from the split file of the case manifest
TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope",
"mixed_pusht_wrist_holdout": "mixed_pushT"}
def default_out_root(ckpt: Path, split: str = "test") -> Path:
"""$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014"""
runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}"
def case_file(task: str, split: str) -> Path:
f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json"
if not f.is_file():
raise SystemExit(f"no {split} cases for task {task}: {f}")
return f
def env(name: str) -> str:
v = os.environ.get(name)
if not v:
raise SystemExit(f"${name} is not set -- run `source env.sh` first")
return v
def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool):
out = out_root / case["output_subdir"]
if skip_done and (out / "complete.json").is_file():
print("[skip] done:", out, flush=True)
return
out.mkdir(parents=True, exist_ok=True)
(out / "case.json").write_text(json.dumps(case, indent=2))
print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True)
data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT")
tree = TREE_OF_SPLIT[Path(man["split_file"]).stem]
overrides = [
f"data.latent_root={data}/{tree}_latents",
f"data.action_root={data}/{tree}_actions",
f"data.split_file={REPO / man['split_file']}",
f"model.nano_checkpoint={ckroot}/Cosmos3-Nano",
f"out_dir={out_root}",
]
subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout",
"--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1",
"--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]),
"--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)),
"--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO)
w = json.loads((out / "window_eval.json").read_text())["results"][0]
if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])):
raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, "
f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)")
# Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once,
# then append each rollout window's 8 future frames.
stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}"
if case["rollout_windows"]:
rolls = json.loads((out / "reference_rollout.json").read_text())
assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout"
assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early"
for mode in (("short", "long") if case["rollout_windows"] else ("short",)):
video_path = out / f"{mode}_gt_pred.mp4"
count = 0
with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer:
for k in range(1 if mode == "short" else case["rollout_windows"]):
suffix = "" if mode == "short" else f"_roll{k}"
strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS]
h = (strips[0].shape[0] - 4) // 2
wd = strips[0].shape[1] // 16 - 2
for t in range(0 if k == 0 else 8, 16):
canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black")
draw = ImageDraw.Draw(canvas)
for col, (stream, strip) in enumerate(zip(STREAMS, strips)):
x = t * (wd + 2)
canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32))
canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64))
draw.text((col * wd + 4, 16), "GT: " + stream, fill="white")
draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white")
label = "observed history" if count < 8 else "predicted future"
draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white")
writer.append_data(np.asarray(canvas))
count += 1
print("[video]", video_path, "frames=", count, flush=True)
(out / "complete.json").write_text(json.dumps(dict(
short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0,
rollout_windows=case["rollout_windows"])))
print("[case done]", out, flush=True)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)")
ap.add_argument("--split", default="test", choices=["test", "val"],
help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)")
ap.add_argument("--case", required=True, help="case number, comma list, or 'all'")
ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases")
ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>")
ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)")
ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)")
ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json")
a = ap.parse_args()
man = json.loads(case_file(a.task, a.split).read_text())
assert man["split"] == a.split, (man["split"], a.split)
ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"])
if not ckpt.is_file():
raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)")
out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split)
out_root.mkdir(parents=True, exist_ok=True)
ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")]
steps = a.steps or man.get("denoising_steps_per_latent_block", 35)
for i in ids:
case = dict(man["cases"][i])
if a.short_only:
case["rollout_windows"] = 0
run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo)
if __name__ == "__main__":
main()