Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 8,466 Bytes
b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 fc95770 b1b6f13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render
GT/prediction videos for the five image streams.
python eval/run_test_case.py --task rope --case 0 # one case
python eval/run_test_case.py --task rope --case all # every case, sequentially
python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt
python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt
The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the
exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py).
Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json,
strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py.
Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT)."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
import imageio.v2 as imageio
import numpy as np
from PIL import Image, ImageDraw
REPO = Path(__file__).resolve().parents[1]
STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
# data tree from the split file of the case manifest
TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope",
"mixed_pusht_wrist_holdout": "mixed_pushT"}
def default_out_root(ckpt: Path, split: str = "test") -> Path:
"""$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014"""
runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}"
def case_file(task: str, split: str) -> Path:
f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json"
if not f.is_file():
raise SystemExit(f"no {split} cases for task {task}: {f}")
return f
def env(name: str) -> str:
v = os.environ.get(name)
if not v:
raise SystemExit(f"${name} is not set -- run `source env.sh` first")
return v
def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool):
out = out_root / case["output_subdir"]
if skip_done and (out / "complete.json").is_file():
print("[skip] done:", out, flush=True)
return
out.mkdir(parents=True, exist_ok=True)
(out / "case.json").write_text(json.dumps(case, indent=2))
print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True)
data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT")
tree = TREE_OF_SPLIT[Path(man["split_file"]).stem]
overrides = [
f"data.latent_root={data}/{tree}_latents",
f"data.action_root={data}/{tree}_actions",
f"data.split_file={REPO / man['split_file']}",
f"model.nano_checkpoint={ckroot}/Cosmos3-Nano",
f"out_dir={out_root}",
]
subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout",
"--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1",
"--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]),
"--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)),
"--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO)
w = json.loads((out / "window_eval.json").read_text())["results"][0]
if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])):
raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, "
f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)")
# Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once,
# then append each rollout window's 8 future frames.
stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}"
if case["rollout_windows"]:
rolls = json.loads((out / "reference_rollout.json").read_text())
assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout"
assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early"
for mode in (("short", "long") if case["rollout_windows"] else ("short",)):
video_path = out / f"{mode}_gt_pred.mp4"
count = 0
with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer:
for k in range(1 if mode == "short" else case["rollout_windows"]):
suffix = "" if mode == "short" else f"_roll{k}"
strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS]
h = (strips[0].shape[0] - 4) // 2
wd = strips[0].shape[1] // 16 - 2
for t in range(0 if k == 0 else 8, 16):
canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black")
draw = ImageDraw.Draw(canvas)
for col, (stream, strip) in enumerate(zip(STREAMS, strips)):
x = t * (wd + 2)
canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32))
canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64))
draw.text((col * wd + 4, 16), "GT: " + stream, fill="white")
draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white")
label = "observed history" if count < 8 else "predicted future"
draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white")
writer.append_data(np.asarray(canvas))
count += 1
print("[video]", video_path, "frames=", count, flush=True)
(out / "complete.json").write_text(json.dumps(dict(
short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0,
rollout_windows=case["rollout_windows"])))
print("[case done]", out, flush=True)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)")
ap.add_argument("--split", default="test", choices=["test", "val"],
help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)")
ap.add_argument("--case", required=True, help="case number, comma list, or 'all'")
ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases")
ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>")
ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)")
ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)")
ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json")
a = ap.parse_args()
man = json.loads(case_file(a.task, a.split).read_text())
assert man["split"] == a.split, (man["split"], a.split)
ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"])
if not ckpt.is_file():
raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)")
out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split)
out_root.mkdir(parents=True, exist_ok=True)
ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")]
steps = a.steps or man.get("denoising_steps_per_latent_block", 35)
for i in ids:
case = dict(man["cases"][i])
if a.short_only:
case["rollout_windows"] = 0
run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo)
if __name__ == "__main__":
main()
|