Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download eval/run_test_case.py from WitneyWW/acts-sensoronly-ablation: direct link, hf CLI and curl.
- Browser
- Download file 8.47 kB
-
https://huggingface.co/WitneyWW/acts-sensoronly-ablation/resolve/main/eval/run_test_case.py
- Command line
-
hf download hf://WitneyWW/acts-sensoronly-ablation/eval/run_test_case.py
-
curl -L -o run_test_case.py https://huggingface.co/WitneyWW/acts-sensoronly-ablation/resolve/main/eval/run_test_case.py
8.47 kB
| """Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render | |
| GT/prediction videos for the five image streams. | |
| python eval/run_test_case.py --task rope --case 0 # one case | |
| python eval/run_test_case.py --task rope --case all # every case, sequentially | |
| python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt | |
| python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt | |
| The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the | |
| exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py). | |
| Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json, | |
| strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py. | |
| Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT).""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| import imageio.v2 as imageio | |
| import numpy as np | |
| from PIL import Image, ImageDraw | |
| REPO = Path(__file__).resolve().parents[1] | |
| STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"] | |
| TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json")) | |
| # data tree from the split file of the case manifest | |
| TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope", | |
| "mixed_pusht_wrist_holdout": "mixed_pushT"} | |
| def default_out_root(ckpt: Path, split: str = "test") -> Path: | |
| """$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014""" | |
| runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs")) | |
| return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}" | |
| def case_file(task: str, split: str) -> Path: | |
| f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json" | |
| if not f.is_file(): | |
| raise SystemExit(f"no {split} cases for task {task}: {f}") | |
| return f | |
| def env(name: str) -> str: | |
| v = os.environ.get(name) | |
| if not v: | |
| raise SystemExit(f"${name} is not set -- run `source env.sh` first") | |
| return v | |
| def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool): | |
| out = out_root / case["output_subdir"] | |
| if skip_done and (out / "complete.json").is_file(): | |
| print("[skip] done:", out, flush=True) | |
| return | |
| out.mkdir(parents=True, exist_ok=True) | |
| (out / "case.json").write_text(json.dumps(case, indent=2)) | |
| print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True) | |
| data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT") | |
| tree = TREE_OF_SPLIT[Path(man["split_file"]).stem] | |
| overrides = [ | |
| f"data.latent_root={data}/{tree}_latents", | |
| f"data.action_root={data}/{tree}_actions", | |
| f"data.split_file={REPO / man['split_file']}", | |
| f"model.nano_checkpoint={ckroot}/Cosmos3-Nano", | |
| f"out_dir={out_root}", | |
| ] | |
| subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout", | |
| "--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1", | |
| "--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]), | |
| "--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)), | |
| "--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO) | |
| w = json.loads((out / "window_eval.json").read_text())["results"][0] | |
| if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])): | |
| raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, " | |
| f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)") | |
| # Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once, | |
| # then append each rollout window's 8 future frames. | |
| stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}" | |
| if case["rollout_windows"]: | |
| rolls = json.loads((out / "reference_rollout.json").read_text()) | |
| assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout" | |
| assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early" | |
| for mode in (("short", "long") if case["rollout_windows"] else ("short",)): | |
| video_path = out / f"{mode}_gt_pred.mp4" | |
| count = 0 | |
| with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer: | |
| for k in range(1 if mode == "short" else case["rollout_windows"]): | |
| suffix = "" if mode == "short" else f"_roll{k}" | |
| strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS] | |
| h = (strips[0].shape[0] - 4) // 2 | |
| wd = strips[0].shape[1] // 16 - 2 | |
| for t in range(0 if k == 0 else 8, 16): | |
| canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black") | |
| draw = ImageDraw.Draw(canvas) | |
| for col, (stream, strip) in enumerate(zip(STREAMS, strips)): | |
| x = t * (wd + 2) | |
| canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32)) | |
| canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64)) | |
| draw.text((col * wd + 4, 16), "GT: " + stream, fill="white") | |
| draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white") | |
| label = "observed history" if count < 8 else "predicted future" | |
| draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white") | |
| writer.append_data(np.asarray(canvas)) | |
| count += 1 | |
| print("[video]", video_path, "frames=", count, flush=True) | |
| (out / "complete.json").write_text(json.dumps(dict( | |
| short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0, | |
| rollout_windows=case["rollout_windows"]))) | |
| print("[case done]", out, flush=True) | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)") | |
| ap.add_argument("--split", default="test", choices=["test", "val"], | |
| help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)") | |
| ap.add_argument("--case", required=True, help="case number, comma list, or 'all'") | |
| ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases") | |
| ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>") | |
| ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)") | |
| ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)") | |
| ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json") | |
| a = ap.parse_args() | |
| man = json.loads(case_file(a.task, a.split).read_text()) | |
| assert man["split"] == a.split, (man["split"], a.split) | |
| ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"]) | |
| if not ckpt.is_file(): | |
| raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)") | |
| out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split) | |
| out_root.mkdir(parents=True, exist_ok=True) | |
| ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")] | |
| steps = a.steps or man.get("denoising_steps_per_latent_block", 35) | |
| for i in ids: | |
| case = dict(man["cases"][i]) | |
| if a.short_only: | |
| case["rollout_windows"] = 0 | |
| run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo) | |
| if __name__ == "__main__": | |
| main() | |