Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 5,257 Bytes
b1b6f13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | """Download the data / checkpoints of this repo (and the two public base models) on a new machine.
python scripts/download.py --what all # everything, ~175 GB
python scripts/download.py --what data raw ckpts nano wan_vae --tasks pusht --splits test # pushT test eval only
Everything except the base models lives in THIS Hugging Face repo (REPO_ID below), under
data/mixed_<tree>_latents/{tac,visual,wrist}/<episode>.pt Wan2.2 VAE latents (5 image streams)
data/mixed_<tree>_actions/<episode>/{sensor_actions_with_dforce_fps15.pt, sensor_actions_fps15_noforce.pt,
view_actions_fps15.pt}
data/raw/videos/<episode>/<stream>.mp4, data/raw/parquet/<episode>.parquet, data/raw/episodes.json
checkpoints/vae/lora_last.pt tactile LoRA of the Wan2.2 VAE (decode / metrics)
checkpoints/nano_action_causal_*_v1/epoch_XXX.pt trained LoRA checkpoints
Files land at <repo>/data/... and <repo>/checkpoints/..., which are the env.sh defaults of ACTS_DATA_ROOT /
ACTS_CKPT_ROOT. (If you point those variables elsewhere, pass --local-dir so that <local-dir>/data == ACTS_DATA_ROOT.)
--what (any subset):
data preprocessed latents + actions of the split episodes
raw 5-stream mp4 + parquet (needed for eval decoding: the VAE anchor frame is re-read from the mp4)
ckpts tactile VAE LoRA + the trained checkpoints of the runs of --tasks
nano nvidia/Cosmos3-Nano (transformer/ + json, ~29 GB) -> checkpoints/Cosmos3-Nano
wan_vae Wan-AI/Wan2.2-TI2V-5B :: Wan2.2_VAE.pth (2.8 GB) -> checkpoints/Wan2.2_VAE.pth
Token: $HF_TOKEN or `hf auth login` (the repo is private).
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
from huggingface_hub import hf_hub_download, snapshot_download
REPO_ID = os.environ.get("ACTS_HF_REPO", "WitneyWW/acts-sensoronly-ablation")
REPO = Path(__file__).resolve().parents[1]
# task -> (data tree, split file, released runs whose checkpoints live in this repo)
TASKS = {
"mb": ("mixed_mb", "mixed_mb_wrist_holdout", ["nano_action_causal_umi_sensoronly_noforce_v1"]),
"pusht": ("mixed_pushT", "mixed_pusht_wrist_holdout", ["nano_action_causal_pusht_v1"]),
"rope": ("mixed_rope", "mixed_rope_wrist_holdout", ["nano_action_causal_rope_v1"]),
}
ACTION_FILES = ["sensor_actions_with_dforce_fps15.pt", "sensor_actions_fps15_noforce.pt", "view_actions_fps15.pt"]
def split_episodes(task: str, parts: list[str]) -> list[str]:
s = json.loads((REPO / "splits" / f"{TASKS[task][1]}.json").read_text())
return [Path(e).stem for p in parts for e in s[p]]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--what", nargs="+", default=["all"], choices=["all", "data", "raw", "ckpts", "nano", "wan_vae"])
ap.add_argument("--tasks", nargs="+", default=list(TASKS), choices=list(TASKS))
ap.add_argument("--splits", nargs="+", default=["train", "val", "test"], choices=["train", "val", "test"],
help="restrict data/raw to these split parts (e.g. --splits test for eval only)")
ap.add_argument("--local-dir", default=str(REPO), help="files go to <local-dir>/data and <local-dir>/checkpoints")
ap.add_argument("--workers", type=int, default=8)
a = ap.parse_args()
what = {"data", "raw", "ckpts", "nano", "wan_vae"} if "all" in a.what else set(a.what)
root = Path(a.local_dir)
for var, sub in (("ACTS_DATA_ROOT", "data"), ("ACTS_CKPT_ROOT", "checkpoints")):
v = os.environ.get(var)
if v and Path(v).resolve() != (root / sub).resolve():
print(f"[warn] ${var}={v} but files go to {root / sub} (use --local-dir)")
pats = []
if what & {"data", "raw"}:
for t in a.tasks:
tree = TASKS[t][0]
for e in split_episodes(t, a.splits):
if "data" in what:
pats += [f"data/{tree}_latents/{sub}/{e}.pt" for sub in ("tac", "visual", "wrist")]
pats += [f"data/{tree}_actions/{e}/{f}" for f in ACTION_FILES]
if "raw" in what:
pats += [f"data/raw/videos/{e}/*", f"data/raw/parquet/{e}.parquet"]
pats.append("data/raw/episodes.json")
if "ckpts" in what:
pats.append("checkpoints/vae/*")
pats += [f"checkpoints/{run}/*" for t in a.tasks for run in TASKS[t][2]]
if pats:
print(f"[{REPO_ID}] {len(pats)} patterns -> {root}")
snapshot_download(REPO_ID, local_dir=root, allow_patterns=pats, max_workers=a.workers)
if "nano" in what:
print(f"[nano] nvidia/Cosmos3-Nano -> {root / 'checkpoints/Cosmos3-Nano'}")
snapshot_download("nvidia/Cosmos3-Nano", local_dir=root / "checkpoints" / "Cosmos3-Nano",
allow_patterns=["*.json", "transformer/*", "*.md"], max_workers=a.workers)
if "wan_vae" in what:
print(f"[wan_vae] Wan-AI/Wan2.2-TI2V-5B/Wan2.2_VAE.pth -> {root / 'checkpoints'}")
hf_hub_download("Wan-AI/Wan2.2-TI2V-5B", "Wan2.2_VAE.pth", local_dir=root / "checkpoints")
print("done")
if __name__ == "__main__":
main()
|