Instructions to use WitneyWW/acts-sensoronly-ablation with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Cosmos
How to use WitneyWW/acts-sensoronly-ablation with Cosmos:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 6,768 Bytes
fc95770 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | """Build eval/cases/<task>_val_cases.json: the contact-stratified case list on the VAL split, same protocol as the
test cases (one case per (val segment x contact class) where such a window exists; the window is drawn with seed 0
from the windows that have room for the 15-window rollout, falling back to a short-only case if none has room).
source env.sh
$PYTHON_BIN scripts/build_val_cases.py --task pusht
$PYTHON_BIN scripts/build_val_cases.py --task rope
The dataset index stored per case is the stride-1 index of the val split built exactly as vtm_cosmos.infer.rollout
builds it. The script checks that the parent config and all TODO configs of the task give the identical window list,
so the same case file is valid for every run of the task (with/without view actions, with/without force).
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
from omegaconf import OmegaConf
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "scripts"))
from select_contact_windows import EVAL_CATS, classify, episode_force # noqa: E402
from vtm_cosmos.data.dataset import VtmWindowDataset # noqa: E402
from vtm_cosmos.data.splits import load_split # noqa: E402
from vtm_cosmos.streams import FUTURE_ACTIONS, WINDOW_FRAMES # noqa: E402
THR = 0.5
R = 15
LONG_TOTAL = WINDOW_FRAMES + FUTURE_ACTIONS * (R - 1) # 128 frames touched by a long rollout
CONFIGS = {
"pusht": ["nano_action_causal_pusht_v1", "nano_action_causal_pusht_sensoronly_v1", "nano_action_causal_pusht_sensoronly_noforce_v1"],
"rope": ["nano_action_causal_rope_v1", "nano_action_causal_rope_sensoronly_v1", "nano_action_causal_rope_sensoronly_noforce_v1"],
}
def build_ds(cfg, split):
episodes = load_split(str(REPO / cfg.data.split_file) if not Path(str(cfg.data.split_file)).is_absolute() else cfg.data.split_file)[split]
af = cfg.model.task_mode == "action_full"
return VtmWindowDataset(cfg.data.latent_root, cfg.data.action_root, episodes, action_file=cfg.data.action_file,
window_stride=1, optional_streams=tuple(cfg.data.get("optional_streams", ()) or ()),
include_view_actions=af, view_action_file=cfg.data.get("view_action_file", "view_actions_fps15.pt"),
require_valid_actions=af,
active_streams=tuple(cfg.model.active_streams) if cfg.model.get("active_streams") else None)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True, choices=sorted(CONFIGS))
ap.add_argument("--split", default="val")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--force-root", default=None,
help="action tree holding sensor_pose7d_force_fps15.pt (absolute GelSight normal force; not shipped in "
"data/, only used to class the windows). Default: data.action_root")
a = ap.parse_args()
cfgs = [OmegaConf.load(REPO / "configs" / f"{n}.yaml") for n in CONFIGS[a.task]]
dss = [build_ds(c, a.split) for c in cfgs]
metas = [[ds.window_meta(i) for i in range(len(ds))] for ds in dss]
for n, m in zip(CONFIGS[a.task][1:], metas[1:]):
if m != metas[0]:
raise SystemExit(f"{n}: {a.split} window index differs from {CONFIGS[a.task][0]} -- cases would not be shared")
cfg, ds, meta = cfgs[0], dss[0], metas[0]
print(f"[val] {a.task} split={a.split} episodes={len(ds.episode_info)} windows={len(ds)} "
f"(identical for {len(cfgs)} configs)", flush=True)
n_frames = {info["episode"]: int(info["num_frames"]) for info in ds.episode_info}
force = {ep: episode_force(Path(a.force_root or cfg.data.action_root), ep) for ep in n_frames}
usable = set(meta)
rng = np.random.RandomState(a.seed)
cases = []
for ep in sorted(n_frames):
idx_ep = [i for i, (e, _) in enumerate(meta) if e == ep]
cls = {i: classify(force[ep][meta[i][1]:meta[i][1] + WINDOW_FRAMES], THR) for i in idx_ep
if meta[i][1] + WINDOW_FRAMES <= n_frames[ep]}
for cat in ("none", "onset", "release", "full"):
pool = [i for i, c in cls.items() if c == cat]
if not pool:
continue
room = [i for i in pool if meta[i][1] + LONG_TOTAL <= n_frames[ep]
and all((ep, meta[i][1] + FUTURE_ACTIONS * k) in usable for k in range(R))]
pick = int(rng.choice(room if room else pool))
t0 = meta[pick][1]
rw = R if room else 0
c = (force[ep] > THR).astype(int)
short = c[t0:t0 + WINDOW_FRAMES]
long_c = c[t0:t0 + LONG_TOTAL] if rw else short
fut_long = long_c[8:]
tr = lambda x: [int(k) for k in np.flatnonzero(np.diff(x)) + 1] # noqa: E731
if not rw:
label = "short_only"
elif cat == "none":
label = "none_throughout" if not long_c.any() else "none_at_start"
else:
label = cat
nz = np.flatnonzero(long_c)
cases.append(dict(
task=len(cases), episode=ep, category=cat, index=pick, t_start=int(t0), rollout_windows=rw,
start_seconds=t0 / 15.0, short_contact=short.tolist(), short_transition_ticks=tr(short),
future_transition_ticks=[k for k in tr(short) if k >= 8],
long_contact_fraction=float(long_c.mean()), long_future_contact_fraction=float(fut_long.mean()),
contact_over_128_frames=long_c.tolist(), first_contact_tick=int(nz[0]) if len(nz) else None,
long_label=label, output_subdir=f"{len(cases):02d}_{Path(ep).stem}_{cat}"))
print(f" {cases[-1]['output_subdir']:45s} index={pick:5d} t_start={t0:5d} pool={len(pool):5d} "
f"room={len(room):5d} -> {label}", flush=True)
test = json.loads((REPO / "eval" / "cases" / f"{a.task}_test_cases.json").read_text())
man = {k: v for k, v in test.items() if k != "cases"}
man.update(split=a.split, num_windows=len(ds), episodes=sorted(n_frames), selection_seed=a.seed,
selection="one case per (val segment x contact class); window drawn with seed 0 among windows with room "
"for the 15-window rollout (short-only if none has room); built by scripts/build_val_cases.py",
valid_for_configs=CONFIGS[a.task], cases=cases)
out = REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json"
out.write_text(json.dumps(man, indent=1))
print(f"[val] wrote {out}: {len(cases)} cases ({sum(c['rollout_windows'] > 0 for c in cases)} with a long rollout)")
if __name__ == "__main__":
main()
|