"""Build eval/cases/_val_cases.json: the contact-stratified case list on the VAL split, same protocol as the test cases (one case per (val segment x contact class) where such a window exists; the window is drawn with seed 0 from the windows that have room for the 15-window rollout, falling back to a short-only case if none has room). source env.sh $PYTHON_BIN scripts/build_val_cases.py --task pusht $PYTHON_BIN scripts/build_val_cases.py --task rope The dataset index stored per case is the stride-1 index of the val split built exactly as vtm_cosmos.infer.rollout builds it. The script checks that the parent config and all TODO configs of the task give the identical window list, so the same case file is valid for every run of the task (with/without view actions, with/without force). """ from __future__ import annotations import argparse import json import sys from pathlib import Path import numpy as np from omegaconf import OmegaConf REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO / "scripts")) from select_contact_windows import EVAL_CATS, classify, episode_force # noqa: E402 from vtm_cosmos.data.dataset import VtmWindowDataset # noqa: E402 from vtm_cosmos.data.splits import load_split # noqa: E402 from vtm_cosmos.streams import FUTURE_ACTIONS, WINDOW_FRAMES # noqa: E402 THR = 0.5 R = 15 LONG_TOTAL = WINDOW_FRAMES + FUTURE_ACTIONS * (R - 1) # 128 frames touched by a long rollout CONFIGS = { "pusht": ["nano_action_causal_pusht_v1", "nano_action_causal_pusht_sensoronly_v1", "nano_action_causal_pusht_sensoronly_noforce_v1"], "rope": ["nano_action_causal_rope_v1", "nano_action_causal_rope_sensoronly_v1", "nano_action_causal_rope_sensoronly_noforce_v1"], } def build_ds(cfg, split): episodes = load_split(str(REPO / cfg.data.split_file) if not Path(str(cfg.data.split_file)).is_absolute() else cfg.data.split_file)[split] af = cfg.model.task_mode == "action_full" return VtmWindowDataset(cfg.data.latent_root, cfg.data.action_root, episodes, action_file=cfg.data.action_file, window_stride=1, optional_streams=tuple(cfg.data.get("optional_streams", ()) or ()), include_view_actions=af, view_action_file=cfg.data.get("view_action_file", "view_actions_fps15.pt"), require_valid_actions=af, active_streams=tuple(cfg.model.active_streams) if cfg.model.get("active_streams") else None) def main(): ap = argparse.ArgumentParser() ap.add_argument("--task", required=True, choices=sorted(CONFIGS)) ap.add_argument("--split", default="val") ap.add_argument("--seed", type=int, default=0) ap.add_argument("--force-root", default=None, help="action tree holding sensor_pose7d_force_fps15.pt (absolute GelSight normal force; not shipped in " "data/, only used to class the windows). Default: data.action_root") a = ap.parse_args() cfgs = [OmegaConf.load(REPO / "configs" / f"{n}.yaml") for n in CONFIGS[a.task]] dss = [build_ds(c, a.split) for c in cfgs] metas = [[ds.window_meta(i) for i in range(len(ds))] for ds in dss] for n, m in zip(CONFIGS[a.task][1:], metas[1:]): if m != metas[0]: raise SystemExit(f"{n}: {a.split} window index differs from {CONFIGS[a.task][0]} -- cases would not be shared") cfg, ds, meta = cfgs[0], dss[0], metas[0] print(f"[val] {a.task} split={a.split} episodes={len(ds.episode_info)} windows={len(ds)} " f"(identical for {len(cfgs)} configs)", flush=True) n_frames = {info["episode"]: int(info["num_frames"]) for info in ds.episode_info} force = {ep: episode_force(Path(a.force_root or cfg.data.action_root), ep) for ep in n_frames} usable = set(meta) rng = np.random.RandomState(a.seed) cases = [] for ep in sorted(n_frames): idx_ep = [i for i, (e, _) in enumerate(meta) if e == ep] cls = {i: classify(force[ep][meta[i][1]:meta[i][1] + WINDOW_FRAMES], THR) for i in idx_ep if meta[i][1] + WINDOW_FRAMES <= n_frames[ep]} for cat in ("none", "onset", "release", "full"): pool = [i for i, c in cls.items() if c == cat] if not pool: continue room = [i for i in pool if meta[i][1] + LONG_TOTAL <= n_frames[ep] and all((ep, meta[i][1] + FUTURE_ACTIONS * k) in usable for k in range(R))] pick = int(rng.choice(room if room else pool)) t0 = meta[pick][1] rw = R if room else 0 c = (force[ep] > THR).astype(int) short = c[t0:t0 + WINDOW_FRAMES] long_c = c[t0:t0 + LONG_TOTAL] if rw else short fut_long = long_c[8:] tr = lambda x: [int(k) for k in np.flatnonzero(np.diff(x)) + 1] # noqa: E731 if not rw: label = "short_only" elif cat == "none": label = "none_throughout" if not long_c.any() else "none_at_start" else: label = cat nz = np.flatnonzero(long_c) cases.append(dict( task=len(cases), episode=ep, category=cat, index=pick, t_start=int(t0), rollout_windows=rw, start_seconds=t0 / 15.0, short_contact=short.tolist(), short_transition_ticks=tr(short), future_transition_ticks=[k for k in tr(short) if k >= 8], long_contact_fraction=float(long_c.mean()), long_future_contact_fraction=float(fut_long.mean()), contact_over_128_frames=long_c.tolist(), first_contact_tick=int(nz[0]) if len(nz) else None, long_label=label, output_subdir=f"{len(cases):02d}_{Path(ep).stem}_{cat}")) print(f" {cases[-1]['output_subdir']:45s} index={pick:5d} t_start={t0:5d} pool={len(pool):5d} " f"room={len(room):5d} -> {label}", flush=True) test = json.loads((REPO / "eval" / "cases" / f"{a.task}_test_cases.json").read_text()) man = {k: v for k, v in test.items() if k != "cases"} man.update(split=a.split, num_windows=len(ds), episodes=sorted(n_frames), selection_seed=a.seed, selection="one case per (val segment x contact class); window drawn with seed 0 among windows with room " "for the 15-window rollout (short-only if none has room); built by scripts/build_val_cases.py", valid_for_configs=CONFIGS[a.task], cases=cases) out = REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json" out.write_text(json.dumps(man, indent=1)) print(f"[val] wrote {out}: {len(cases)} cases ({sum(c['rollout_windows'] > 0 for c in cases)} with a long rollout)") if __name__ == "__main__": main()