acts-sensoronly-ablation / scripts /build_val_cases.py
WitneyWW's picture
1116 TODO: pushT/rope val cases + val inference at epochs 14/15 (infer_val.sh), README
fc95770 verified
Raw History Blame Contribute Delete
6.77 kB
"""Build eval/cases/<task>_val_cases.json: the contact-stratified case list on the VAL split, same protocol as the
test cases (one case per (val segment x contact class) where such a window exists; the window is drawn with seed 0
from the windows that have room for the 15-window rollout, falling back to a short-only case if none has room).
source env.sh
$PYTHON_BIN scripts/build_val_cases.py --task pusht
$PYTHON_BIN scripts/build_val_cases.py --task rope
The dataset index stored per case is the stride-1 index of the val split built exactly as vtm_cosmos.infer.rollout
builds it. The script checks that the parent config and all TODO configs of the task give the identical window list,
so the same case file is valid for every run of the task (with/without view actions, with/without force).
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
from omegaconf import OmegaConf
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "scripts"))
from select_contact_windows import EVAL_CATS, classify, episode_force # noqa: E402
from vtm_cosmos.data.dataset import VtmWindowDataset # noqa: E402
from vtm_cosmos.data.splits import load_split # noqa: E402
from vtm_cosmos.streams import FUTURE_ACTIONS, WINDOW_FRAMES # noqa: E402
THR = 0.5
R = 15
LONG_TOTAL = WINDOW_FRAMES + FUTURE_ACTIONS * (R - 1) # 128 frames touched by a long rollout
CONFIGS = {
"pusht": ["nano_action_causal_pusht_v1", "nano_action_causal_pusht_sensoronly_v1", "nano_action_causal_pusht_sensoronly_noforce_v1"],
"rope": ["nano_action_causal_rope_v1", "nano_action_causal_rope_sensoronly_v1", "nano_action_causal_rope_sensoronly_noforce_v1"],
}
def build_ds(cfg, split):
episodes = load_split(str(REPO / cfg.data.split_file) if not Path(str(cfg.data.split_file)).is_absolute() else cfg.data.split_file)[split]
af = cfg.model.task_mode == "action_full"
return VtmWindowDataset(cfg.data.latent_root, cfg.data.action_root, episodes, action_file=cfg.data.action_file,
window_stride=1, optional_streams=tuple(cfg.data.get("optional_streams", ()) or ()),
include_view_actions=af, view_action_file=cfg.data.get("view_action_file", "view_actions_fps15.pt"),
require_valid_actions=af,
active_streams=tuple(cfg.model.active_streams) if cfg.model.get("active_streams") else None)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--task", required=True, choices=sorted(CONFIGS))
ap.add_argument("--split", default="val")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--force-root", default=None,
help="action tree holding sensor_pose7d_force_fps15.pt (absolute GelSight normal force; not shipped in "
"data/, only used to class the windows). Default: data.action_root")
a = ap.parse_args()
cfgs = [OmegaConf.load(REPO / "configs" / f"{n}.yaml") for n in CONFIGS[a.task]]
dss = [build_ds(c, a.split) for c in cfgs]
metas = [[ds.window_meta(i) for i in range(len(ds))] for ds in dss]
for n, m in zip(CONFIGS[a.task][1:], metas[1:]):
if m != metas[0]:
raise SystemExit(f"{n}: {a.split} window index differs from {CONFIGS[a.task][0]} -- cases would not be shared")
cfg, ds, meta = cfgs[0], dss[0], metas[0]
print(f"[val] {a.task} split={a.split} episodes={len(ds.episode_info)} windows={len(ds)} "
f"(identical for {len(cfgs)} configs)", flush=True)
n_frames = {info["episode"]: int(info["num_frames"]) for info in ds.episode_info}
force = {ep: episode_force(Path(a.force_root or cfg.data.action_root), ep) for ep in n_frames}
usable = set(meta)
rng = np.random.RandomState(a.seed)
cases = []
for ep in sorted(n_frames):
idx_ep = [i for i, (e, _) in enumerate(meta) if e == ep]
cls = {i: classify(force[ep][meta[i][1]:meta[i][1] + WINDOW_FRAMES], THR) for i in idx_ep
if meta[i][1] + WINDOW_FRAMES <= n_frames[ep]}
for cat in ("none", "onset", "release", "full"):
pool = [i for i, c in cls.items() if c == cat]
if not pool:
continue
room = [i for i in pool if meta[i][1] + LONG_TOTAL <= n_frames[ep]
and all((ep, meta[i][1] + FUTURE_ACTIONS * k) in usable for k in range(R))]
pick = int(rng.choice(room if room else pool))
t0 = meta[pick][1]
rw = R if room else 0
c = (force[ep] > THR).astype(int)
short = c[t0:t0 + WINDOW_FRAMES]
long_c = c[t0:t0 + LONG_TOTAL] if rw else short
fut_long = long_c[8:]
tr = lambda x: [int(k) for k in np.flatnonzero(np.diff(x)) + 1] # noqa: E731
if not rw:
label = "short_only"
elif cat == "none":
label = "none_throughout" if not long_c.any() else "none_at_start"
else:
label = cat
nz = np.flatnonzero(long_c)
cases.append(dict(
task=len(cases), episode=ep, category=cat, index=pick, t_start=int(t0), rollout_windows=rw,
start_seconds=t0 / 15.0, short_contact=short.tolist(), short_transition_ticks=tr(short),
future_transition_ticks=[k for k in tr(short) if k >= 8],
long_contact_fraction=float(long_c.mean()), long_future_contact_fraction=float(fut_long.mean()),
contact_over_128_frames=long_c.tolist(), first_contact_tick=int(nz[0]) if len(nz) else None,
long_label=label, output_subdir=f"{len(cases):02d}_{Path(ep).stem}_{cat}"))
print(f" {cases[-1]['output_subdir']:45s} index={pick:5d} t_start={t0:5d} pool={len(pool):5d} "
f"room={len(room):5d} -> {label}", flush=True)
test = json.loads((REPO / "eval" / "cases" / f"{a.task}_test_cases.json").read_text())
man = {k: v for k, v in test.items() if k != "cases"}
man.update(split=a.split, num_windows=len(ds), episodes=sorted(n_frames), selection_seed=a.seed,
selection="one case per (val segment x contact class); window drawn with seed 0 among windows with room "
"for the 15-window rollout (short-only if none has room); built by scripts/build_val_cases.py",
valid_for_configs=CONFIGS[a.task], cases=cases)
out = REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json"
out.write_text(json.dumps(man, indent=1))
print(f"[val] wrote {out}: {len(cases)} cases ({sum(c['rollout_windows'] > 0 for c in cases)} with a long rollout)")
if __name__ == "__main__":
main()