File size: 9,276 Bytes
b1b6f13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
"""Stratified contact-window selection for `vtm_cosmos.infer.rollout`.

Every window of a split is classed from the recorded GelSight normal force
(`max(force_left, force_right) > --thr N`) over its 16 fps15 frames, using the same rule as
`vm_diffusion/scripts/wrist_contact_eval.py`:

  none    : no frame in contact
  full    : >= 80 % of the 16 frames in contact
  onset   : last 3 context frames free, last 4 future frames in contact   (no-contact -> contact)
  release : last 3 context frames in contact, last 4 future frames free  (contact -> no-contact)
  mixed   : everything else (counted, never evaluated)

`--n-per-cat` windows are drawn per class with a fixed seed and written out as *dataset indices*
for `rollout.py --window-stride 1 --windows <idx ...>`. The long-horizon picks are a nested subset
of the short-horizon picks (same windows, filtered to those with room for the full rollout), so the
two horizons are a paired comparison.

    python scripts/select_contact_windows.py --config runs/nano_lora_mb_v1/config.yaml \
        --split val --out runs/nano_lora_mb_v1/contact/sel_val.json
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path

import numpy as np
import torch
from omegaconf import OmegaConf

from vtm_cosmos.data.dataset import VtmWindowDataset
from vtm_cosmos.data.splits import load_split
from vtm_cosmos.streams import FUTURE_ACTIONS, WINDOW_FRAMES

CATS = ("none", "full", "onset", "release", "mixed")
EVAL_CATS = ("none", "full", "onset", "release")
FORCE_FILE = "sensor_pose7d_force_fps15.pt"


def classify(c: np.ndarray, thr: float) -> str:
    """Contact class of one 16-frame window. Order matters: none/full win over the transitions."""
    w = c > thr
    ctx, fut = w[:8], w[8:]
    if not w.any():
        return "none"
    if w.mean() >= 0.8:
        return "full"
    if not ctx[-3:].any() and fut[-4:].all():
        return "onset"
    if ctx[-3:].all() and not fut[-4:].any():
        return "release"
    return "mixed"


def episode_force(action_root: Path, episode: str) -> np.ndarray:
    """Per-frame max(F_left, F_right) in newtons, on the same fps15 grid as the latent tree."""
    p = Path(action_root) / Path(episode).stem / FORCE_FILE
    if not p.exists():
        raise FileNotFoundError(f"{episode}: force file {p} not found")
    d = torch.load(p, map_location="cpu", weights_only=False)
    return np.maximum(d["force_left"].numpy(), d["force_right"].numpy())


def classify_split(ds: VtmWindowDataset, action_root: Path, thr: float):
    """-> (cls array over dataset indices, [(episode, t_start)], {episode: num_frames})."""
    n_frames = {info["episode"]: int(info["num_frames"]) for info in ds.episode_info}
    force = {}
    for ep in n_frames:
        f = episode_force(action_root, ep)
        if len(f) != n_frames[ep]:
            raise ValueError(f"{ep}: force file has {len(f)} frames, latent tree has {n_frames[ep]}")
        force[ep] = f
    cls = np.empty(len(ds), dtype=object)
    meta = []
    for i in range(len(ds)):
        ep, t0 = ds.window_meta(i)
        c = force[ep][t0:t0 + WINDOW_FRAMES]
        cls[i] = classify(c, thr) if len(c) == WINDOW_FRAMES else "mixed"
        meta.append((ep, t0))
    return cls, meta, n_frames


def write_fixed(args, ds: VtmWindowDataset, episodes: list[str]) -> None:
    """Re-select the exact windows of an earlier eval as stride-1 indices, split into shards."""
    prev = json.loads(Path(args.fixed_from).read_text())["results"]
    want = [(r["episode"], int(r["t_start"])) for r in prev]
    lookup = {}
    for i in range(len(ds)):
        lookup.setdefault(ds.window_meta(i), i)
    missing = [k for k in want if k not in lookup]
    if missing:
        raise ValueError(f"{len(missing)} windows of {args.fixed_from} are not in this split, e.g. {missing[:3]}")
    rows = [{"index": lookup[k], "episode": k[0], "t_start": k[1]} for k in want]
    n = args.fixed_shards
    size = -(-len(rows) // n)
    short = {f"fixed_{chr(ord('a') + s)}": rows[s * size:(s + 1) * size] for s in range(n)}
    out = {"split": args.split, "config": str(args.config), "fixed_from": str(args.fixed_from),
           "thr": args.thr, "seed": None, "rollout_windows": 0, "window_stride": 1,
           "episodes": episodes, "num_windows": len(ds), "short": short, "long": {}}
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    Path(args.out).write_text(json.dumps(out, indent=1))
    print(f"[select] fixed: {len(rows)} windows from {args.fixed_from} -> "
          + ", ".join(f"{k} {len(v)}" for k, v in short.items()) + f" -> {args.out}", flush=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--config", required=True, help="a run's config.yaml (data.latent_root / action_root / split_file)")
    ap.add_argument("--split", default="val", choices=["train", "val", "test"])
    ap.add_argument("--out", required=True)
    ap.add_argument("--thr", type=float, default=0.5, help="contact threshold in newtons")
    ap.add_argument("--n-per-cat", type=int, default=25, help="short-horizon windows per class")
    ap.add_argument("--n-long-per-cat", type=int, default=8, help="long-horizon windows per class (nested subset)")
    ap.add_argument("--rollout-windows", type=int, default=15, help="chained windows a long rollout needs")
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--cats", default=",".join(EVAL_CATS))
    ap.add_argument("--fixed-from", default=None,
                    help="window_eval.json of an earlier run: re-select exactly its windows (as stride-1 indices) "
                         "instead of sampling by contact class")
    ap.add_argument("--fixed-shards", type=int, default=2, help="split the fixed windows into this many pseudo-classes")
    args = ap.parse_args()

    cfg = OmegaConf.load(args.config)
    action_root = Path(cfg.data.action_root)
    episodes = load_split(cfg.data.split_file)[args.split]

    # window_stride=1 so the dataset index enumerates every window; rollout must use the same.
    # action_full also drops windows with an invalid action transition, which renumbers the index --
    # build the dataset exactly as rollout.py does or the picks point at different windows.
    action_full = str(cfg.model.get("task_mode", "action_causal")) == "action_full"
    ds = VtmWindowDataset(
        episodes=episodes,
        latent_root=cfg.data.latent_root,
        action_root=cfg.data.action_root,
        action_file=cfg.data.action_file,
        window_stride=1,
        optional_streams=tuple(cfg.data.get("optional_streams", ()) or ()),
        include_view_actions=action_full,
        view_action_file=cfg.data.get("view_action_file", "view_actions_fps15.pt"),
        require_valid_actions=action_full,
    )
    print(f"[select] split={args.split} episodes={len(episodes)} windows={len(ds)} "
          f"presence={ds.presence_summary()}", flush=True)

    if args.fixed_from:
        write_fixed(args, ds, episodes)
        return

    cls, meta, n_frames = classify_split(ds, action_root, args.thr)
    counts = {c: int((cls == c).sum()) for c in CATS}
    print(f"[select] windows by class: {counts}", flush=True)

    # a long rollout scores t0 + 8*k for k in 0..R-1, so the last GT window starts at t0 + 8*(R-1)
    long_span = FUTURE_ACTIONS * (args.rollout_windows - 1) + WINDOW_FRAMES

    # every window a long rollout touches must itself be conditionable (present in this index)
    usable = set(meta)
    rng = np.random.RandomState(args.seed)
    short: dict[str, list[dict]] = {}
    long_: dict[str, list[dict]] = {}
    for cat in args.cats.split(","):
        pool = np.flatnonzero(cls == cat)
        if len(pool) < args.n_per_cat:
            print(f"[select] WARNING class '{cat}' has only {len(pool)} windows, "
                  f"asked for {args.n_per_cat}", flush=True)
        pick = np.sort(rng.choice(pool, min(args.n_per_cat, len(pool)), replace=False))
        rows = [{"index": int(i), "episode": meta[i][0], "t_start": int(meta[i][1])} for i in pick]
        short[cat] = rows
        room = [r for r in rows if r["t_start"] + long_span <= n_frames[r["episode"]]
                and all((r["episode"], r["t_start"] + FUTURE_ACTIONS * k) in usable
                        for k in range(args.rollout_windows))]
        long_[cat] = room[: args.n_long_per_cat]
        print(f"[select] {cat:8s} pool={len(pool):6d} short={len(rows):3d} "
              f"long={len(long_[cat]):3d} (of {len(room)} with room)", flush=True)

    out = {
        "split": args.split, "config": str(args.config), "thr": args.thr, "seed": args.seed,
        "n_per_cat": args.n_per_cat, "n_long_per_cat": args.n_long_per_cat,
        "rollout_windows": args.rollout_windows, "long_span_frames": long_span,
        "window_stride": 1, "counts": counts,
        "episodes": episodes, "num_windows": len(ds),
        "short": short, "long": long_,
    }
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    Path(args.out).write_text(json.dumps(out, indent=1))
    n_s = sum(len(v) for v in short.values()); n_l = sum(len(v) for v in long_.values())
    print(f"[select] wrote {args.out}: {n_s} short + {n_l} long windows", flush=True)


if __name__ == "__main__":
    main()