Nadeem-Shoukath115113/opensarm / tools /predict_act_pri.py
Nadeem-Shoukath115113's picture
download
raw
14.6 kB
#!/usr/bin/env python
"""Run a trained act_pri checkpoint over a LeRobot dataset and write predictions.
Mirrors how the stage estimator is fed at inference inside SARM2
(`sarm2_ws._slice_act_pri_inputs`): a window of `n_obs_steps` frames spaced by
`frame_gap`, ending at the frame being classified, encoded by the frozen SigLIP
backbone, plus the normalized proprioceptive state.
Predictions are written into the same label store `review_act_pri_ui.py` reads,
so several checkpoints can be compared side by side per group:
<out-dir>/<session>/groups/episode_XXXXXX.json .models[<name>] per group
<out-dir>/<session>/images/episode_XXXXXX/*.jpg the 3 camera views
<out-dir>/<session>/per_frame/episode_XXXXXX.npy expanded to every frame
Example
-------
python tools/predict_act_pri.py \\
--dataset-root ~/.cache/huggingface/hub/datasets--.../snapshots/<rev>/<session> \\
--checkpoint outputs/.../act_pri_best.pt --name act_pri_best
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
from typing import Dict, List, Optional, Sequence
import numpy as np
import torch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from models.action_estimator import ActionTransformer # noqa: E402
from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402
from tools.act_pri_common import ( # noqa: E402
PRIMITIVES,
SARM2_TASK_LIST,
TASK_TO_INDEX,
DatasetMeta,
EpisodeMeta,
build_groups,
group_size_for,
groups_path,
images_dir,
load_dataset_meta,
save_episode_labels,
task_to_class_id,
write_per_frame,
write_vocab,
)
from tools.build_act_pri_clips import load_source_table # noqa: E402
from utils.train_utils import get_normalizer_from_calculated # noqa: E402
def decode_episode_frames(
video_path: Path, start_time: float, fps: float, num_frames: int
):
"""Decode every frame of one episode from a (possibly aggregated) mp4."""
import av
offset = int(round(start_time * fps))
container = av.open(str(video_path))
stream = container.streams.video[0]
stream.thread_type = "AUTO"
time_base = float(stream.time_base)
try:
container.seek(int(max(0.0, start_time - 0.5) / time_base), stream=stream, backward=True)
except Exception:
container.seek(0)
frames: Dict[int, "np.ndarray"] = {}
for frame in container.decode(video=0):
local = int(round(float(frame.pts * time_base) * fps)) - offset
if local < 0:
continue
if local >= num_frames:
break
if local not in frames:
frames[local] = frame.to_ndarray(format="rgb24")
container.close()
if len(frames) != num_frames:
missing = [i for i in range(num_frames) if i not in frames]
raise SystemExit(f"{video_path.name}: missing frames {missing[:5]} ...")
return [frames[i] for i in range(num_frames)]
def window_indices(end_frame: int, n_obs_steps: int, frame_gap: int, ep_start: int = 0):
"""The frame window act_pri sees, matching FrameGapLeRobotDataset(stage_model=True)."""
idx = end_frame
steps_between = n_obs_steps - 1
if idx - ep_start >= frame_gap * n_obs_steps:
mid = [idx - frame_gap * j for j in range(steps_between, 0, -1)]
else:
span = idx - ep_start
mid = [ep_start + round(span * k / n_obs_steps) for k in range(1, n_obs_steps)]
frames = mid + [idx]
for i in range(1, len(frames)):
frames[i] = max(frames[i], frames[i - 1])
return frames
@torch.no_grad()
def encode_episode(
encoder: FrozenSiglipEncoder, frames_per_cam: Dict[str, list], batch_size: int, device
) -> Dict[str, torch.Tensor]:
"""SigLIP-embed every frame once; windows are then assembled for free."""
out: Dict[str, torch.Tensor] = {}
for cam, frames in frames_per_cam.items():
embeddings = []
for start in range(0, len(frames), batch_size):
chunk = np.stack(frames[start : start + batch_size])
tensor = torch.from_numpy(chunk).permute(0, 3, 1, 2).float().div_(255.0)
embeddings.append(encoder.encode_image(tensor.to(device)).float().cpu())
out[cam] = torch.cat(embeddings)
return out
def main() -> None:
ap = argparse.ArgumentParser(
description="Run a trained act_pri checkpoint over a dataset.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
ap.add_argument("--dataset-root", required=True, help="a LeRobot dataset directory")
ap.add_argument("--out-dir", default="act_pri_preds", help="prediction store root")
ap.add_argument("--session", default=None, help="sub-directory name (default: dataset dir name)")
ap.add_argument(
"--checkpoint", action="append", required=True,
help="checkpoint path; repeat to score several models over the same frames",
)
ap.add_argument(
"--name", action="append", default=None,
help="display name per --checkpoint (default: the file stem)",
)
ap.add_argument("--state-norm", required=True, help="state_norm.json used at training time")
ap.add_argument("--vision-ckpt", default="google/siglip2-base-patch16-224")
ap.add_argument("--n-obs-steps", type=int, default=6)
ap.add_argument("--frame-gap", type=int, default=30,
help="must match cfg.model.frame_gap used at training time")
ap.add_argument("--d-model", type=int, default=512)
ap.add_argument("--n-layers", type=int, default=4)
ap.add_argument("--n-heads", type=int, default=8)
ap.add_argument("--state-dim", type=int, default=12)
ap.add_argument("--num-tasks", type=int, default=22)
ap.add_argument("--num-classes", type=int, default=8)
ap.add_argument("--group-seconds", type=float, default=1.0, help="one prediction per this many seconds")
ap.add_argument("--key-position", choices=["start", "mid", "end"], default="mid")
ap.add_argument("--image-size", type=int, default=512, help="longest side of the saved review images")
ap.add_argument("--jpeg-quality", type=int, default=88)
ap.add_argument("--encode-batch", type=int, default=64)
ap.add_argument("--episodes", default="all")
ap.add_argument("--device", default="cuda")
ap.add_argument("--no-images", action="store_true", help="skip writing review images")
args = ap.parse_args()
names = args.name or []
if len(names) < len(args.checkpoint):
names += [Path(c).stem for c in args.checkpoint[len(names):]]
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
meta = load_dataset_meta(Path(args.dataset_root))
cameras = meta.camera_keys
session = args.session or meta.name
out_dir = Path(args.out_dir).expanduser().resolve() / session
out_dir.mkdir(parents=True, exist_ok=True)
write_vocab(out_dir / "vocab.json")
group_size = group_size_for(meta.fps, args.group_seconds)
if args.episodes == "all":
episode_ids = [e.index for e in meta.episodes]
else:
episode_ids = []
for part in args.episodes.split(","):
if "-" in part:
lo, hi = part.split("-", 1)
episode_ids.extend(range(int(lo), int(hi) + 1))
elif part.strip():
episode_ids.append(int(part))
print(f"[data ] {session}: v{meta.codebase_version} {meta.fps:g} fps "
f"{len(episode_ids)} episodes {meta.total_frames} frames", flush=True)
print(f"[model] {len(args.checkpoint)} checkpoint(s): {names}", flush=True)
encoder = FrozenSiglipEncoder(args.vision_ckpt, device)
normalizer = get_normalizer_from_calculated(args.state_norm, device)
models = {}
for name, ckpt_path in zip(names, args.checkpoint):
model = ActionTransformer(
d_model=args.d_model, vis_emb_dim=768, state_dim=args.state_dim,
n_layers=args.n_layers, n_heads=args.n_heads, dropout=0.0,
num_cameras=len(cameras), num_tasks=args.num_tasks, num_classes=args.num_classes,
).to(device)
ckpt = torch.load(ckpt_path, map_location=device, weights_only=False)
model.load_state_dict(ckpt["model"])
model.eval()
models[name] = model
print(f"[model] {name}: {ckpt_path} (epoch {ckpt.get('epoch')})", flush=True)
t0 = time.time()
tally = {name: {p: 0 for p in PRIMITIVES} for name in names}
off_vocab = {name: 0 for name in names}
for ep_id in episode_ids:
episode = meta.episode(ep_id)
groups = build_groups(episode.length, group_size, args.key_position)
frames_per_cam = {
cam: decode_episode_frames(
episode.videos[cam].path, episode.videos[cam].start_time, meta.fps, episode.length
)
for cam in cameras
}
if not args.no_images:
from PIL import Image
dest = images_dir(out_dir, ep_id)
dest.mkdir(parents=True, exist_ok=True)
for cam in cameras:
short = cam.split(".")[-1]
for g in groups:
img = Image.fromarray(frames_per_cam[cam][g.key_frame])
if args.image_size and max(img.size) > args.image_size:
scale = args.image_size / max(img.size)
img = img.resize((int(img.width * scale), int(img.height * scale)), Image.BICUBIC)
img.save(dest / f"g{g.group_index:04d}_{short}.jpg", quality=args.jpeg_quality)
embeddings = encode_episode(encoder, frames_per_cam, args.encode_batch, device)
del frames_per_cam
state = torch.from_numpy(load_source_table(meta, episode)["state"]).float()
windows = [window_indices(g.key_frame, args.n_obs_steps, args.frame_gap) for g in groups]
window_tensor = torch.tensor(windows, dtype=torch.long) # (G, T)
img_emb = torch.stack(
[embeddings[cam][window_tensor.reshape(-1)].view(len(groups), args.n_obs_steps, -1)
for cam in cameras],
dim=1,
).to(device) # (G, N, T, D)
state_win = state[window_tensor.reshape(-1)].view(len(groups), args.n_obs_steps, -1).to(device)
state_win = normalizer.normalize(state_win)
lens = torch.full((len(groups),), args.n_obs_steps, dtype=torch.int32, device=device)
per_model: Dict[str, List[dict]] = {}
with torch.no_grad():
for name, model in models.items():
logits, _ = model(img_emb, state_win, lens)
probs = torch.softmax(logits.float(), dim=-1).cpu()
picks = probs.argmax(dim=-1).tolist()
rows = []
for gi, pick in enumerate(picks):
label = SARM2_TASK_LIST[pick]
if label not in PRIMITIVES:
off_vocab[name] += 1
else:
tally[name][label] += 1
rows.append({
"label": label,
"index": int(pick),
"scores": {p: round(float(probs[gi, TASK_TO_INDEX[p]]), 5) for p in PRIMITIVES},
"confidence": round(float(probs[gi, pick]), 5),
})
per_model[name] = rows
primary = names[0]
records = []
for gi, g in enumerate(groups):
pred = per_model[primary][gi]
records.append({
"group_index": g.group_index,
"start_frame": g.start_frame,
"end_frame": g.end_frame,
"num_frames": g.num_frames,
"start_time": round(g.start_frame / meta.fps, 3),
"end_time": round((g.end_frame + 1) / meta.fps, 3),
"key_frame": g.key_frame,
"key_time": round(g.key_frame / meta.fps, 3),
"pred": pred["label"],
"pred_index": pred["index"],
"pred_raw": "",
"scores": pred["scores"],
"label": pred["label"],
"label_index": pred["index"],
"modified": False,
"reviewed": False,
"models": {name: per_model[name][gi] for name in names},
"images": {
cam: f"images/episode_{ep_id:06d}/g{g.group_index:04d}_{cam.split('.')[-1]}.jpg"
for cam in cameras
},
})
payload = {
"dataset_root": str(meta.root),
"dataset_name": session,
"codebase_version": meta.codebase_version,
"episode_index": ep_id,
"task": episode.task,
"fps": meta.fps,
"num_frames": episode.length,
"dataset_from_index": episode.from_index,
"dataset_to_index": episode.to_index,
"group_size": group_size,
"group_seconds": group_size / meta.fps,
"key_position": args.key_position,
"cameras": list(cameras),
"model": primary,
"model_names": list(names),
"checkpoints": {n: str(c) for n, c in zip(names, args.checkpoint)},
"n_obs_steps": args.n_obs_steps,
"frame_gap": args.frame_gap,
"primitives": PRIMITIVES,
"primitive_index": {p: TASK_TO_INDEX[p] for p in PRIMITIVES},
"primitive_gate": {p: task_to_class_id(p) for p in PRIMITIVES},
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
"groups": records,
}
save_episode_labels(out_dir, payload)
write_per_frame(out_dir, payload)
agree = sum(
1 for gi in range(len(groups))
if len({per_model[n][gi]["label"] for n in names}) == 1
)
print(f"[ep {ep_id:03d}] {episode.length} frames -> {len(groups)} groups | "
f"models agree on {agree}/{len(groups)} | {time.time() - t0:.0f}s", flush=True)
del img_emb, state_win, embeddings
torch.cuda.empty_cache()
print(f"\n[done] {time.time() - t0:.0f}s", flush=True)
for name in names:
total = sum(tally[name].values()) + off_vocab[name]
print(f" {name}: {tally[name]}" + (f" off-vocabulary: {off_vocab[name]}/{total}" if off_vocab[name] else ""))
print(f"\nReview with: python tools/annotate_act_pri_ui.py --labels-dir {Path(args.out_dir).resolve()} --dataset-root {Path(args.dataset_root).resolve()}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
14.6 kB
·
Xet hash:
8b06e750934f8fccb26816559ca98d7df9b0c9c64c2dc338b205a21b0b00b795

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.