Buckets:
| #!/usr/bin/env python | |
| """Run a trained act_pri checkpoint over a LeRobot dataset and write predictions. | |
| Mirrors how the stage estimator is fed at inference inside SARM2 | |
| (`sarm2_ws._slice_act_pri_inputs`): a window of `n_obs_steps` frames spaced by | |
| `frame_gap`, ending at the frame being classified, encoded by the frozen SigLIP | |
| backbone, plus the normalized proprioceptive state. | |
| Predictions are written into the same label store `review_act_pri_ui.py` reads, | |
| so several checkpoints can be compared side by side per group: | |
| <out-dir>/<session>/groups/episode_XXXXXX.json .models[<name>] per group | |
| <out-dir>/<session>/images/episode_XXXXXX/*.jpg the 3 camera views | |
| <out-dir>/<session>/per_frame/episode_XXXXXX.npy expanded to every frame | |
| Example | |
| ------- | |
| python tools/predict_act_pri.py \\ | |
| --dataset-root ~/.cache/huggingface/hub/datasets--.../snapshots/<rev>/<session> \\ | |
| --checkpoint outputs/.../act_pri_best.pt --name act_pri_best | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Sequence | |
| import numpy as np | |
| import torch | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from models.action_estimator import ActionTransformer # noqa: E402 | |
| from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402 | |
| from tools.act_pri_common import ( # noqa: E402 | |
| PRIMITIVES, | |
| SARM2_TASK_LIST, | |
| TASK_TO_INDEX, | |
| DatasetMeta, | |
| EpisodeMeta, | |
| build_groups, | |
| group_size_for, | |
| groups_path, | |
| images_dir, | |
| load_dataset_meta, | |
| save_episode_labels, | |
| task_to_class_id, | |
| write_per_frame, | |
| write_vocab, | |
| ) | |
| from tools.build_act_pri_clips import load_source_table # noqa: E402 | |
| from utils.train_utils import get_normalizer_from_calculated # noqa: E402 | |
| def decode_episode_frames( | |
| video_path: Path, start_time: float, fps: float, num_frames: int | |
| ): | |
| """Decode every frame of one episode from a (possibly aggregated) mp4.""" | |
| import av | |
| offset = int(round(start_time * fps)) | |
| container = av.open(str(video_path)) | |
| stream = container.streams.video[0] | |
| stream.thread_type = "AUTO" | |
| time_base = float(stream.time_base) | |
| try: | |
| container.seek(int(max(0.0, start_time - 0.5) / time_base), stream=stream, backward=True) | |
| except Exception: | |
| container.seek(0) | |
| frames: Dict[int, "np.ndarray"] = {} | |
| for frame in container.decode(video=0): | |
| local = int(round(float(frame.pts * time_base) * fps)) - offset | |
| if local < 0: | |
| continue | |
| if local >= num_frames: | |
| break | |
| if local not in frames: | |
| frames[local] = frame.to_ndarray(format="rgb24") | |
| container.close() | |
| if len(frames) != num_frames: | |
| missing = [i for i in range(num_frames) if i not in frames] | |
| raise SystemExit(f"{video_path.name}: missing frames {missing[:5]} ...") | |
| return [frames[i] for i in range(num_frames)] | |
| def window_indices(end_frame: int, n_obs_steps: int, frame_gap: int, ep_start: int = 0): | |
| """The frame window act_pri sees, matching FrameGapLeRobotDataset(stage_model=True).""" | |
| idx = end_frame | |
| steps_between = n_obs_steps - 1 | |
| if idx - ep_start >= frame_gap * n_obs_steps: | |
| mid = [idx - frame_gap * j for j in range(steps_between, 0, -1)] | |
| else: | |
| span = idx - ep_start | |
| mid = [ep_start + round(span * k / n_obs_steps) for k in range(1, n_obs_steps)] | |
| frames = mid + [idx] | |
| for i in range(1, len(frames)): | |
| frames[i] = max(frames[i], frames[i - 1]) | |
| return frames | |
| def encode_episode( | |
| encoder: FrozenSiglipEncoder, frames_per_cam: Dict[str, list], batch_size: int, device | |
| ) -> Dict[str, torch.Tensor]: | |
| """SigLIP-embed every frame once; windows are then assembled for free.""" | |
| out: Dict[str, torch.Tensor] = {} | |
| for cam, frames in frames_per_cam.items(): | |
| embeddings = [] | |
| for start in range(0, len(frames), batch_size): | |
| chunk = np.stack(frames[start : start + batch_size]) | |
| tensor = torch.from_numpy(chunk).permute(0, 3, 1, 2).float().div_(255.0) | |
| embeddings.append(encoder.encode_image(tensor.to(device)).float().cpu()) | |
| out[cam] = torch.cat(embeddings) | |
| return out | |
| def main() -> None: | |
| ap = argparse.ArgumentParser( | |
| description="Run a trained act_pri checkpoint over a dataset.", | |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter, | |
| ) | |
| ap.add_argument("--dataset-root", required=True, help="a LeRobot dataset directory") | |
| ap.add_argument("--out-dir", default="act_pri_preds", help="prediction store root") | |
| ap.add_argument("--session", default=None, help="sub-directory name (default: dataset dir name)") | |
| ap.add_argument( | |
| "--checkpoint", action="append", required=True, | |
| help="checkpoint path; repeat to score several models over the same frames", | |
| ) | |
| ap.add_argument( | |
| "--name", action="append", default=None, | |
| help="display name per --checkpoint (default: the file stem)", | |
| ) | |
| ap.add_argument("--state-norm", required=True, help="state_norm.json used at training time") | |
| ap.add_argument("--vision-ckpt", default="google/siglip2-base-patch16-224") | |
| ap.add_argument("--n-obs-steps", type=int, default=6) | |
| ap.add_argument("--frame-gap", type=int, default=30, | |
| help="must match cfg.model.frame_gap used at training time") | |
| ap.add_argument("--d-model", type=int, default=512) | |
| ap.add_argument("--n-layers", type=int, default=4) | |
| ap.add_argument("--n-heads", type=int, default=8) | |
| ap.add_argument("--state-dim", type=int, default=12) | |
| ap.add_argument("--num-tasks", type=int, default=22) | |
| ap.add_argument("--num-classes", type=int, default=8) | |
| ap.add_argument("--group-seconds", type=float, default=1.0, help="one prediction per this many seconds") | |
| ap.add_argument("--key-position", choices=["start", "mid", "end"], default="mid") | |
| ap.add_argument("--image-size", type=int, default=512, help="longest side of the saved review images") | |
| ap.add_argument("--jpeg-quality", type=int, default=88) | |
| ap.add_argument("--encode-batch", type=int, default=64) | |
| ap.add_argument("--episodes", default="all") | |
| ap.add_argument("--device", default="cuda") | |
| ap.add_argument("--no-images", action="store_true", help="skip writing review images") | |
| args = ap.parse_args() | |
| names = args.name or [] | |
| if len(names) < len(args.checkpoint): | |
| names += [Path(c).stem for c in args.checkpoint[len(names):]] | |
| device = torch.device(args.device if torch.cuda.is_available() else "cpu") | |
| meta = load_dataset_meta(Path(args.dataset_root)) | |
| cameras = meta.camera_keys | |
| session = args.session or meta.name | |
| out_dir = Path(args.out_dir).expanduser().resolve() / session | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| write_vocab(out_dir / "vocab.json") | |
| group_size = group_size_for(meta.fps, args.group_seconds) | |
| if args.episodes == "all": | |
| episode_ids = [e.index for e in meta.episodes] | |
| else: | |
| episode_ids = [] | |
| for part in args.episodes.split(","): | |
| if "-" in part: | |
| lo, hi = part.split("-", 1) | |
| episode_ids.extend(range(int(lo), int(hi) + 1)) | |
| elif part.strip(): | |
| episode_ids.append(int(part)) | |
| print(f"[data ] {session}: v{meta.codebase_version} {meta.fps:g} fps " | |
| f"{len(episode_ids)} episodes {meta.total_frames} frames", flush=True) | |
| print(f"[model] {len(args.checkpoint)} checkpoint(s): {names}", flush=True) | |
| encoder = FrozenSiglipEncoder(args.vision_ckpt, device) | |
| normalizer = get_normalizer_from_calculated(args.state_norm, device) | |
| models = {} | |
| for name, ckpt_path in zip(names, args.checkpoint): | |
| model = ActionTransformer( | |
| d_model=args.d_model, vis_emb_dim=768, state_dim=args.state_dim, | |
| n_layers=args.n_layers, n_heads=args.n_heads, dropout=0.0, | |
| num_cameras=len(cameras), num_tasks=args.num_tasks, num_classes=args.num_classes, | |
| ).to(device) | |
| ckpt = torch.load(ckpt_path, map_location=device, weights_only=False) | |
| model.load_state_dict(ckpt["model"]) | |
| model.eval() | |
| models[name] = model | |
| print(f"[model] {name}: {ckpt_path} (epoch {ckpt.get('epoch')})", flush=True) | |
| t0 = time.time() | |
| tally = {name: {p: 0 for p in PRIMITIVES} for name in names} | |
| off_vocab = {name: 0 for name in names} | |
| for ep_id in episode_ids: | |
| episode = meta.episode(ep_id) | |
| groups = build_groups(episode.length, group_size, args.key_position) | |
| frames_per_cam = { | |
| cam: decode_episode_frames( | |
| episode.videos[cam].path, episode.videos[cam].start_time, meta.fps, episode.length | |
| ) | |
| for cam in cameras | |
| } | |
| if not args.no_images: | |
| from PIL import Image | |
| dest = images_dir(out_dir, ep_id) | |
| dest.mkdir(parents=True, exist_ok=True) | |
| for cam in cameras: | |
| short = cam.split(".")[-1] | |
| for g in groups: | |
| img = Image.fromarray(frames_per_cam[cam][g.key_frame]) | |
| if args.image_size and max(img.size) > args.image_size: | |
| scale = args.image_size / max(img.size) | |
| img = img.resize((int(img.width * scale), int(img.height * scale)), Image.BICUBIC) | |
| img.save(dest / f"g{g.group_index:04d}_{short}.jpg", quality=args.jpeg_quality) | |
| embeddings = encode_episode(encoder, frames_per_cam, args.encode_batch, device) | |
| del frames_per_cam | |
| state = torch.from_numpy(load_source_table(meta, episode)["state"]).float() | |
| windows = [window_indices(g.key_frame, args.n_obs_steps, args.frame_gap) for g in groups] | |
| window_tensor = torch.tensor(windows, dtype=torch.long) # (G, T) | |
| img_emb = torch.stack( | |
| [embeddings[cam][window_tensor.reshape(-1)].view(len(groups), args.n_obs_steps, -1) | |
| for cam in cameras], | |
| dim=1, | |
| ).to(device) # (G, N, T, D) | |
| state_win = state[window_tensor.reshape(-1)].view(len(groups), args.n_obs_steps, -1).to(device) | |
| state_win = normalizer.normalize(state_win) | |
| lens = torch.full((len(groups),), args.n_obs_steps, dtype=torch.int32, device=device) | |
| per_model: Dict[str, List[dict]] = {} | |
| with torch.no_grad(): | |
| for name, model in models.items(): | |
| logits, _ = model(img_emb, state_win, lens) | |
| probs = torch.softmax(logits.float(), dim=-1).cpu() | |
| picks = probs.argmax(dim=-1).tolist() | |
| rows = [] | |
| for gi, pick in enumerate(picks): | |
| label = SARM2_TASK_LIST[pick] | |
| if label not in PRIMITIVES: | |
| off_vocab[name] += 1 | |
| else: | |
| tally[name][label] += 1 | |
| rows.append({ | |
| "label": label, | |
| "index": int(pick), | |
| "scores": {p: round(float(probs[gi, TASK_TO_INDEX[p]]), 5) for p in PRIMITIVES}, | |
| "confidence": round(float(probs[gi, pick]), 5), | |
| }) | |
| per_model[name] = rows | |
| primary = names[0] | |
| records = [] | |
| for gi, g in enumerate(groups): | |
| pred = per_model[primary][gi] | |
| records.append({ | |
| "group_index": g.group_index, | |
| "start_frame": g.start_frame, | |
| "end_frame": g.end_frame, | |
| "num_frames": g.num_frames, | |
| "start_time": round(g.start_frame / meta.fps, 3), | |
| "end_time": round((g.end_frame + 1) / meta.fps, 3), | |
| "key_frame": g.key_frame, | |
| "key_time": round(g.key_frame / meta.fps, 3), | |
| "pred": pred["label"], | |
| "pred_index": pred["index"], | |
| "pred_raw": "", | |
| "scores": pred["scores"], | |
| "label": pred["label"], | |
| "label_index": pred["index"], | |
| "modified": False, | |
| "reviewed": False, | |
| "models": {name: per_model[name][gi] for name in names}, | |
| "images": { | |
| cam: f"images/episode_{ep_id:06d}/g{g.group_index:04d}_{cam.split('.')[-1]}.jpg" | |
| for cam in cameras | |
| }, | |
| }) | |
| payload = { | |
| "dataset_root": str(meta.root), | |
| "dataset_name": session, | |
| "codebase_version": meta.codebase_version, | |
| "episode_index": ep_id, | |
| "task": episode.task, | |
| "fps": meta.fps, | |
| "num_frames": episode.length, | |
| "dataset_from_index": episode.from_index, | |
| "dataset_to_index": episode.to_index, | |
| "group_size": group_size, | |
| "group_seconds": group_size / meta.fps, | |
| "key_position": args.key_position, | |
| "cameras": list(cameras), | |
| "model": primary, | |
| "model_names": list(names), | |
| "checkpoints": {n: str(c) for n, c in zip(names, args.checkpoint)}, | |
| "n_obs_steps": args.n_obs_steps, | |
| "frame_gap": args.frame_gap, | |
| "primitives": PRIMITIVES, | |
| "primitive_index": {p: TASK_TO_INDEX[p] for p in PRIMITIVES}, | |
| "primitive_gate": {p: task_to_class_id(p) for p in PRIMITIVES}, | |
| "created": time.strftime("%Y-%m-%dT%H:%M:%S"), | |
| "groups": records, | |
| } | |
| save_episode_labels(out_dir, payload) | |
| write_per_frame(out_dir, payload) | |
| agree = sum( | |
| 1 for gi in range(len(groups)) | |
| if len({per_model[n][gi]["label"] for n in names}) == 1 | |
| ) | |
| print(f"[ep {ep_id:03d}] {episode.length} frames -> {len(groups)} groups | " | |
| f"models agree on {agree}/{len(groups)} | {time.time() - t0:.0f}s", flush=True) | |
| del img_emb, state_win, embeddings | |
| torch.cuda.empty_cache() | |
| print(f"\n[done] {time.time() - t0:.0f}s", flush=True) | |
| for name in names: | |
| total = sum(tally[name].values()) + off_vocab[name] | |
| print(f" {name}: {tally[name]}" + (f" off-vocabulary: {off_vocab[name]}/{total}" if off_vocab[name] else "")) | |
| print(f"\nReview with: python tools/annotate_act_pri_ui.py --labels-dir {Path(args.out_dir).resolve()} --dataset-root {Path(args.dataset_root).resolve()}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.6 kB
- Xet hash:
- 8b06e750934f8fccb26816559ca98d7df9b0c9c64c2dc338b205a21b0b00b795
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.