Buckets:
| #!/usr/bin/env python | |
| """Plot SARM2's predicted progress over time for a few episodes. | |
| Mirrors the inference path in `sarm2_ws.eval_video` exactly: SigLIP embeds the | |
| [episode-start] + n_obs_steps window, act_pri predicts the primitive (which both | |
| supplies the value head's primitive embedding and selects its MoE gate), and the | |
| MMoE value head predicts `steps_to_go`. Progress is 1 - steps_to_go. | |
| python tools/visualise_sarm2.py --episodes 5 --out sarm2_progress.png | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt # noqa: E402 | |
| from omegaconf import OmegaConf # noqa: E402 | |
| from models.action_estimator import ActionTransformer # noqa: E402 | |
| from models.moe_deco_gate_reward_model import RewardTransformer # noqa: E402 | |
| from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402 | |
| from tools.act_pri_common import SARM2_TASK_LIST, load_dataset_meta, task_to_class_id # noqa: E402 | |
| from tools.build_act_pri_clips import load_source_table # noqa: E402 | |
| from tools.predict_act_pri import decode_episode_frames, encode_episode # noqa: E402 | |
| from utils.data_utils import get_valid_episodes, split_train_eval_episodes # noqa: E402 | |
| from utils.train_utils import get_normalizer_from_calculated # noqa: E402 | |
| def sequence_indices(idx: int, n_obs_steps: int, frame_gap: int, ep_end: int) -> list[int]: | |
| """[episode start] + n_obs_steps mid frames ending at idx (get_frame_indices, stage_model=False).""" | |
| idx = max(0, min(idx, ep_end)) | |
| if n_obs_steps == 0: | |
| return [idx] | |
| steps_between = n_obs_steps - 1 | |
| if steps_between <= 0: | |
| return [0, idx] | |
| if idx >= frame_gap * n_obs_steps: | |
| mid = [idx - frame_gap * j for j in range(steps_between, 0, -1)] | |
| else: | |
| mid = [round(idx * k / n_obs_steps) for k in range(1, n_obs_steps)] | |
| frames = [0] + mid + [idx] | |
| for i in range(1, len(frames)): | |
| frames[i] = max(frames[i], frames[i - 1]) | |
| return frames | |
| def load_is_policy(meta, episode): | |
| """DAgger intervention flag per frame, if the dataset carries one.""" | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| info = json.loads((meta.root / "meta" / "info.json").read_text()) | |
| if "is_policy" not in info.get("features", {}): | |
| return None | |
| tpl = info["data_path"] | |
| if "{file_index" in tpl or meta.codebase_version.startswith("v3"): | |
| tables = [pq.read_table(f) for f in sorted((meta.root / "data").rglob("*.parquet"))] | |
| table = pa.concat_tables(tables) if len(tables) > 1 else tables[0] | |
| table = table.slice(episode.from_index, episode.length) | |
| else: | |
| table = pq.read_table(meta.root / tpl.format( | |
| episode_chunk=episode.index // int(info.get("chunks_size", 1000)), | |
| episode_index=episode.index)) | |
| v = np.asarray(table.column("is_policy").to_pylist()) | |
| return (v[:, 0] if v.ndim > 1 else v).astype(float) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) | |
| ap.add_argument("--config", default="config/sarm2.yaml") | |
| ap.add_argument("--run", default=None, help="run dir holding checkpoints (default: newest sarm2 run)") | |
| ap.add_argument("--reward-ckpt", default=None, help="explicit reward checkpoint (default: <run>/reward_best.pt)") | |
| ap.add_argument("--act-pri-ckpt", default=None, | |
| help="explicit act_pri checkpoint (default: <run>/act_pri_best.pt, " | |
| "falling back to cfg.model.act_pri_model_path when act_pri was frozen)") | |
| ap.add_argument("--dataset-root", default=None, help="override the dataset to score") | |
| ap.add_argument("--episodes", type=int, default=5, help="how many episodes to plot") | |
| ap.add_argument("--episode-ids", default=None, help="explicit comma-separated ids") | |
| ap.add_argument("--split", choices=["val", "train", "all"], default="val") | |
| ap.add_argument("--stride", type=int, default=5, help="predict every Nth frame") | |
| ap.add_argument("--seed", type=int, default=0) | |
| ap.add_argument("--out", default="sarm2_progress.png") | |
| ap.add_argument("--device", default="cuda") | |
| args = ap.parse_args() | |
| cfg = OmegaConf.load(args.config).cfg | |
| device = torch.device(args.device if torch.cuda.is_available() else "cpu") | |
| cams = list(cfg.general.camera_names) | |
| n_obs, gap = cfg.model.n_obs_steps, cfg.model.frame_gap | |
| run = Path(args.run) if args.run else sorted( | |
| Path("outputs").glob("*/*/sarm2/*/checkpoints"), key=lambda p: p.stat().st_mtime | |
| )[-1] | |
| ckpt_dir = run if run.name == "checkpoints" else run / "sarm2" / cfg.general.task_name / "checkpoints" | |
| reward_ckpt = Path(args.reward_ckpt) if args.reward_ckpt else ckpt_dir / "reward_best.pt" | |
| act_pri_ckpt = Path(args.act_pri_ckpt) if args.act_pri_ckpt else ckpt_dir / "act_pri_best.pt" | |
| if not act_pri_ckpt.exists(): | |
| act_pri_ckpt = Path(cfg.model.act_pri_model_path) # frozen runs keep no copy in the run dir | |
| print(f"[ckpt] reward: {reward_ckpt}") | |
| print(f"[ckpt] act_pri: {act_pri_ckpt}") | |
| encoder = FrozenSiglipEncoder(cfg.encoders.vision_ckpt, device) | |
| normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, device) | |
| task_list = list(cfg.model.task_list) | |
| act_pri_feature = encoder.encode_text(task_list).to(device) | |
| gate_of = torch.tensor([task_to_class_id(t) for t in task_list], device=device) | |
| act_pri_model = ActionTransformer( | |
| d_model=cfg.model.act_pri.d_model, vis_emb_dim=768, state_dim=cfg.model.act_pri.state_dim, | |
| n_layers=cfg.model.act_pri.n_layers, n_heads=cfg.model.act_pri.n_heads, dropout=0.0, | |
| num_cameras=len(cams), num_tasks=cfg.model.num_tasks, num_classes=cfg.model.num_class, | |
| ).to(device) | |
| act_pri_model.load_state_dict(torch.load(act_pri_ckpt, map_location=device, | |
| weights_only=False)["model"]) | |
| act_pri_model.eval() | |
| moe = cfg.model.moe | |
| reward_model = RewardTransformer( | |
| d_model=cfg.model.d_model, vis_emb_dim=768, text_emb_dim=768, state_dim=cfg.model.state_dim, | |
| n_layers=cfg.model.n_layers, n_heads=cfg.model.n_heads, dropout=0.0, num_cameras=len(cams), | |
| num_gates=cfg.model.num_class, num_experts=moe.num_experts, top_k=moe.top_k, | |
| gate_units=list(moe.gate_units), gate_act=moe.gate_act, gate_dropout=moe.gate_dropout, | |
| expert_units=list(moe.expert_units), expert_act=moe.expert_act, expert_dropout=moe.expert_dropout, | |
| lambda_balance=moe.lambda_balance, lambda_entropy=moe.lambda_entropy, | |
| lambda_importance=moe.lambda_importance, | |
| ).to(device) | |
| reward_model.load_state_dict(torch.load(reward_ckpt, map_location=device, | |
| weights_only=False)["model"]) | |
| reward_model.eval() | |
| root = Path(args.dataset_root) if args.dataset_root else ( | |
| Path.home() / ".cache" / "huggingface" / "lerobot" / cfg.general.repo_id) | |
| meta = load_dataset_meta(root) | |
| if args.episode_ids: | |
| chosen = [int(x) for x in args.episode_ids.split(",")] | |
| else: | |
| valid = get_valid_episodes(cfg.general.repo_id) if not args.dataset_root else [ | |
| e.index for e in meta.episodes] | |
| tr, va = split_train_eval_episodes(valid, 1 - cfg.train.val_portion, seed=cfg.general.seed) | |
| pool = {"val": va, "train": tr, "all": valid}[args.split] | |
| rng = np.random.default_rng(args.seed) | |
| chosen = sorted(rng.choice(pool, size=min(args.episodes, len(pool)), replace=False).tolist()) | |
| print(f"[eval] {args.split} episodes: {chosen}") | |
| results = [] | |
| for ep_id in chosen: | |
| episode = meta.episode(ep_id) | |
| n = episode.length | |
| frames = {c: decode_episode_frames(episode.videos[c].path, episode.videos[c].start_time, | |
| meta.fps, n) for c in cams} | |
| thumb = frames[cams[0]][n // 2] | |
| emb = encode_episode(encoder, frames, 96, device) | |
| del frames | |
| state = torch.from_numpy(load_source_table(meta, episode)["state"]).float() | |
| is_policy = load_is_policy(meta, episode) | |
| query = list(range(0, n, args.stride)) | |
| if query[-1] != n - 1: | |
| query.append(n - 1) | |
| windows = torch.tensor([sequence_indices(t, n_obs, gap, n - 1) for t in query], dtype=torch.long) | |
| T = windows.shape[1] | |
| inst = encoder.encode_text([episode.task]).to(device) | |
| preds = [] | |
| with torch.no_grad(): | |
| for lo in range(0, len(query), 256): | |
| w = windows[lo:lo + 256] | |
| b = w.shape[0] | |
| img = torch.stack([emb[c][w.reshape(-1)].view(b, T, -1) for c in cams], 1).to(device) | |
| st = normalizer.normalize(state[w.reshape(-1)].view(b, T, -1).to(device)) | |
| lens = torch.full((b,), T, dtype=torch.int32, device=device) | |
| # act_pri sees the mid window only (_slice_act_pri_inputs) | |
| ap_logits, _ = act_pri_model(img[:, :, 1:n_obs + 1, :], st[:, 1:n_obs + 1, :], | |
| torch.full((b,), n_obs, dtype=torch.int32, device=device)) | |
| ap_idx = F.softmax(ap_logits, -1).argmax(-1) | |
| out = reward_model(img, act_pri_feature[ap_idx], inst.expand(b, -1), st, lens, | |
| gate_idx=gate_of[ap_idx]) | |
| reward_pred = out[0] if isinstance(out, (tuple, list)) else out | |
| preds.append(torch.clip(reward_pred, 0, 1)[:, -1].float().cpu()) | |
| steps_to_go = torch.cat(preds).numpy() | |
| results.append(dict(ep=ep_id, task=episode.task, thumb=thumb, | |
| t=np.array(query) / meta.fps, | |
| progress=1.0 - steps_to_go, | |
| gt=np.array(query) / (n - 1), | |
| is_policy=is_policy, fps=meta.fps)) | |
| print(f" ep {ep_id:03d}: {n} frames, {len(query)} predictions", flush=True) | |
| # ---- plot ---- | |
| k = len(results) | |
| fig, axes = plt.subplots(k, 2, figsize=(9, 2.6 * k), | |
| gridspec_kw={"width_ratios": [1, 1.25]}) | |
| axes = np.atleast_2d(axes) | |
| for i, r in enumerate(results): | |
| ax_img, ax = axes[i, 0], axes[i, 1] | |
| ax_img.imshow(r["thumb"]); ax_img.axis("off") | |
| frac = f" - policy {r['is_policy'].mean()*100:.0f}%" if r.get("is_policy") is not None else "" | |
| ax_img.set_title(f"episode {r['ep']}{frac}", fontsize=10, fontweight="bold") | |
| ip = r.get("is_policy") | |
| if ip is not None: | |
| edges = np.flatnonzero(np.diff(np.concatenate(([0], ip, [0])))) / r["fps"] | |
| for lo, hi in zip(edges[0::2], edges[1::2]): | |
| ax.axvspan(lo, hi, color="#ffd28a", alpha=0.55, lw=0, zorder=0) | |
| ax.axvspan(np.nan, np.nan, color="#ffd28a", alpha=0.55, lw=0, label="policy in control") | |
| ax.plot(r["t"], r["gt"], color="#b0b0b0", lw=1.6, ls="--", label="ground truth t/(T-1)") | |
| ax.plot(r["t"], r["progress"], color="#20c4c0", lw=2.4, label="SARM2 predicted") | |
| ax.set_ylim(-0.02, 1.05); ax.set_xlim(r["t"][0], r["t"][-1]) | |
| ax.set_ylabel("Progress", fontsize=11, fontweight="bold") | |
| ax.grid(alpha=0.3, color="white", lw=1.2) | |
| ax.set_facecolor("#eef0f4") | |
| for s in ax.spines.values(): s.set_visible(False) | |
| if i == 0: ax.legend(fontsize=8, loc="upper left", framealpha=0.9) | |
| if i == k - 1: ax.set_xlabel("Time (s)", fontsize=11, fontweight="bold") | |
| fig.suptitle(results[0]["task"], fontsize=14, fontweight="bold") | |
| fig.tight_layout(rect=[0, 0, 1, 0.98]) | |
| fig.savefig(args.out, dpi=140) | |
| print(f"\n[out] {Path(args.out).resolve()}") | |
| for r in results: | |
| err = np.abs(r["progress"] - r["gt"]) | |
| print(f" ep {r['ep']:03d}: mean|pred-gt| = {err.mean():.3f}, final progress = {r['progress'][-1]:.3f}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 12.1 kB
- Xet hash:
- c16ca6d0398a8b9837b4f1272d21ac869c25c4cdc3b732545a7c0fc51ceadc68
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.