Nadeem-Shoukath115113/opensarm / tools /visualise_sarm2.py
Nadeem-Shoukath115113's picture
download
raw
12.1 kB
#!/usr/bin/env python
"""Plot SARM2's predicted progress over time for a few episodes.
Mirrors the inference path in `sarm2_ws.eval_video` exactly: SigLIP embeds the
[episode-start] + n_obs_steps window, act_pri predicts the primitive (which both
supplies the value head's primitive embedding and selects its MoE gate), and the
MMoE value head predicts `steps_to_go`. Progress is 1 - steps_to_go.
python tools/visualise_sarm2.py --episodes 5 --out sarm2_progress.png
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
import torch
import torch.nn.functional as F
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from omegaconf import OmegaConf # noqa: E402
from models.action_estimator import ActionTransformer # noqa: E402
from models.moe_deco_gate_reward_model import RewardTransformer # noqa: E402
from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402
from tools.act_pri_common import SARM2_TASK_LIST, load_dataset_meta, task_to_class_id # noqa: E402
from tools.build_act_pri_clips import load_source_table # noqa: E402
from tools.predict_act_pri import decode_episode_frames, encode_episode # noqa: E402
from utils.data_utils import get_valid_episodes, split_train_eval_episodes # noqa: E402
from utils.train_utils import get_normalizer_from_calculated # noqa: E402
def sequence_indices(idx: int, n_obs_steps: int, frame_gap: int, ep_end: int) -> list[int]:
"""[episode start] + n_obs_steps mid frames ending at idx (get_frame_indices, stage_model=False)."""
idx = max(0, min(idx, ep_end))
if n_obs_steps == 0:
return [idx]
steps_between = n_obs_steps - 1
if steps_between <= 0:
return [0, idx]
if idx >= frame_gap * n_obs_steps:
mid = [idx - frame_gap * j for j in range(steps_between, 0, -1)]
else:
mid = [round(idx * k / n_obs_steps) for k in range(1, n_obs_steps)]
frames = [0] + mid + [idx]
for i in range(1, len(frames)):
frames[i] = max(frames[i], frames[i - 1])
return frames
def load_is_policy(meta, episode):
"""DAgger intervention flag per frame, if the dataset carries one."""
import pyarrow as pa
import pyarrow.parquet as pq
info = json.loads((meta.root / "meta" / "info.json").read_text())
if "is_policy" not in info.get("features", {}):
return None
tpl = info["data_path"]
if "{file_index" in tpl or meta.codebase_version.startswith("v3"):
tables = [pq.read_table(f) for f in sorted((meta.root / "data").rglob("*.parquet"))]
table = pa.concat_tables(tables) if len(tables) > 1 else tables[0]
table = table.slice(episode.from_index, episode.length)
else:
table = pq.read_table(meta.root / tpl.format(
episode_chunk=episode.index // int(info.get("chunks_size", 1000)),
episode_index=episode.index))
v = np.asarray(table.column("is_policy").to_pylist())
return (v[:, 0] if v.ndim > 1 else v).astype(float)
def main() -> None:
ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
ap.add_argument("--config", default="config/sarm2.yaml")
ap.add_argument("--run", default=None, help="run dir holding checkpoints (default: newest sarm2 run)")
ap.add_argument("--reward-ckpt", default=None, help="explicit reward checkpoint (default: <run>/reward_best.pt)")
ap.add_argument("--act-pri-ckpt", default=None,
help="explicit act_pri checkpoint (default: <run>/act_pri_best.pt, "
"falling back to cfg.model.act_pri_model_path when act_pri was frozen)")
ap.add_argument("--dataset-root", default=None, help="override the dataset to score")
ap.add_argument("--episodes", type=int, default=5, help="how many episodes to plot")
ap.add_argument("--episode-ids", default=None, help="explicit comma-separated ids")
ap.add_argument("--split", choices=["val", "train", "all"], default="val")
ap.add_argument("--stride", type=int, default=5, help="predict every Nth frame")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--out", default="sarm2_progress.png")
ap.add_argument("--device", default="cuda")
args = ap.parse_args()
cfg = OmegaConf.load(args.config).cfg
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
cams = list(cfg.general.camera_names)
n_obs, gap = cfg.model.n_obs_steps, cfg.model.frame_gap
run = Path(args.run) if args.run else sorted(
Path("outputs").glob("*/*/sarm2/*/checkpoints"), key=lambda p: p.stat().st_mtime
)[-1]
ckpt_dir = run if run.name == "checkpoints" else run / "sarm2" / cfg.general.task_name / "checkpoints"
reward_ckpt = Path(args.reward_ckpt) if args.reward_ckpt else ckpt_dir / "reward_best.pt"
act_pri_ckpt = Path(args.act_pri_ckpt) if args.act_pri_ckpt else ckpt_dir / "act_pri_best.pt"
if not act_pri_ckpt.exists():
act_pri_ckpt = Path(cfg.model.act_pri_model_path) # frozen runs keep no copy in the run dir
print(f"[ckpt] reward: {reward_ckpt}")
print(f"[ckpt] act_pri: {act_pri_ckpt}")
encoder = FrozenSiglipEncoder(cfg.encoders.vision_ckpt, device)
normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, device)
task_list = list(cfg.model.task_list)
act_pri_feature = encoder.encode_text(task_list).to(device)
gate_of = torch.tensor([task_to_class_id(t) for t in task_list], device=device)
act_pri_model = ActionTransformer(
d_model=cfg.model.act_pri.d_model, vis_emb_dim=768, state_dim=cfg.model.act_pri.state_dim,
n_layers=cfg.model.act_pri.n_layers, n_heads=cfg.model.act_pri.n_heads, dropout=0.0,
num_cameras=len(cams), num_tasks=cfg.model.num_tasks, num_classes=cfg.model.num_class,
).to(device)
act_pri_model.load_state_dict(torch.load(act_pri_ckpt, map_location=device,
weights_only=False)["model"])
act_pri_model.eval()
moe = cfg.model.moe
reward_model = RewardTransformer(
d_model=cfg.model.d_model, vis_emb_dim=768, text_emb_dim=768, state_dim=cfg.model.state_dim,
n_layers=cfg.model.n_layers, n_heads=cfg.model.n_heads, dropout=0.0, num_cameras=len(cams),
num_gates=cfg.model.num_class, num_experts=moe.num_experts, top_k=moe.top_k,
gate_units=list(moe.gate_units), gate_act=moe.gate_act, gate_dropout=moe.gate_dropout,
expert_units=list(moe.expert_units), expert_act=moe.expert_act, expert_dropout=moe.expert_dropout,
lambda_balance=moe.lambda_balance, lambda_entropy=moe.lambda_entropy,
lambda_importance=moe.lambda_importance,
).to(device)
reward_model.load_state_dict(torch.load(reward_ckpt, map_location=device,
weights_only=False)["model"])
reward_model.eval()
root = Path(args.dataset_root) if args.dataset_root else (
Path.home() / ".cache" / "huggingface" / "lerobot" / cfg.general.repo_id)
meta = load_dataset_meta(root)
if args.episode_ids:
chosen = [int(x) for x in args.episode_ids.split(",")]
else:
valid = get_valid_episodes(cfg.general.repo_id) if not args.dataset_root else [
e.index for e in meta.episodes]
tr, va = split_train_eval_episodes(valid, 1 - cfg.train.val_portion, seed=cfg.general.seed)
pool = {"val": va, "train": tr, "all": valid}[args.split]
rng = np.random.default_rng(args.seed)
chosen = sorted(rng.choice(pool, size=min(args.episodes, len(pool)), replace=False).tolist())
print(f"[eval] {args.split} episodes: {chosen}")
results = []
for ep_id in chosen:
episode = meta.episode(ep_id)
n = episode.length
frames = {c: decode_episode_frames(episode.videos[c].path, episode.videos[c].start_time,
meta.fps, n) for c in cams}
thumb = frames[cams[0]][n // 2]
emb = encode_episode(encoder, frames, 96, device)
del frames
state = torch.from_numpy(load_source_table(meta, episode)["state"]).float()
is_policy = load_is_policy(meta, episode)
query = list(range(0, n, args.stride))
if query[-1] != n - 1:
query.append(n - 1)
windows = torch.tensor([sequence_indices(t, n_obs, gap, n - 1) for t in query], dtype=torch.long)
T = windows.shape[1]
inst = encoder.encode_text([episode.task]).to(device)
preds = []
with torch.no_grad():
for lo in range(0, len(query), 256):
w = windows[lo:lo + 256]
b = w.shape[0]
img = torch.stack([emb[c][w.reshape(-1)].view(b, T, -1) for c in cams], 1).to(device)
st = normalizer.normalize(state[w.reshape(-1)].view(b, T, -1).to(device))
lens = torch.full((b,), T, dtype=torch.int32, device=device)
# act_pri sees the mid window only (_slice_act_pri_inputs)
ap_logits, _ = act_pri_model(img[:, :, 1:n_obs + 1, :], st[:, 1:n_obs + 1, :],
torch.full((b,), n_obs, dtype=torch.int32, device=device))
ap_idx = F.softmax(ap_logits, -1).argmax(-1)
out = reward_model(img, act_pri_feature[ap_idx], inst.expand(b, -1), st, lens,
gate_idx=gate_of[ap_idx])
reward_pred = out[0] if isinstance(out, (tuple, list)) else out
preds.append(torch.clip(reward_pred, 0, 1)[:, -1].float().cpu())
steps_to_go = torch.cat(preds).numpy()
results.append(dict(ep=ep_id, task=episode.task, thumb=thumb,
t=np.array(query) / meta.fps,
progress=1.0 - steps_to_go,
gt=np.array(query) / (n - 1),
is_policy=is_policy, fps=meta.fps))
print(f" ep {ep_id:03d}: {n} frames, {len(query)} predictions", flush=True)
# ---- plot ----
k = len(results)
fig, axes = plt.subplots(k, 2, figsize=(9, 2.6 * k),
gridspec_kw={"width_ratios": [1, 1.25]})
axes = np.atleast_2d(axes)
for i, r in enumerate(results):
ax_img, ax = axes[i, 0], axes[i, 1]
ax_img.imshow(r["thumb"]); ax_img.axis("off")
frac = f" - policy {r['is_policy'].mean()*100:.0f}%" if r.get("is_policy") is not None else ""
ax_img.set_title(f"episode {r['ep']}{frac}", fontsize=10, fontweight="bold")
ip = r.get("is_policy")
if ip is not None:
edges = np.flatnonzero(np.diff(np.concatenate(([0], ip, [0])))) / r["fps"]
for lo, hi in zip(edges[0::2], edges[1::2]):
ax.axvspan(lo, hi, color="#ffd28a", alpha=0.55, lw=0, zorder=0)
ax.axvspan(np.nan, np.nan, color="#ffd28a", alpha=0.55, lw=0, label="policy in control")
ax.plot(r["t"], r["gt"], color="#b0b0b0", lw=1.6, ls="--", label="ground truth t/(T-1)")
ax.plot(r["t"], r["progress"], color="#20c4c0", lw=2.4, label="SARM2 predicted")
ax.set_ylim(-0.02, 1.05); ax.set_xlim(r["t"][0], r["t"][-1])
ax.set_ylabel("Progress", fontsize=11, fontweight="bold")
ax.grid(alpha=0.3, color="white", lw=1.2)
ax.set_facecolor("#eef0f4")
for s in ax.spines.values(): s.set_visible(False)
if i == 0: ax.legend(fontsize=8, loc="upper left", framealpha=0.9)
if i == k - 1: ax.set_xlabel("Time (s)", fontsize=11, fontweight="bold")
fig.suptitle(results[0]["task"], fontsize=14, fontweight="bold")
fig.tight_layout(rect=[0, 0, 1, 0.98])
fig.savefig(args.out, dpi=140)
print(f"\n[out] {Path(args.out).resolve()}")
for r in results:
err = np.abs(r["progress"] - r["gt"])
print(f" ep {r['ep']:03d}: mean|pred-gt| = {err.mean():.3f}, final progress = {r['progress'][-1]:.3f}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
12.1 kB
·
Xet hash:
c16ca6d0398a8b9837b4f1272d21ac869c25c4cdc3b732545a7c0fc51ceadc68

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.