Nadeem-Shoukath115113/opensarm / tools /make_sarm2_video.py
Nadeem-Shoukath115113's picture
download
raw
20.3 kB
#!/usr/bin/env python
"""Render side-by-side progress videos with the repo's own `produce_video`.
Runs SARM2 on every frame of an episode, writes the arrays `produce_video`
expects (pred.npy / gt.npy), extracts that episode's camera view to an mp4, and
calls `utils.make_demo_video.produce_video` to combine them.
python tools/make_sarm2_video.py --dataset-root <dir> --episodes 3
"""
from __future__ import annotations
import argparse
import sys
from fractions import Fraction
from pathlib import Path
import numpy as np
import pyarrow as pa
from pyarrow import concat_tables as pa_concat_tables
import torch
import torch.nn.functional as F
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from omegaconf import OmegaConf # noqa: E402
from models.action_estimator import ActionTransformer # noqa: E402
from models.moe_deco_gate_reward_model import RewardTransformer # noqa: E402
from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402
from tools.act_pri_common import load_dataset_meta, task_to_class_id # noqa: E402
from tools.build_act_pri_clips import load_source_table # noqa: E402
from tools.predict_act_pri import decode_episode_frames, encode_episode # noqa: E402
from tools.visualise_sarm2 import sequence_indices # noqa: E402
from utils.data_utils import get_valid_episodes, split_train_eval_episodes # noqa: E402
from utils.make_demo_video import produce_video # noqa: E402
from utils.train_utils import get_normalizer_from_calculated # noqa: E402
def write_mp4(frames, path: Path, fps: int) -> None:
import av
path.parent.mkdir(parents=True, exist_ok=True)
container = av.open(str(path), mode="w")
stream = container.add_stream("libx264", rate=int(fps))
stream.width, stream.height = frames[0].shape[1], frames[0].shape[0]
stream.pix_fmt = "yuv420p"
stream.time_base = Fraction(1, int(fps))
stream.options = {"crf": "20", "preset": "veryfast"}
for i, arr in enumerate(frames):
f = av.VideoFrame.from_ndarray(arr, format="rgb24").reformat(format="yuv420p")
f.pts = i
f.time_base = Fraction(1, int(fps))
for pkt in stream.encode(f):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
container.close()
def render_progressive(frames, series, fps, out_path: Path, title: str, ylim=None, spans=None):
"""Camera view + a progress plot that draws itself as time advances.
The repo's `draw_plot_frame` plots the whole curve on every frame and marks the
current step with a vertical line. Here the line is drawn only up to the current
frame, so the curve grows with the video.
ylim fixes every panel's y range (default: fit each series). spans is an optional list of
(start_s, end_s, colour) shaded behind every panel, e.g. labelled segments.
"""
from fractions import Fraction
import av
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
# series: list of (label, values, colour) drawn as stacked panels sharing the x axis.
# The plot is rendered at a fixed readable size; the camera is scaled to match its
# height. Sizing it from the camera (224px here) made the fonts collide.
n = len(series[0][1])
t = np.arange(n) / fps
dpi = 100
plot_h_px = max(420, frames[0].shape[0])
fig, axes = plt.subplots(len(series), 1, sharex=True,
figsize=(plot_h_px * 1.45 / dpi, plot_h_px / dpi), dpi=dpi)
axes = np.atleast_1d(axes)
fig.patch.set_facecolor("white")
lines, cursors = [], []
for ax, (label, values, colour) in zip(axes, series):
ax.set_facecolor("#eef0f4")
lo, hi = (float(np.min(values)), float(np.max(values))) if ylim is None else ylim
pad = 0.08 * max(1e-6, hi - lo)
ax.set_xlim(0, max(t[-1], 1e-6))
ax.set_ylim(lo - pad, hi + pad)
for s0, s1, c in spans or ():
ax.axvspan(s0, s1, color=c, lw=0, zorder=0)
ax.set_ylabel(label, fontsize=10, fontweight="bold", labelpad=6)
ax.grid(alpha=0.35, color="white", lw=1.2)
for s in ax.spines.values():
s.set_visible(False)
ax.tick_params(labelsize=8)
ax.yaxis.set_major_locator(plt.MaxNLocator(4))
if lo < 0 < hi:
ax.axhline(0, color="#b0b0b0", lw=1.0)
(ln,) = ax.plot([], [], color=colour, lw=2.4, solid_joinstyle="round")
lines.append(ln)
cursors.append(ax.axvline(0, color="#888", lw=1.3, ls=":"))
axes[-1].set_xlabel("Time (s)", fontsize=10, fontweight="bold")
if title:
short = title if len(title) <= 42 else title[:39] + "..."
fig.suptitle(short, fontsize=11, fontweight="bold")
fig.tight_layout(rect=[0.02, 0.02, 0.99, 0.93])
canvas = FigureCanvas(fig)
container = av.open(str(out_path), mode="w")
stream = None
for i in range(n):
for ln, cur, (_, values, _) in zip(lines, cursors, series):
ln.set_data(t[: i + 1], values[: i + 1])
cur.set_xdata([t[i], t[i]])
canvas.draw()
plot = np.asarray(canvas.buffer_rgba(), dtype=np.uint8).reshape(
canvas.get_width_height()[::-1] + (4,))[:, :, :3]
cam = frames[i]
if cam.shape[0] != plot.shape[0]: # match heights
from PIL import Image
new_w = int(cam.shape[1] * plot.shape[0] / cam.shape[0])
cam = np.asarray(Image.fromarray(cam).resize((new_w, plot.shape[0]), Image.BILINEAR))
combined = np.concatenate([cam, plot], axis=1)
if combined.shape[1] % 2:
combined = combined[:, :-1]
if combined.shape[0] % 2:
combined = combined[:-1, :]
if stream is None:
stream = container.add_stream("libx264", rate=int(fps))
stream.width, stream.height = combined.shape[1], combined.shape[0]
stream.pix_fmt = "yuv420p"
stream.time_base = Fraction(1, int(fps))
stream.options = {"crf": "20", "preset": "veryfast"}
vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(combined), format="rgb24").reformat(
format="yuv420p")
vf.pts = i
vf.time_base = Fraction(1, int(fps))
for pkt in stream.encode(vf):
container.mux(pkt)
for pkt in stream.encode():
container.mux(pkt)
container.close()
plt.close(fig)
def plot_series(progress, show_delta: bool):
"""Panels for render_progressive: Progress, plus the per-frame Reward Δ when asked for."""
series = [("Progress", progress, "#20c4c0")]
if show_delta:
per_step = np.zeros_like(progress)
# progress is a cumsum from 0, so frame 0's delta is progress[0], not 0.
per_step[0] = progress[0]
per_step[1:] = np.diff(progress)
series.append(("Reward Δ", per_step, "#f0a832"))
return series
def make_videos_from_dataset(args) -> None:
"""Render the labelled dataset's own `reward` column against its own videos.
Uses load_dataset_meta so both v2.x (per-episode) and v3.0 (aggregated) layouts work.
"""
import pyarrow.parquet as pq
root = Path(args.dataset_root) if args.dataset_root else (
Path.home() / ".cache" / "huggingface" / "lerobot" / args.from_dataset)
meta = load_dataset_meta(root)
cams = list(meta.camera_keys)
show_cam = args.camera or cams[0]
all_ids = [e.index for e in meta.episodes]
if args.episode_ids:
ids = [int(x) for x in args.episode_ids.split(",")]
elif args.all:
ids = sorted(all_ids)
else:
rng = np.random.default_rng(args.seed)
ids = sorted(rng.choice(all_ids, size=min(args.episodes, len(all_ids)), replace=False).tolist())
print(f"[data] {root}\n[eval] episodes: {ids} camera: {show_cam}")
# Load every data parquet once and concat by dataset index.
parquets = sorted((root / "data").rglob("*.parquet"))
tables = [pq.read_table(p) for p in parquets]
full = tables[0] if len(tables) == 1 else pa_concat_tables(tables)
# `reward` is the per-frame DELTA (~0.002 here); `progress` is the cumulative 0->1 curve.
# Default keeps the original preference; --column progress plots the curve instead.
if args.column != "auto":
if args.column not in full.column_names:
raise SystemExit(f"dataset has no `{args.column}` column")
reward_col = args.column
else:
reward_col = "reward" if "reward" in full.column_names else (
"progress" if "progress" in full.column_names else None)
if reward_col is None:
raise SystemExit("dataset has no reward/progress column to render")
all_reward = np.array([x[0] for x in full.column(reward_col).to_pylist()], dtype=np.float32)
fps = int(round(meta.fps))
out_root = Path(args.out).resolve()
for ep_id in ids:
ep = meta.episode(ep_id)
reward = all_reward[ep.from_index:ep.to_index]
progress = reward.astype(np.float32)
name = f"episode_{ep_id:06d}"
ep_dir = out_root / name
ep_dir.mkdir(parents=True, exist_ok=True)
np.save(ep_dir / "pred.npy", progress)
np.save(ep_dir / "reward.npy", reward)
cam_frames = decode_episode_frames(
ep.videos[show_cam].path, ep.videos[show_cam].start_time, meta.fps, ep.length)
if args.repo_style:
tmp_video = ep_dir / "camera.mp4"
write_mp4(cam_frames, tmp_video, fps)
produce_video(save_dir=out_root, middle_video=tmp_video, episode_num=name,
raw_data=True, x_offset=0, frame_rate=fps)
else:
m = min(len(cam_frames), len(progress))
render_progressive(cam_frames[:m], plot_series(progress[:m], args.show_delta),
fps, ep_dir / "combined_video.mp4", title=ep.task or "")
print(f" ep {ep_id:03d}: {len(progress)} frames | progress {progress[0]:+.3f} -> "
f"{progress[-1]:+.3f} -> {ep_dir / 'combined_video.mp4'}", flush=True)
print(f"\n[out] {out_root}")
def main() -> None:
ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
ap.add_argument("--config", default="config/sarm2.yaml")
ap.add_argument("--run", default=None)
ap.add_argument("--reward-ckpt", default=None, help="explicit reward checkpoint (default: <run>/reward_best.pt)")
ap.add_argument("--act-pri-ckpt", default=None,
help="explicit act_pri checkpoint (default: <run>/act_pri_best.pt, "
"falling back to cfg.model.act_pri_model_path when act_pri was frozen)")
ap.add_argument("--dataset-root", default=None)
ap.add_argument("--episodes", type=int, default=3)
ap.add_argument("--all", action="store_true", help="render every episode in the dataset")
ap.add_argument("--episode-ids", default=None)
ap.add_argument("--split", choices=["val", "train", "all"], default="all")
ap.add_argument("--camera", default=None, help="camera to show (default: the first one)")
ap.add_argument("--cameras", default=None,
help="comma-separated camera keys to override cfg.general.camera_names "
"(order matters: must match the training arity)")
ap.add_argument("--out", default="sarm2_videos")
ap.add_argument("--smooth-window", type=int, default=0,
help="odd window size for a centred moving average over the predicted "
"progress before plotting (0 = off).")
ap.add_argument("--monotone", action="store_true",
help="enforce monotone-increasing progress via cumulative max (applied "
"after --smooth-window).")
ap.add_argument("--keep-camera", action="store_true",
help="also write the camera-only camera.mp4 next to combined_video.mp4 "
"(default: combined video only)")
ap.add_argument("--repo-style", action="store_true",
help="use the repo's draw_plot_frame (whole curve + moving line) instead of "
"drawing the curve progressively")
ap.add_argument("--from-dataset", default=None,
help="repo id of a SPIRAL-labelled dataset: plot its written `progress` "
"column instead of re-running the model (uses the dataset's own mp4s)")
ap.add_argument("--column", choices=["auto", "reward", "progress"], default="auto",
help="--from-dataset only: which column to plot. `reward` is the per-frame "
"delta, `progress` the cumulative curve. auto = reward if present.")
ap.add_argument("--show-delta", action="store_true",
help="also draw the per-frame Reward Δ panel under Progress (default: Progress only)")
ap.add_argument("--rm-instruction", default=None,
help="instruction fed to the reward model (e.g. the string in its training "
"config's task_inst_list) instead of each episode's own task string")
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--device", default="cuda")
args = ap.parse_args()
if args.from_dataset:
make_videos_from_dataset(args)
return
cfg = OmegaConf.load(args.config).cfg
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
cams = [c.strip() for c in args.cameras.split(",")] if args.cameras else list(cfg.general.camera_names)
show_cam = args.camera or cams[0]
n_obs, gap = cfg.model.n_obs_steps, cfg.model.frame_gap
ckpt_dir = Path(args.run) if args.run else sorted(
Path("outputs").glob("*/*/sarm2/*/checkpoints"), key=lambda p: p.stat().st_mtime)[-1]
reward_ckpt = Path(args.reward_ckpt) if args.reward_ckpt else ckpt_dir / "reward_best.pt"
act_pri_ckpt = Path(args.act_pri_ckpt) if args.act_pri_ckpt else ckpt_dir / "act_pri_best.pt"
if not act_pri_ckpt.exists():
act_pri_ckpt = Path(cfg.model.act_pri_model_path) # frozen runs keep no copy in the run dir
print(f"[ckpt] reward: {reward_ckpt}")
print(f"[ckpt] act_pri: {act_pri_ckpt}")
encoder = FrozenSiglipEncoder(cfg.encoders.vision_ckpt, device)
normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, device)
task_list = list(cfg.model.task_list)
act_pri_feature = encoder.encode_text(task_list).to(device)
gate_of = torch.tensor([task_to_class_id(t) for t in task_list], device=device)
act_pri_model = ActionTransformer(
d_model=cfg.model.act_pri.d_model, vis_emb_dim=768, state_dim=cfg.model.act_pri.state_dim,
n_layers=cfg.model.act_pri.n_layers, n_heads=cfg.model.act_pri.n_heads, dropout=0.0,
num_cameras=len(cams), num_tasks=cfg.model.num_tasks, num_classes=cfg.model.num_class).to(device)
act_pri_model.load_state_dict(torch.load(act_pri_ckpt, map_location=device,
weights_only=False)["model"])
act_pri_model.eval()
moe = cfg.model.moe
reward_model = RewardTransformer(
d_model=cfg.model.d_model, vis_emb_dim=768, text_emb_dim=768, state_dim=cfg.model.state_dim,
n_layers=cfg.model.n_layers, n_heads=cfg.model.n_heads, dropout=0.0, num_cameras=len(cams),
num_gates=cfg.model.num_class, num_experts=moe.num_experts, top_k=moe.top_k,
gate_units=list(moe.gate_units), gate_act=moe.gate_act, gate_dropout=moe.gate_dropout,
expert_units=list(moe.expert_units), expert_act=moe.expert_act,
expert_dropout=moe.expert_dropout, lambda_balance=moe.lambda_balance,
lambda_entropy=moe.lambda_entropy, lambda_importance=moe.lambda_importance).to(device)
reward_model.load_state_dict(torch.load(reward_ckpt, map_location=device,
weights_only=False)["model"])
reward_model.eval()
root = Path(args.dataset_root) if args.dataset_root else (
Path.home() / ".cache" / "huggingface" / "lerobot" / cfg.general.repo_id)
meta = load_dataset_meta(root)
if args.episode_ids:
chosen = [int(x) for x in args.episode_ids.split(",")]
else:
valid = [e.index for e in meta.episodes] if args.dataset_root else get_valid_episodes(cfg.general.repo_id)
tr, va = split_train_eval_episodes(valid, 1 - cfg.train.val_portion, seed=cfg.general.seed)
pool = {"val": va, "train": tr, "all": valid}[args.split]
rng = np.random.default_rng(args.seed)
chosen = sorted(rng.choice(pool, size=min(args.episodes, len(pool)), replace=False).tolist())
print(f"[eval] episodes: {chosen}")
out_root = Path(args.out).resolve()
fps = int(round(meta.fps))
for ep_id in chosen:
episode = meta.episode(ep_id)
n = episode.length
frames = {c: decode_episode_frames(episode.videos[c].path, episode.videos[c].start_time,
meta.fps, n) for c in cams}
emb = encode_episode(encoder, frames, 96, device)
state = torch.from_numpy(load_source_table(meta, episode)["state"]).float()
windows = torch.tensor([sequence_indices(t, n_obs, gap, n - 1) for t in range(n)], dtype=torch.long)
T = windows.shape[1]
inst = encoder.encode_text([args.rm_instruction or episode.task]).to(device)
preds = []
with torch.no_grad():
for lo in range(0, n, 256):
w = windows[lo:lo + 256]
b = w.shape[0]
img = torch.stack([emb[c][w.reshape(-1)].view(b, T, -1) for c in cams], 1).to(device)
st = normalizer.normalize(state[w.reshape(-1)].view(b, T, -1).to(device))
lens = torch.full((b,), T, dtype=torch.int32, device=device)
ap_logits, _ = act_pri_model(img[:, :, 1:n_obs + 1, :], st[:, 1:n_obs + 1, :],
torch.full((b,), n_obs, dtype=torch.int32, device=device))
ap_idx = F.softmax(ap_logits, -1).argmax(-1)
out = reward_model(img, act_pri_feature[ap_idx], inst.expand(b, -1), st, lens,
gate_idx=gate_of[ap_idx])
r = out[0] if isinstance(out, (tuple, list)) else out
preds.append(torch.clip(r, 0, 1)[:, -1].float().cpu())
progress = 1.0 - torch.cat(preds).numpy()
name = f"episode_{ep_id:06d}"
ep_dir = out_root / name
ep_dir.mkdir(parents=True, exist_ok=True)
progress = progress.astype(np.float32)
np.save(ep_dir / "pred_raw.npy", progress)
if args.smooth_window and args.smooth_window > 1:
w = int(args.smooth_window) | 1 # force odd
k = np.ones(w, dtype=np.float32) / w
progress = np.convolve(progress, k, mode="same").astype(np.float32)
if args.monotone:
progress = np.maximum.accumulate(progress).astype(np.float32)
np.save(ep_dir / "pred.npy", progress)
np.save(ep_dir / "gt.npy", (np.arange(n) / max(1, n - 1)).astype(np.float32))
cam_frames = frames[show_cam]
# The camera-only mp4 is just an input to the repo-style renderer; with the default
# renderer it duplicates combined_video.mp4, so it is not written unless asked for.
video_path = ep_dir / "camera.mp4"
if args.repo_style or args.keep_camera:
write_mp4(cam_frames, video_path, fps)
if args.repo_style:
produce_video(save_dir=out_root, middle_video=video_path, episode_num=name,
raw_data=True, x_offset=0, frame_rate=fps)
else:
m = min(len(cam_frames), len(progress))
render_progressive(cam_frames[:m], plot_series(progress[:m], args.show_delta),
fps, ep_dir / "combined_video.mp4", title=episode.task or "")
del frames, emb
print(f" ep {ep_id:03d}: {n} frames | progress {progress[0]:+.3f} -> "
f"{progress[-1]:+.3f} -> {ep_dir/'combined_video.mp4'}", flush=True)
print(f"\n[out] {out_root}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
20.3 kB
·
Xet hash:
98d0bedc305ef7c50151506676d3e68b0aca48e5225ca2faa8fde569357516a2

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.