Buckets:
| #!/usr/bin/env python | |
| """Render side-by-side progress videos with the repo's own `produce_video`. | |
| Runs SARM2 on every frame of an episode, writes the arrays `produce_video` | |
| expects (pred.npy / gt.npy), extracts that episode's camera view to an mp4, and | |
| calls `utils.make_demo_video.produce_video` to combine them. | |
| python tools/make_sarm2_video.py --dataset-root <dir> --episodes 3 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import sys | |
| from fractions import Fraction | |
| from pathlib import Path | |
| import numpy as np | |
| import pyarrow as pa | |
| from pyarrow import concat_tables as pa_concat_tables | |
| import torch | |
| import torch.nn.functional as F | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from omegaconf import OmegaConf # noqa: E402 | |
| from models.action_estimator import ActionTransformer # noqa: E402 | |
| from models.moe_deco_gate_reward_model import RewardTransformer # noqa: E402 | |
| from models.siglip_encoder import FrozenSiglipEncoder # noqa: E402 | |
| from tools.act_pri_common import load_dataset_meta, task_to_class_id # noqa: E402 | |
| from tools.build_act_pri_clips import load_source_table # noqa: E402 | |
| from tools.predict_act_pri import decode_episode_frames, encode_episode # noqa: E402 | |
| from tools.visualise_sarm2 import sequence_indices # noqa: E402 | |
| from utils.data_utils import get_valid_episodes, split_train_eval_episodes # noqa: E402 | |
| from utils.make_demo_video import produce_video # noqa: E402 | |
| from utils.train_utils import get_normalizer_from_calculated # noqa: E402 | |
| def write_mp4(frames, path: Path, fps: int) -> None: | |
| import av | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| container = av.open(str(path), mode="w") | |
| stream = container.add_stream("libx264", rate=int(fps)) | |
| stream.width, stream.height = frames[0].shape[1], frames[0].shape[0] | |
| stream.pix_fmt = "yuv420p" | |
| stream.time_base = Fraction(1, int(fps)) | |
| stream.options = {"crf": "20", "preset": "veryfast"} | |
| for i, arr in enumerate(frames): | |
| f = av.VideoFrame.from_ndarray(arr, format="rgb24").reformat(format="yuv420p") | |
| f.pts = i | |
| f.time_base = Fraction(1, int(fps)) | |
| for pkt in stream.encode(f): | |
| container.mux(pkt) | |
| for pkt in stream.encode(): | |
| container.mux(pkt) | |
| container.close() | |
| def render_progressive(frames, series, fps, out_path: Path, title: str, ylim=None, spans=None): | |
| """Camera view + a progress plot that draws itself as time advances. | |
| The repo's `draw_plot_frame` plots the whole curve on every frame and marks the | |
| current step with a vertical line. Here the line is drawn only up to the current | |
| frame, so the curve grows with the video. | |
| ylim fixes every panel's y range (default: fit each series). spans is an optional list of | |
| (start_s, end_s, colour) shaded behind every panel, e.g. labelled segments. | |
| """ | |
| from fractions import Fraction | |
| import av | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas | |
| # series: list of (label, values, colour) drawn as stacked panels sharing the x axis. | |
| # The plot is rendered at a fixed readable size; the camera is scaled to match its | |
| # height. Sizing it from the camera (224px here) made the fonts collide. | |
| n = len(series[0][1]) | |
| t = np.arange(n) / fps | |
| dpi = 100 | |
| plot_h_px = max(420, frames[0].shape[0]) | |
| fig, axes = plt.subplots(len(series), 1, sharex=True, | |
| figsize=(plot_h_px * 1.45 / dpi, plot_h_px / dpi), dpi=dpi) | |
| axes = np.atleast_1d(axes) | |
| fig.patch.set_facecolor("white") | |
| lines, cursors = [], [] | |
| for ax, (label, values, colour) in zip(axes, series): | |
| ax.set_facecolor("#eef0f4") | |
| lo, hi = (float(np.min(values)), float(np.max(values))) if ylim is None else ylim | |
| pad = 0.08 * max(1e-6, hi - lo) | |
| ax.set_xlim(0, max(t[-1], 1e-6)) | |
| ax.set_ylim(lo - pad, hi + pad) | |
| for s0, s1, c in spans or (): | |
| ax.axvspan(s0, s1, color=c, lw=0, zorder=0) | |
| ax.set_ylabel(label, fontsize=10, fontweight="bold", labelpad=6) | |
| ax.grid(alpha=0.35, color="white", lw=1.2) | |
| for s in ax.spines.values(): | |
| s.set_visible(False) | |
| ax.tick_params(labelsize=8) | |
| ax.yaxis.set_major_locator(plt.MaxNLocator(4)) | |
| if lo < 0 < hi: | |
| ax.axhline(0, color="#b0b0b0", lw=1.0) | |
| (ln,) = ax.plot([], [], color=colour, lw=2.4, solid_joinstyle="round") | |
| lines.append(ln) | |
| cursors.append(ax.axvline(0, color="#888", lw=1.3, ls=":")) | |
| axes[-1].set_xlabel("Time (s)", fontsize=10, fontweight="bold") | |
| if title: | |
| short = title if len(title) <= 42 else title[:39] + "..." | |
| fig.suptitle(short, fontsize=11, fontweight="bold") | |
| fig.tight_layout(rect=[0.02, 0.02, 0.99, 0.93]) | |
| canvas = FigureCanvas(fig) | |
| container = av.open(str(out_path), mode="w") | |
| stream = None | |
| for i in range(n): | |
| for ln, cur, (_, values, _) in zip(lines, cursors, series): | |
| ln.set_data(t[: i + 1], values[: i + 1]) | |
| cur.set_xdata([t[i], t[i]]) | |
| canvas.draw() | |
| plot = np.asarray(canvas.buffer_rgba(), dtype=np.uint8).reshape( | |
| canvas.get_width_height()[::-1] + (4,))[:, :, :3] | |
| cam = frames[i] | |
| if cam.shape[0] != plot.shape[0]: # match heights | |
| from PIL import Image | |
| new_w = int(cam.shape[1] * plot.shape[0] / cam.shape[0]) | |
| cam = np.asarray(Image.fromarray(cam).resize((new_w, plot.shape[0]), Image.BILINEAR)) | |
| combined = np.concatenate([cam, plot], axis=1) | |
| if combined.shape[1] % 2: | |
| combined = combined[:, :-1] | |
| if combined.shape[0] % 2: | |
| combined = combined[:-1, :] | |
| if stream is None: | |
| stream = container.add_stream("libx264", rate=int(fps)) | |
| stream.width, stream.height = combined.shape[1], combined.shape[0] | |
| stream.pix_fmt = "yuv420p" | |
| stream.time_base = Fraction(1, int(fps)) | |
| stream.options = {"crf": "20", "preset": "veryfast"} | |
| vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(combined), format="rgb24").reformat( | |
| format="yuv420p") | |
| vf.pts = i | |
| vf.time_base = Fraction(1, int(fps)) | |
| for pkt in stream.encode(vf): | |
| container.mux(pkt) | |
| for pkt in stream.encode(): | |
| container.mux(pkt) | |
| container.close() | |
| plt.close(fig) | |
| def plot_series(progress, show_delta: bool): | |
| """Panels for render_progressive: Progress, plus the per-frame Reward Δ when asked for.""" | |
| series = [("Progress", progress, "#20c4c0")] | |
| if show_delta: | |
| per_step = np.zeros_like(progress) | |
| # progress is a cumsum from 0, so frame 0's delta is progress[0], not 0. | |
| per_step[0] = progress[0] | |
| per_step[1:] = np.diff(progress) | |
| series.append(("Reward Δ", per_step, "#f0a832")) | |
| return series | |
| def make_videos_from_dataset(args) -> None: | |
| """Render the labelled dataset's own `reward` column against its own videos. | |
| Uses load_dataset_meta so both v2.x (per-episode) and v3.0 (aggregated) layouts work. | |
| """ | |
| import pyarrow.parquet as pq | |
| root = Path(args.dataset_root) if args.dataset_root else ( | |
| Path.home() / ".cache" / "huggingface" / "lerobot" / args.from_dataset) | |
| meta = load_dataset_meta(root) | |
| cams = list(meta.camera_keys) | |
| show_cam = args.camera or cams[0] | |
| all_ids = [e.index for e in meta.episodes] | |
| if args.episode_ids: | |
| ids = [int(x) for x in args.episode_ids.split(",")] | |
| elif args.all: | |
| ids = sorted(all_ids) | |
| else: | |
| rng = np.random.default_rng(args.seed) | |
| ids = sorted(rng.choice(all_ids, size=min(args.episodes, len(all_ids)), replace=False).tolist()) | |
| print(f"[data] {root}\n[eval] episodes: {ids} camera: {show_cam}") | |
| # Load every data parquet once and concat by dataset index. | |
| parquets = sorted((root / "data").rglob("*.parquet")) | |
| tables = [pq.read_table(p) for p in parquets] | |
| full = tables[0] if len(tables) == 1 else pa_concat_tables(tables) | |
| # `reward` is the per-frame DELTA (~0.002 here); `progress` is the cumulative 0->1 curve. | |
| # Default keeps the original preference; --column progress plots the curve instead. | |
| if args.column != "auto": | |
| if args.column not in full.column_names: | |
| raise SystemExit(f"dataset has no `{args.column}` column") | |
| reward_col = args.column | |
| else: | |
| reward_col = "reward" if "reward" in full.column_names else ( | |
| "progress" if "progress" in full.column_names else None) | |
| if reward_col is None: | |
| raise SystemExit("dataset has no reward/progress column to render") | |
| all_reward = np.array([x[0] for x in full.column(reward_col).to_pylist()], dtype=np.float32) | |
| fps = int(round(meta.fps)) | |
| out_root = Path(args.out).resolve() | |
| for ep_id in ids: | |
| ep = meta.episode(ep_id) | |
| reward = all_reward[ep.from_index:ep.to_index] | |
| progress = reward.astype(np.float32) | |
| name = f"episode_{ep_id:06d}" | |
| ep_dir = out_root / name | |
| ep_dir.mkdir(parents=True, exist_ok=True) | |
| np.save(ep_dir / "pred.npy", progress) | |
| np.save(ep_dir / "reward.npy", reward) | |
| cam_frames = decode_episode_frames( | |
| ep.videos[show_cam].path, ep.videos[show_cam].start_time, meta.fps, ep.length) | |
| if args.repo_style: | |
| tmp_video = ep_dir / "camera.mp4" | |
| write_mp4(cam_frames, tmp_video, fps) | |
| produce_video(save_dir=out_root, middle_video=tmp_video, episode_num=name, | |
| raw_data=True, x_offset=0, frame_rate=fps) | |
| else: | |
| m = min(len(cam_frames), len(progress)) | |
| render_progressive(cam_frames[:m], plot_series(progress[:m], args.show_delta), | |
| fps, ep_dir / "combined_video.mp4", title=ep.task or "") | |
| print(f" ep {ep_id:03d}: {len(progress)} frames | progress {progress[0]:+.3f} -> " | |
| f"{progress[-1]:+.3f} -> {ep_dir / 'combined_video.mp4'}", flush=True) | |
| print(f"\n[out] {out_root}") | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) | |
| ap.add_argument("--config", default="config/sarm2.yaml") | |
| ap.add_argument("--run", default=None) | |
| ap.add_argument("--reward-ckpt", default=None, help="explicit reward checkpoint (default: <run>/reward_best.pt)") | |
| ap.add_argument("--act-pri-ckpt", default=None, | |
| help="explicit act_pri checkpoint (default: <run>/act_pri_best.pt, " | |
| "falling back to cfg.model.act_pri_model_path when act_pri was frozen)") | |
| ap.add_argument("--dataset-root", default=None) | |
| ap.add_argument("--episodes", type=int, default=3) | |
| ap.add_argument("--all", action="store_true", help="render every episode in the dataset") | |
| ap.add_argument("--episode-ids", default=None) | |
| ap.add_argument("--split", choices=["val", "train", "all"], default="all") | |
| ap.add_argument("--camera", default=None, help="camera to show (default: the first one)") | |
| ap.add_argument("--cameras", default=None, | |
| help="comma-separated camera keys to override cfg.general.camera_names " | |
| "(order matters: must match the training arity)") | |
| ap.add_argument("--out", default="sarm2_videos") | |
| ap.add_argument("--smooth-window", type=int, default=0, | |
| help="odd window size for a centred moving average over the predicted " | |
| "progress before plotting (0 = off).") | |
| ap.add_argument("--monotone", action="store_true", | |
| help="enforce monotone-increasing progress via cumulative max (applied " | |
| "after --smooth-window).") | |
| ap.add_argument("--keep-camera", action="store_true", | |
| help="also write the camera-only camera.mp4 next to combined_video.mp4 " | |
| "(default: combined video only)") | |
| ap.add_argument("--repo-style", action="store_true", | |
| help="use the repo's draw_plot_frame (whole curve + moving line) instead of " | |
| "drawing the curve progressively") | |
| ap.add_argument("--from-dataset", default=None, | |
| help="repo id of a SPIRAL-labelled dataset: plot its written `progress` " | |
| "column instead of re-running the model (uses the dataset's own mp4s)") | |
| ap.add_argument("--column", choices=["auto", "reward", "progress"], default="auto", | |
| help="--from-dataset only: which column to plot. `reward` is the per-frame " | |
| "delta, `progress` the cumulative curve. auto = reward if present.") | |
| ap.add_argument("--show-delta", action="store_true", | |
| help="also draw the per-frame Reward Δ panel under Progress (default: Progress only)") | |
| ap.add_argument("--rm-instruction", default=None, | |
| help="instruction fed to the reward model (e.g. the string in its training " | |
| "config's task_inst_list) instead of each episode's own task string") | |
| ap.add_argument("--seed", type=int, default=0) | |
| ap.add_argument("--device", default="cuda") | |
| args = ap.parse_args() | |
| if args.from_dataset: | |
| make_videos_from_dataset(args) | |
| return | |
| cfg = OmegaConf.load(args.config).cfg | |
| device = torch.device(args.device if torch.cuda.is_available() else "cpu") | |
| cams = [c.strip() for c in args.cameras.split(",")] if args.cameras else list(cfg.general.camera_names) | |
| show_cam = args.camera or cams[0] | |
| n_obs, gap = cfg.model.n_obs_steps, cfg.model.frame_gap | |
| ckpt_dir = Path(args.run) if args.run else sorted( | |
| Path("outputs").glob("*/*/sarm2/*/checkpoints"), key=lambda p: p.stat().st_mtime)[-1] | |
| reward_ckpt = Path(args.reward_ckpt) if args.reward_ckpt else ckpt_dir / "reward_best.pt" | |
| act_pri_ckpt = Path(args.act_pri_ckpt) if args.act_pri_ckpt else ckpt_dir / "act_pri_best.pt" | |
| if not act_pri_ckpt.exists(): | |
| act_pri_ckpt = Path(cfg.model.act_pri_model_path) # frozen runs keep no copy in the run dir | |
| print(f"[ckpt] reward: {reward_ckpt}") | |
| print(f"[ckpt] act_pri: {act_pri_ckpt}") | |
| encoder = FrozenSiglipEncoder(cfg.encoders.vision_ckpt, device) | |
| normalizer = get_normalizer_from_calculated(cfg.general.state_norm_path, device) | |
| task_list = list(cfg.model.task_list) | |
| act_pri_feature = encoder.encode_text(task_list).to(device) | |
| gate_of = torch.tensor([task_to_class_id(t) for t in task_list], device=device) | |
| act_pri_model = ActionTransformer( | |
| d_model=cfg.model.act_pri.d_model, vis_emb_dim=768, state_dim=cfg.model.act_pri.state_dim, | |
| n_layers=cfg.model.act_pri.n_layers, n_heads=cfg.model.act_pri.n_heads, dropout=0.0, | |
| num_cameras=len(cams), num_tasks=cfg.model.num_tasks, num_classes=cfg.model.num_class).to(device) | |
| act_pri_model.load_state_dict(torch.load(act_pri_ckpt, map_location=device, | |
| weights_only=False)["model"]) | |
| act_pri_model.eval() | |
| moe = cfg.model.moe | |
| reward_model = RewardTransformer( | |
| d_model=cfg.model.d_model, vis_emb_dim=768, text_emb_dim=768, state_dim=cfg.model.state_dim, | |
| n_layers=cfg.model.n_layers, n_heads=cfg.model.n_heads, dropout=0.0, num_cameras=len(cams), | |
| num_gates=cfg.model.num_class, num_experts=moe.num_experts, top_k=moe.top_k, | |
| gate_units=list(moe.gate_units), gate_act=moe.gate_act, gate_dropout=moe.gate_dropout, | |
| expert_units=list(moe.expert_units), expert_act=moe.expert_act, | |
| expert_dropout=moe.expert_dropout, lambda_balance=moe.lambda_balance, | |
| lambda_entropy=moe.lambda_entropy, lambda_importance=moe.lambda_importance).to(device) | |
| reward_model.load_state_dict(torch.load(reward_ckpt, map_location=device, | |
| weights_only=False)["model"]) | |
| reward_model.eval() | |
| root = Path(args.dataset_root) if args.dataset_root else ( | |
| Path.home() / ".cache" / "huggingface" / "lerobot" / cfg.general.repo_id) | |
| meta = load_dataset_meta(root) | |
| if args.episode_ids: | |
| chosen = [int(x) for x in args.episode_ids.split(",")] | |
| else: | |
| valid = [e.index for e in meta.episodes] if args.dataset_root else get_valid_episodes(cfg.general.repo_id) | |
| tr, va = split_train_eval_episodes(valid, 1 - cfg.train.val_portion, seed=cfg.general.seed) | |
| pool = {"val": va, "train": tr, "all": valid}[args.split] | |
| rng = np.random.default_rng(args.seed) | |
| chosen = sorted(rng.choice(pool, size=min(args.episodes, len(pool)), replace=False).tolist()) | |
| print(f"[eval] episodes: {chosen}") | |
| out_root = Path(args.out).resolve() | |
| fps = int(round(meta.fps)) | |
| for ep_id in chosen: | |
| episode = meta.episode(ep_id) | |
| n = episode.length | |
| frames = {c: decode_episode_frames(episode.videos[c].path, episode.videos[c].start_time, | |
| meta.fps, n) for c in cams} | |
| emb = encode_episode(encoder, frames, 96, device) | |
| state = torch.from_numpy(load_source_table(meta, episode)["state"]).float() | |
| windows = torch.tensor([sequence_indices(t, n_obs, gap, n - 1) for t in range(n)], dtype=torch.long) | |
| T = windows.shape[1] | |
| inst = encoder.encode_text([args.rm_instruction or episode.task]).to(device) | |
| preds = [] | |
| with torch.no_grad(): | |
| for lo in range(0, n, 256): | |
| w = windows[lo:lo + 256] | |
| b = w.shape[0] | |
| img = torch.stack([emb[c][w.reshape(-1)].view(b, T, -1) for c in cams], 1).to(device) | |
| st = normalizer.normalize(state[w.reshape(-1)].view(b, T, -1).to(device)) | |
| lens = torch.full((b,), T, dtype=torch.int32, device=device) | |
| ap_logits, _ = act_pri_model(img[:, :, 1:n_obs + 1, :], st[:, 1:n_obs + 1, :], | |
| torch.full((b,), n_obs, dtype=torch.int32, device=device)) | |
| ap_idx = F.softmax(ap_logits, -1).argmax(-1) | |
| out = reward_model(img, act_pri_feature[ap_idx], inst.expand(b, -1), st, lens, | |
| gate_idx=gate_of[ap_idx]) | |
| r = out[0] if isinstance(out, (tuple, list)) else out | |
| preds.append(torch.clip(r, 0, 1)[:, -1].float().cpu()) | |
| progress = 1.0 - torch.cat(preds).numpy() | |
| name = f"episode_{ep_id:06d}" | |
| ep_dir = out_root / name | |
| ep_dir.mkdir(parents=True, exist_ok=True) | |
| progress = progress.astype(np.float32) | |
| np.save(ep_dir / "pred_raw.npy", progress) | |
| if args.smooth_window and args.smooth_window > 1: | |
| w = int(args.smooth_window) | 1 # force odd | |
| k = np.ones(w, dtype=np.float32) / w | |
| progress = np.convolve(progress, k, mode="same").astype(np.float32) | |
| if args.monotone: | |
| progress = np.maximum.accumulate(progress).astype(np.float32) | |
| np.save(ep_dir / "pred.npy", progress) | |
| np.save(ep_dir / "gt.npy", (np.arange(n) / max(1, n - 1)).astype(np.float32)) | |
| cam_frames = frames[show_cam] | |
| # The camera-only mp4 is just an input to the repo-style renderer; with the default | |
| # renderer it duplicates combined_video.mp4, so it is not written unless asked for. | |
| video_path = ep_dir / "camera.mp4" | |
| if args.repo_style or args.keep_camera: | |
| write_mp4(cam_frames, video_path, fps) | |
| if args.repo_style: | |
| produce_video(save_dir=out_root, middle_video=video_path, episode_num=name, | |
| raw_data=True, x_offset=0, frame_rate=fps) | |
| else: | |
| m = min(len(cam_frames), len(progress)) | |
| render_progressive(cam_frames[:m], plot_series(progress[:m], args.show_delta), | |
| fps, ep_dir / "combined_video.mp4", title=episode.task or "") | |
| del frames, emb | |
| print(f" ep {ep_id:03d}: {n} frames | progress {progress[0]:+.3f} -> " | |
| f"{progress[-1]:+.3f} -> {ep_dir/'combined_video.mp4'}", flush=True) | |
| print(f"\n[out] {out_root}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 20.3 kB
- Xet hash:
- 98d0bedc305ef7c50151506676d3e68b0aca48e5225ca2faa8fde569357516a2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.