Nadeem-Shoukath115113/opensarm / tools /make_data_all_videos.py
Nadeem-Shoukath115113's picture
download
raw
3.52 kB
#!/usr/bin/env python
"""Render the first N episodes of every SOURCE inside a SPIRAL-labelled data_all dataset.
python tools/make_data_all_videos.py --repo-id local/<ds> --per-source 20 [--dry-run]
Why this is not a plain loop over episodes: `source_dataset` in meta/episodes.jsonl stores only
the folder BASENAME, so the three different `policy_rollouts_1` directories all record the same
name and grouping by it merges them. The real per-source blocks are recovered by walking episodes
in index order and starting a new block wherever `source_episode` fails to increase.
Only combined_video.mp4 is written (plus small pred/reward .npy); --repo-style, which would also
leave a camera.mp4, is deliberately not passed.
--column progress is REQUIRED downstream: the default (auto) picks the `reward` column, which is
the per-frame delta (~75% exact zeros), then labels that curve "Progress" and plots its diff as
"Reward Delta" -- both panels mislabelled.
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from pathlib import Path
def blocks(meta_dir: Path) -> list[tuple[str, list[int]]]:
rows = [json.loads(l) for l in (meta_dir / "episodes.jsonl").open()]
rows.sort(key=lambda r: r["episode_index"])
out: list[list[dict]] = []
prev = None
for r in rows:
if (prev is None
or r["source_dataset"] != prev["source_dataset"]
or r["source_episode"] <= prev["source_episode"]):
out.append([])
out[-1].append(r)
prev = r
# Prefix with the block's position so names are unique AND ordered. A bare-basename scheme
# collides: three sources are called policy_rollouts_1, and a "__2" suffix on the second of
# them reads almost identically to the genuine folder policy_rollouts_2_2.
return [(f"{i:02d}_{b[0]['source_dataset']}", [r["episode_index"] for r in b])
for i, b in enumerate(out, 1)]
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--repo-id", required=True)
ap.add_argument("--out", default=None, help="default: videos/<basename of repo-id>")
ap.add_argument("--per-source", type=int, default=20)
ap.add_argument("--column", default="progress", choices=["progress", "reward", "auto"])
ap.add_argument("--dry-run", action="store_true", help="print the plan, render nothing")
args = ap.parse_args()
root = Path.home() / ".cache/huggingface/lerobot" / args.repo_id
if not (root / "meta" / "episodes.jsonl").exists():
raise SystemExit(f"no meta/episodes.jsonl under {root}")
out_root = Path(args.out or f"videos/{args.repo_id.split('/')[-1]}")
plan = [(n, ids[: args.per_source]) for n, ids in blocks(root / "meta")]
total = sum(len(ids) for _, ids in plan)
print(f"[plan] {args.repo_id} -> {out_root}")
print(f"[plan] {len(plan)} sources, {total} videos (first {args.per_source} of each)")
for n, ids in plan:
print(f" {n:52s} {len(ids):3d} {ids[0]}..{ids[-1]}")
if args.dry_run:
return
for n, ids in plan:
print(f"=== {n}", flush=True)
subprocess.run([sys.executable, "tools/make_sarm2_video.py",
"--from-dataset", args.repo_id,
"--episode-ids", ",".join(map(str, ids)),
"--column", args.column,
"--out", str(out_root / n)], check=True)
print(f"\n[out] {out_root}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
3.52 kB
·
Xet hash:
a53bc87a79d8dd8840cd5ce1bcd07bd9046aa17b5b6008152bc4d0305a16c6e0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.