Buckets:
| #!/usr/bin/env python | |
| """Render the first N episodes of every SOURCE inside a SPIRAL-labelled data_all dataset. | |
| python tools/make_data_all_videos.py --repo-id local/<ds> --per-source 20 [--dry-run] | |
| Why this is not a plain loop over episodes: `source_dataset` in meta/episodes.jsonl stores only | |
| the folder BASENAME, so the three different `policy_rollouts_1` directories all record the same | |
| name and grouping by it merges them. The real per-source blocks are recovered by walking episodes | |
| in index order and starting a new block wherever `source_episode` fails to increase. | |
| Only combined_video.mp4 is written (plus small pred/reward .npy); --repo-style, which would also | |
| leave a camera.mp4, is deliberately not passed. | |
| --column progress is REQUIRED downstream: the default (auto) picks the `reward` column, which is | |
| the per-frame delta (~75% exact zeros), then labels that curve "Progress" and plots its diff as | |
| "Reward Delta" -- both panels mislabelled. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| def blocks(meta_dir: Path) -> list[tuple[str, list[int]]]: | |
| rows = [json.loads(l) for l in (meta_dir / "episodes.jsonl").open()] | |
| rows.sort(key=lambda r: r["episode_index"]) | |
| out: list[list[dict]] = [] | |
| prev = None | |
| for r in rows: | |
| if (prev is None | |
| or r["source_dataset"] != prev["source_dataset"] | |
| or r["source_episode"] <= prev["source_episode"]): | |
| out.append([]) | |
| out[-1].append(r) | |
| prev = r | |
| # Prefix with the block's position so names are unique AND ordered. A bare-basename scheme | |
| # collides: three sources are called policy_rollouts_1, and a "__2" suffix on the second of | |
| # them reads almost identically to the genuine folder policy_rollouts_2_2. | |
| return [(f"{i:02d}_{b[0]['source_dataset']}", [r["episode_index"] for r in b]) | |
| for i, b in enumerate(out, 1)] | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--repo-id", required=True) | |
| ap.add_argument("--out", default=None, help="default: videos/<basename of repo-id>") | |
| ap.add_argument("--per-source", type=int, default=20) | |
| ap.add_argument("--column", default="progress", choices=["progress", "reward", "auto"]) | |
| ap.add_argument("--dry-run", action="store_true", help="print the plan, render nothing") | |
| args = ap.parse_args() | |
| root = Path.home() / ".cache/huggingface/lerobot" / args.repo_id | |
| if not (root / "meta" / "episodes.jsonl").exists(): | |
| raise SystemExit(f"no meta/episodes.jsonl under {root}") | |
| out_root = Path(args.out or f"videos/{args.repo_id.split('/')[-1]}") | |
| plan = [(n, ids[: args.per_source]) for n, ids in blocks(root / "meta")] | |
| total = sum(len(ids) for _, ids in plan) | |
| print(f"[plan] {args.repo_id} -> {out_root}") | |
| print(f"[plan] {len(plan)} sources, {total} videos (first {args.per_source} of each)") | |
| for n, ids in plan: | |
| print(f" {n:52s} {len(ids):3d} {ids[0]}..{ids[-1]}") | |
| if args.dry_run: | |
| return | |
| for n, ids in plan: | |
| print(f"=== {n}", flush=True) | |
| subprocess.run([sys.executable, "tools/make_sarm2_video.py", | |
| "--from-dataset", args.repo_id, | |
| "--episode-ids", ",".join(map(str, ids)), | |
| "--column", args.column, | |
| "--out", str(out_root / n)], check=True) | |
| print(f"\n[out] {out_root}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.52 kB
- Xet hash:
- a53bc87a79d8dd8840cd5ce1bcd07bd9046aa17b5b6008152bc4d0305a16c6e0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.