Buckets:
| #!/usr/bin/env python | |
| """Clone a built SARM2 v2.0 dataset, replacing the heuristic-boosted `reward` | |
| curve with plain linear progress (t+1)/T. | |
| Everything else is byte-identical to the source: same episodes, same order, | |
| same act_pri labels, same state/actions, same videos (symlinked, not copied). | |
| That makes the pair a controlled ablation -- the only variable is the reward | |
| ground truth. | |
| Usage: | |
| python tools/make_linear_reward_dataset.py \\ | |
| --src ~/.cache/huggingface/lerobot/local/so101_sarm2_defaultsplit67 \\ | |
| --dst ~/.cache/huggingface/lerobot/local/so101_sarm2_defaultsplit67_linear | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import shutil | |
| from pathlib import Path | |
| import numpy as np | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| def _fsl_f32(values: np.ndarray) -> pa.FixedSizeListArray: | |
| return pa.FixedSizeListArray.from_arrays(pa.array(values.astype(np.float32), type=pa.float32()), 1) | |
| def main() -> None: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--src", required=True) | |
| ap.add_argument("--dst", required=True) | |
| ap.add_argument("--copy-videos", action="store_true", help="Copy videos instead of symlinking") | |
| ap.add_argument("--overwrite", action="store_true") | |
| args = ap.parse_args() | |
| src = Path(args.src).expanduser().resolve() | |
| dst = Path(args.dst).expanduser().resolve() | |
| if not (src / "meta" / "info.json").exists(): | |
| raise SystemExit(f"{src}: not a v2.0 LeRobot dataset (no meta/info.json)") | |
| if dst.exists(): | |
| if not args.overwrite: | |
| raise SystemExit(f"{dst} exists; pass --overwrite") | |
| shutil.rmtree(dst) | |
| dst.mkdir(parents=True) | |
| shutil.copytree(src / "meta", dst / "meta") | |
| if (src / "state_norm.json").exists(): | |
| shutil.copy2(src / "state_norm.json", dst / "state_norm.json") | |
| if args.copy_videos: | |
| shutil.copytree(src / "videos", dst / "videos") | |
| else: | |
| (dst / "videos").symlink_to(src / "videos", target_is_directory=True) | |
| parquets = sorted((src / "data").rglob("*.parquet")) | |
| if not parquets: | |
| raise SystemExit(f"{src}: no parquet files under data/") | |
| reward_all: list[np.ndarray] = [] | |
| delta_all: list[np.ndarray] = [] | |
| max_dev = 0.0 | |
| for p in parquets: | |
| table = pq.read_table(p) | |
| n = table.num_rows | |
| lin = ((np.arange(n) + 1) / n).astype(np.float32) | |
| delta = np.diff(lin, prepend=lin[0]).astype(np.float32) | |
| old = np.asarray(table["reward"].to_pylist(), dtype=np.float64).reshape(-1) | |
| max_dev = max(max_dev, float(np.abs(old - lin).max())) | |
| table = table.set_column(table.column_names.index("reward"), "reward", _fsl_f32(lin)) | |
| table = table.set_column(table.column_names.index("reward_delta"), "reward_delta", _fsl_f32(delta)) | |
| table = table.replace_schema_metadata({}) | |
| out = dst / p.relative_to(src) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| pq.write_table(table, out) | |
| reward_all.append(lin.astype(np.float64)) | |
| delta_all.append(delta.astype(np.float64)) | |
| reward = np.concatenate(reward_all) | |
| reward_delta = np.concatenate(delta_all) | |
| stats = json.loads((dst / "meta" / "stats.json").read_text()) | |
| for key, v in (("reward", reward), ("reward_delta", reward_delta)): | |
| stats[key] = { | |
| "mean": [float(v.mean())], | |
| "std": [float(max(v.std(), 1e-6))], | |
| "min": [float(v.min())], | |
| "max": [float(v.max())], | |
| } | |
| (dst / "meta" / "stats.json").write_text(json.dumps(stats, indent=2)) | |
| print(f"Wrote {len(parquets)} episodes ({reward.size} frames) to {dst}") | |
| print(f"videos: {'copied' if args.copy_videos else 'symlinked -> ' + str(src / 'videos')}") | |
| print(f"max |heuristic - linear| over the dataset: {max_dev:.4f}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.94 kB
- Xet hash:
- a222ee7e7d3ef9445b8778b8e4a7610b1fbfd25e5981d3a2fe312db4b30b912a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.