Nadeem-Shoukath115113/opensarm / tools /make_linear_reward_dataset.py
Nadeem-Shoukath115113's picture
download
raw
3.94 kB
#!/usr/bin/env python
"""Clone a built SARM2 v2.0 dataset, replacing the heuristic-boosted `reward`
curve with plain linear progress (t+1)/T.
Everything else is byte-identical to the source: same episodes, same order,
same act_pri labels, same state/actions, same videos (symlinked, not copied).
That makes the pair a controlled ablation -- the only variable is the reward
ground truth.
Usage:
python tools/make_linear_reward_dataset.py \\
--src ~/.cache/huggingface/lerobot/local/so101_sarm2_defaultsplit67 \\
--dst ~/.cache/huggingface/lerobot/local/so101_sarm2_defaultsplit67_linear
"""
from __future__ import annotations
import argparse
import json
import shutil
from pathlib import Path
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
def _fsl_f32(values: np.ndarray) -> pa.FixedSizeListArray:
return pa.FixedSizeListArray.from_arrays(pa.array(values.astype(np.float32), type=pa.float32()), 1)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--src", required=True)
ap.add_argument("--dst", required=True)
ap.add_argument("--copy-videos", action="store_true", help="Copy videos instead of symlinking")
ap.add_argument("--overwrite", action="store_true")
args = ap.parse_args()
src = Path(args.src).expanduser().resolve()
dst = Path(args.dst).expanduser().resolve()
if not (src / "meta" / "info.json").exists():
raise SystemExit(f"{src}: not a v2.0 LeRobot dataset (no meta/info.json)")
if dst.exists():
if not args.overwrite:
raise SystemExit(f"{dst} exists; pass --overwrite")
shutil.rmtree(dst)
dst.mkdir(parents=True)
shutil.copytree(src / "meta", dst / "meta")
if (src / "state_norm.json").exists():
shutil.copy2(src / "state_norm.json", dst / "state_norm.json")
if args.copy_videos:
shutil.copytree(src / "videos", dst / "videos")
else:
(dst / "videos").symlink_to(src / "videos", target_is_directory=True)
parquets = sorted((src / "data").rglob("*.parquet"))
if not parquets:
raise SystemExit(f"{src}: no parquet files under data/")
reward_all: list[np.ndarray] = []
delta_all: list[np.ndarray] = []
max_dev = 0.0
for p in parquets:
table = pq.read_table(p)
n = table.num_rows
lin = ((np.arange(n) + 1) / n).astype(np.float32)
delta = np.diff(lin, prepend=lin[0]).astype(np.float32)
old = np.asarray(table["reward"].to_pylist(), dtype=np.float64).reshape(-1)
max_dev = max(max_dev, float(np.abs(old - lin).max()))
table = table.set_column(table.column_names.index("reward"), "reward", _fsl_f32(lin))
table = table.set_column(table.column_names.index("reward_delta"), "reward_delta", _fsl_f32(delta))
table = table.replace_schema_metadata({})
out = dst / p.relative_to(src)
out.parent.mkdir(parents=True, exist_ok=True)
pq.write_table(table, out)
reward_all.append(lin.astype(np.float64))
delta_all.append(delta.astype(np.float64))
reward = np.concatenate(reward_all)
reward_delta = np.concatenate(delta_all)
stats = json.loads((dst / "meta" / "stats.json").read_text())
for key, v in (("reward", reward), ("reward_delta", reward_delta)):
stats[key] = {
"mean": [float(v.mean())],
"std": [float(max(v.std(), 1e-6))],
"min": [float(v.min())],
"max": [float(v.max())],
}
(dst / "meta" / "stats.json").write_text(json.dumps(stats, indent=2))
print(f"Wrote {len(parquets)} episodes ({reward.size} frames) to {dst}")
print(f"videos: {'copied' if args.copy_videos else 'symlinked -> ' + str(src / 'videos')}")
print(f"max |heuristic - linear| over the dataset: {max_dev:.4f}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
3.94 kB
·
Xet hash:
a222ee7e7d3ef9445b8778b8e4a7610b1fbfd25e5981d3a2fe312db4b30b912a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.