"""Entry points for training pi0.5 on the dexhand sub-task data on the A100 box (Autoresearch3), without editing openpi. Adapted from the gripper project's runner ($SYT/code/runner/run.py): same commands and fixes, config from train/dexhand_openpi.py (interleaved left / right actions, see there). Run from the openpi dir with its venv after `source $SYT/env.sh` (train.sh does this): prompt-len tokenised prompt length (task + discretised 54-D state) over sampled frames -> choose max_token_len roundtrip InterleaveLR / DeinterleaveLR and the full input -> output transform chain reproduce the actions smoke one batch through the real data loader (shapes) norm-stats state / action quantile stats (after the arm deltas, before the interleave), JAX_PLATFORMS=cpu loader-speed steady-state data loader throughput train openpi training (--fake-data: memory / speed probe without the data loader) restore-check restore the latest checkpoint as --resume does, without the data loader Paths: data $HF_LEROBOT_HOME/ (default glob:dexhand/cover_v2_7seg_*) norm stats $SYT/assets//dual_marvin_wuji/norm_stats.json checkpoints $SYT/checkpoints/dexhand_ilv// """ import argparse import importlib.util import os import pathlib import sys import time SYT = pathlib.Path(os.environ["SYT"]) OPENPI = SYT / "code/XPolicyLab/policy/Pi_05/openpi" HERE = pathlib.Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) sys.path.insert(0, str(SYT / "code/runner")) # grad_accum.py of the gripper project (unchanged) import cv2 # noqa: E402 cv2.setNumThreads(1) import dexhand_openpi as dx # noqa: E402 dx.install_dataset_patch() def _load_script(name): spec = importlib.util.spec_from_file_location(f"openpi_{name}", OPENPI / "scripts" / f"{name}.py") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def _patch_train_state(train): """Copied from the gripper runner (run.py there documents both fixes): A. fresh start: re-place the initial state from host memory (allocator fragmentation -> OOM on 40 GB cards); B. --resume under orbax 0.11.13: restore with explicit PyTreeRestore args carrying the FSDP sharding.""" import jax import orbax.checkpoint as ocp from openpi.shared import array_typing as at from openpi.training import checkpoints as _checkpoints init_train_state = train.init_train_state def init_with_sharding(config, init_rng, mesh, *, resume): state, state_sharding = init_train_state(config, init_rng, mesh, resume=resume) if resume: state = jax.tree.map(lambda s, sh: jax.ShapeDtypeStruct(s.shape, s.dtype, sharding=sh), state, state_sharding) else: with at.disable_typechecking(): host = jax.device_get(state) jax.tree.map(lambda x: x.delete(), state) state = jax.device_put(host, state_sharding) return state, state_sharding def restore_state(checkpoint_manager, state, data_loader, step=None): del data_loader with at.disable_typechecking(): train_state, params = _checkpoints._split_params(state) items = {"train_state": train_state, "params": {"params": params}} restored = checkpoint_manager.restore(step, args=ocp.args.Composite(**{ k: ocp.args.PyTreeRestore(item=v, restore_args=ocp.checkpoint_utils.construct_restore_args(v)) for k, v in items.items()})) return _checkpoints._merge_params(restored["train_state"], restored["params"]) train.init_train_state = init_with_sharding _checkpoints.restore_state = restore_state def build_config(a, **overrides): return dx.get_config( a.repo_id, T=a.horizon, max_token_len=a.max_token_len, assets_base_dir=str(SYT / "assets" / a.stats), checkpoint_base_dir=str(SYT / "checkpoints"), wandb_enabled=False, **overrides) def _raw_dataset(cfg, with_model_tf=False): """Dataset through repack + data transforms (+ Normalize + model transforms if with_model_tf).""" from openpi import transforms as _t from openpi.training import data_loader as dl dc = cfg.data.create(cfg.assets_dirs, cfg.model) ds = dl.create_torch_dataset(dc, cfg.model.action_horizon, cfg.model) tfs = [*dc.repack_transforms.inputs, *dc.data_transforms.inputs] if with_model_tf: tfs += [_t.Normalize(dc.norm_stats, use_quantiles=dc.use_quantile_norm), *dc.model_transforms.inputs] return dl.TransformedDataset(ds, tfs), dc def cmd_prompt_len(a): import numpy as np from openpi import transforms as _t from openpi.models import tokenizer as _tok cfg = build_config(a) ds, dc = _raw_dataset(cfg) norm = _t.Normalize(dc.norm_stats, use_quantiles=dc.use_quantile_norm) tok = _tok.PaligemmaTokenizer(4096) rng = np.random.default_rng(0) lens = [] for i in rng.integers(0, len(ds), a.samples): d = norm(dict(ds[int(i)])) ids, mask = tok.tokenize(d["prompt"], np.asarray(d["state"])) lens.append(int(np.asarray(mask).sum())) lens = np.array(lens) print(f"prompt tokens over {len(lens)} frames: min {lens.min()} median {int(np.median(lens))} " f"p99 {int(np.percentile(lens, 99))} max {lens.max()} (max_token_len now {cfg.model.max_token_len})") def cmd_roundtrip(a): import numpy as np rng = np.random.default_rng(0) x = rng.normal(size=(3, a.horizon, 54)).astype(np.float32) y = dx.InterleaveLR()({"actions": x.copy()})["actions"] assert y.shape == (3, 2 * a.horizon, dx.PAD), y.shape assert np.array_equal(y[:, 0, :27], x[:, 0, :27]) and np.array_equal(y[:, 1, :27], x[:, 0, 27:]) assert not y[..., 27:].any() z = dx.DeinterleaveLR()({"actions": y})["actions"] assert np.array_equal(z, x) print("interleave round trip: exact") # full chain on real frames: inputs (incl. Normalize, delta, interleave) then outputs (de-interleave, Unnormalize, # absolute, trim) must give back the recorded absolute 54-D actions from openpi import transforms as _t cfg = build_config(a) ds, dc = _raw_dataset(cfg) from openpi.training import data_loader as dl raw = dl.create_torch_dataset(dc, cfg.model.action_horizon, cfg.model) rep = _t.compose(dc.repack_transforms.inputs) tin = _t.compose([*dc.data_transforms.inputs, _t.Normalize(dc.norm_stats, use_quantiles=dc.use_quantile_norm), *dc.model_transforms.inputs]) tout = _t.compose([*dc.model_transforms.outputs, _t.Unnormalize(dc.norm_stats, use_quantiles=dc.use_quantile_norm), *dc.data_transforms.outputs]) worst = 0.0 for i in rng.integers(0, len(raw), a.samples): d = rep(dict(raw[int(i)])) ref = np.asarray(d["actions"], np.float64).copy() m = tin(d) out = tout({"state": m["state"], "actions": np.asarray(m["actions"])}) worst = max(worst, float(np.abs(np.asarray(out["actions"]) - ref).max())) print(f"full transform chain on {a.samples} real samples: max |actions - recorded| = {worst:.2e} " f"(quantile clipping can only show up outside [q01, q99])") def cmd_smoke(a): from openpi.training import data_loader as dl cfg = build_config(a, batch_size=8, num_workers=0) obs, act = next(iter(dl.create_data_loader(cfg, shuffle=True, num_batches=1, skip_norm_stats=a.skip_norm))) print({k: v.shape for k, v in obs.images.items()}, "state", obs.state.shape, "actions", act.shape) print("prompt tokens:", None if obs.tokenized_prompt is None else obs.tokenized_prompt.shape, "used:", None if obs.tokenized_prompt_mask is None else int(obs.tokenized_prompt_mask.sum(-1).max())) def cmd_norm_stats(a): import numpy as np import tqdm from openpi.shared import normalize from openpi.training import data_loader as dl cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) ds, _ = _raw_dataset(cfg) ds = dl.TransformedDataset(ds, [dx.RemoveStrings()]) if a.max_frames is not None and a.max_frames < len(ds): n, shuffle = a.max_frames // cfg.batch_size, True else: n, shuffle = len(ds) // cfg.batch_size, False loader = dl.TorchDataLoader(ds, local_batch_size=cfg.batch_size, num_workers=cfg.num_workers, shuffle=shuffle, num_batches=n) stats = {k: normalize.RunningStats() for k in ("state", "actions")} for batch in tqdm.tqdm(loader, total=n, desc="norm stats", mininterval=60): for k in stats: stats[k].update(np.asarray(batch[k])) out = SYT / "assets" / a.stats / dx.ASSET_ID normalize.save(out, {k: s.get_statistics() for k, s in stats.items()}) print("wrote", out / "norm_stats.json") def cmd_norm_stats_parquet(a): """The same statistics as norm-stats, from the parquet columns only (no video decoding; 50x faster): for every frame t, state[t] and the action chunk action[t : t+T] (clamped at the episode end, as LeRobot's delta_timestamps pads with the last frame) with the arm dims made relative to state[t] (DeltaActions mask 7,-20,7,-20).""" import glob import numpy as np import pandas as pd from openpi.shared import normalize home = pathlib.Path(os.environ["HF_LEROBOT_HOME"]) arm = np.array([True] * 7 + [False] * 20 + [True] * 7 + [False] * 20) stats = {k: normalize.RunningStats() for k in ("state", "actions")} n = 0 for r in dx.expand_repo_ids(a.repo_id): for f in sorted(glob.glob(str(home / r / "data" / "*" / "*.parquet"))): df = pd.read_parquet(f, columns=["observation.state", "action", "episode_index"]) S = np.stack(df["observation.state"].to_numpy()).astype(np.float32) A = np.stack(df["action"].to_numpy()).astype(np.float32) ep = df["episode_index"].to_numpy() starts = np.r_[0, np.flatnonzero(np.diff(ep)) + 1] ends = np.r_[starts[1:], len(ep)] last = np.repeat(ends - 1, ends - starts) # last frame of each frame's episode idx = np.minimum(np.arange(len(ep))[:, None] + np.arange(a.horizon)[None, :], last[:, None]) chunk = A[idx] - np.where(arm, S, 0.0)[:, None, :] stats["state"].update(S) stats["actions"].update(chunk) n += len(ep) out = SYT / "assets" / a.stats / dx.ASSET_ID normalize.save(out, {k: s.get_statistics() for k, s in stats.items()}) print(f"wrote {out / 'norm_stats.json'} from {n} frames") def cmd_loader_speed(a): from openpi.training import data_loader as dl cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) it = iter(dl.create_data_loader(cfg, shuffle=True, num_batches=a.batches + 1)) t0 = time.time() next(it) t1 = time.time() for _ in range(a.batches): next(it) print(f"loader: first batch {t1 - t0:.0f} s, then {(time.time() - t1) / a.batches:.2f} s/batch over {a.batches} " f"batches (batch {a.batch_size}, num_workers {a.num_workers})") def cmd_train(a): import jax from openpi.training import optimizer as _optimizer warmup = a.warmup if a.warmup is not None else _optimizer.CosineDecaySchedule().warmup_steps if (a.decay_steps or a.steps) <= warmup: raise SystemExit(f"decay end {a.decay_steps or a.steps} <= warmup {warmup}; short probes need --decay-steps") if a.batch_size % (a.grad_accum * a.fsdp): raise SystemExit(f"batch {a.batch_size} must divide into {a.grad_accum} micro-batches x {a.fsdp} GPUs") for d in jax.devices(): # claim every GPU before the data-loader warm-up (another job took GPUs once) jax.device_put(0.0, d).block_until_ready() print(f"claimed {jax.device_count()} GPUs: {[d.id for d in jax.devices()]}", flush=True) train = _load_script("train") base = _optimizer.CosineDecaySchedule() cfg = build_config( a, exp_name=a.exp_name, batch_size=a.batch_size, num_workers=a.num_workers, num_train_steps=a.steps, lr_schedule=_optimizer.CosineDecaySchedule(warmup_steps=warmup, peak_lr=a.peak_lr or base.peak_lr, decay_steps=a.decay_steps or a.steps, decay_lr=(a.peak_lr or base.peak_lr) * ( 1.0 if a.constant_lr else base.decay_lr / base.peak_lr)), save_interval=a.save_interval, keep_period=a.keep_period or a.save_interval, fsdp_devices=a.fsdp, log_interval=a.log_interval, overwrite=a.overwrite, resume=a.resume) if a.init_from: import dataclasses from openpi.training import weight_loaders cfg = dataclasses.replace(cfg, weight_loader=weight_loaders.CheckpointWeightLoader(a.init_from)) print("init from", a.init_from, flush=True) if a.freeze_vlm: import dataclasses import flax.nnx as nnx from openpi.shared import nnx_utils vlm = nnx.Any(nnx_utils.PathRegex(".*img.*"), nnx.All(nnx_utils.PathRegex(".*llm.*"), nnx.Not(nnx_utils.PathRegex(".*llm.*_1.*")))) cfg = dataclasses.replace(cfg, freeze_filter=vlm, ema_decay=None) print("freeze: vision encoder + PaliGemma LLM (action expert trainable), no EMA", flush=True) if a.fake_data: import dataclasses from openpi.training import config as _config cfg = dataclasses.replace(cfg, data=_config.FakeDataConfig(), num_workers=0) print("model:", cfg.model, "\nlr schedule:", cfg.lr_schedule, flush=True) _patch_train_state(train) if a.grad_accum > 1: import grad_accum train.train_step = grad_accum.make_train_step(a.grad_accum) train.main(cfg) def cmd_restore_check(a): import jax from openpi.training import checkpoints as _checkpoints from openpi.training import sharding train = _load_script("train") _patch_train_state(train) cfg = build_config(a, exp_name=a.exp_name, fsdp_devices=a.fsdp, resume=True) mngr, resuming = _checkpoints.initialize_checkpoint_dir(cfg.checkpoint_dir, keep_period=None, overwrite=False, resume=True) assert resuming, f"nothing to resume in {cfg.checkpoint_dir}" mesh = sharding.make_mesh(cfg.fsdp_devices) shape, _ = train.init_train_state(cfg, jax.random.key(cfg.seed), mesh, resume=True) state = _checkpoints.restore_state(mngr, shape, None) print("restored step", int(state.step), "of", mngr.all_steps(), flush=True) def main(): p = argparse.ArgumentParser() p.add_argument("cmd", choices=["prompt-len", "roundtrip", "smoke", "norm-stats", "norm-stats-parquet", "loader-speed", "train", "restore-check"]) p.add_argument("--repo-id", default="glob:dexhand/cover_v2_7seg_*") p.add_argument("--stats", default="dexhand_cover_v2", help="norm stats dir name under $SYT/assets") p.add_argument("--horizon", type=int, default=50, help="T future actions (model horizon 2T)") p.add_argument("--max-token-len", type=int, default=288) p.add_argument("--exp-name", default=None) p.add_argument("--batch-size", type=int, default=256) p.add_argument("--num-workers", type=int, default=16) p.add_argument("--max-frames", type=int, default=None) p.add_argument("--samples", type=int, default=2000) p.add_argument("--batches", type=int, default=20) p.add_argument("--steps", type=int, default=20_000) p.add_argument("--decay-steps", type=int, default=None) p.add_argument("--save-interval", type=int, default=5_000) p.add_argument("--log-interval", type=int, default=100) p.add_argument("--fsdp", type=int, default=8) p.add_argument("--grad-accum", type=int, default=1) p.add_argument("--fake-data", action="store_true") p.add_argument("--skip-norm", action="store_true", help="smoke: before norm stats exist") p.add_argument("--overwrite", action="store_true") p.add_argument("--resume", action="store_true") p.add_argument("--init-from", default=None, help="train: initialise from these params (e.g. a trained checkpoint's " "/params) instead of pi05_base - fine-tuning on a data supplement") p.add_argument("--peak-lr", type=float, default=None, help="train: peak LR (default openpi's 2.5e-5)") p.add_argument("--warmup", type=int, default=None, help="train: warmup steps (default openpi's 1000)") p.add_argument("--constant-lr", action="store_true", help="train: the peak LR after the warmup, no cosine decay " "(a cosine whose end value is the peak) - training can be extended / resumed without a schedule edge") p.add_argument("--keep-period", type=int, default=None, help="train: keep the checkpoints at multiples of this step " "(default --save-interval); the newest one is always kept. Every checkpoint holds train_state " "(optimizer + EMA, 31 GB for a full fine-tune) for --resume") p.add_argument("--balance-tasks", action="store_true", help="sample tasks uniformly (dexhand_openpi.TaskBalancedDataset); " "mid-training over several batches") p.add_argument("--freeze-vlm", action="store_true", help="train: freeze the vision encoder + PaliGemma LLM (train the " "action expert and projections only) and drop EMA - fits a fine-tune on 3 x A100 40GB, where the full " "train state (params + Adam + EMA) ran out of memory (2026-10-15)") a = p.parse_args() dx.BALANCE_TASKS = bool(a.balance_tasks) {"prompt-len": cmd_prompt_len, "roundtrip": cmd_roundtrip, "smoke": cmd_smoke, "norm-stats": cmd_norm_stats, "norm-stats-parquet": cmd_norm_stats_parquet, "loader-speed": cmd_loader_speed, "train": cmd_train, "restore-check": cmd_restore_check}[a.cmd](a) if __name__ == "__main__": main()