"""Entry points for training pi0.5 on our RoboDojo augmentation data on the A100 box, without editing openpi. Builds the config with aug/train/configs.py (get_config) and hands it to openpi's own scripts. Run from the openpi dir with the openpi venv, after `source $SYT/env.sh`: norm-stats compute state/action normalisation stats on the training set .venv/bin/python $SYT/code/runner/run.py norm-stats --repo-id cover_sub smoke build the data loader and print one batch's shapes (no norm stats needed) .venv/bin/python $SYT/code/runner/run.py smoke --repo-id cover_sub_test loader-speed time N batches of the real training data loader (batch 256) .venv/bin/python $SYT/code/runner/run.py loader-speed --repo-id cover_sub --batches 20 --num-workers 32 train openpi training (probe: --steps 100 --save-interval 100000) .venv/bin/python $SYT/code/runner/run.py train --repo-id cover_sub --exp-name cover_sub_v1 --grad-accum K: K micro-batches per optimizer step (grad_accum.py), for batches that do not fit in memory --fake-data: openpi's FakeDataConfig (same tensor shapes, no data loader warm-up): memory/speed probes in minutes restore-check restore the latest checkpoint of --exp-name the way --resume does (no data loader, ~minutes) and print its step and norms; proves a run is resumable .venv/bin/python $SYT/code/runner/run.py restore-check --repo-id cover_sub --exp-name cover_sub_v2 --fsdp 4 Paths: norm stats $SYT/assets//arx_x5_sim/norm_stats.json checkpoints $SYT/checkpoints/robodojo_aug_// """ import argparse import importlib.util import os import pathlib import sys import time SYT = pathlib.Path(os.environ["SYT"]) OPENPI = SYT / "code/XPolicyLab/policy/Pi_05/openpi" ASSET_ID = "arx_x5_sim" # asset_id of the official config; our stats live under our own assets dir sys.path.insert(0, str(SYT / "code/aug")) sys.path.insert(0, str(SYT / "code/runner")) import lazy_masks # noqa: E402 decode target masks only for the sampled frame (see lazy_masks.py) lazy_masks.install() def _load_script(name): spec = importlib.util.spec_from_file_location(f"openpi_{name}", OPENPI / "scripts" / f"{name}.py") mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def _patch_train_state(train): """Two fixes around openpi's train-state setup (no change to what is trained): A. Fresh start: init_train_state puts a replicated copy of all pi05_base weights on every GPU and then shards it. Freeing that copy leaves the GPU allocator fragmented (4x A100-40GB: ~22 GiB free in two ~11 GiB holes, while one step needs a 14-15 GiB temp buffer -> OOM). Re-place the state from host memory so it is packed at the start. B. --resume with this orbax (0.11.13), which refuses to restore an array without a target sharding ("sharding passed to deserialization should be specified ... Got None"): 1. init_train_state(resume=True) returns jax.eval_shape() structs; attach the FSDP sharding it computes anyway; 2. restore_state passes bare pytrees (legacy API), from which orbax does not take that sharding; pass explicit PyTreeRestore args built from it instead. Same items, same split/merge of EMA params as openpi. Restoring onto the current mesh also reshards, e.g. a FSDP-8 checkpoint into a FSDP-4 run.""" import jax import orbax.checkpoint as ocp from openpi.shared import array_typing as at from openpi.training import checkpoints as _checkpoints init_train_state = train.init_train_state def init_with_sharding(config, init_rng, mesh, *, resume): state, state_sharding = init_train_state(config, init_rng, mesh, resume=resume) if resume: state = jax.tree.map(lambda s, sh: jax.ShapeDtypeStruct(s.shape, s.dtype, sharding=sh), state, state_sharding) else: with at.disable_typechecking(): host = jax.device_get(state) jax.tree.map(lambda x: x.delete(), state) state = jax.device_put(host, state_sharding) return state, state_sharding def restore_state(checkpoint_manager, state, data_loader, step=None): del data_loader with at.disable_typechecking(): train_state, params = _checkpoints._split_params(state) items = {"train_state": train_state, "params": {"params": params}} restored = checkpoint_manager.restore(step, args=ocp.args.Composite(**{ k: ocp.args.PyTreeRestore(item=v, restore_args=ocp.checkpoint_utils.construct_restore_args(v)) for k, v in items.items()})) return _checkpoints._merge_params(restored["train_state"], restored["params"]) train.init_train_state = init_with_sharding _checkpoints.restore_state = restore_state # train.main looks it up as _checkpoints.restore_state def build_config(a, **overrides): from train.configs import get_config return get_config( a.kind, repo_id=a.repo_id, norm_assets_dir=str(SYT / "assets" / a.repo_id), assets_base_dir=str(SYT / "assets"), checkpoint_base_dir=str(SYT / "checkpoints"), wandb_enabled=False, **overrides, ) def cmd_smoke(a): from openpi.training import data_loader as dl cfg = build_config(a, batch_size=8, num_workers=0) # must divide by the 8 devices obs, act = next(iter(dl.create_data_loader(cfg, shuffle=True, num_batches=1, skip_norm_stats=True))) print({k: v.shape for k, v in obs.images.items()}, obs.state.shape, act.shape) print("prompt tokens:", None if obs.tokenized_prompt is None else obs.tokenized_prompt.shape) def cmd_norm_stats(a): """openpi scripts/compute_norm_stats.py (torch-dataset branch), with RemoveStrings from an importable module.""" import numpy as np import tqdm from openpi.shared import normalize from openpi.training import data_loader as dl cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) data_config = cfg.data.create(cfg.assets_dirs, cfg.model) dataset = dl.create_torch_dataset(data_config, cfg.model.action_horizon, cfg.model) dataset = dl.TransformedDataset( dataset, [*data_config.repack_transforms.inputs, *data_config.data_transforms.inputs, lazy_masks.RemoveStrings()] ) if a.max_frames is not None and a.max_frames < len(dataset): n, shuffle = a.max_frames // cfg.batch_size, True else: n, shuffle = len(dataset) // cfg.batch_size, False loader = dl.TorchDataLoader(dataset, local_batch_size=cfg.batch_size, num_workers=cfg.num_workers, shuffle=shuffle, num_batches=n) stats = {k: normalize.RunningStats() for k in ("state", "actions")} for batch in tqdm.tqdm(loader, total=n, desc="norm stats", mininterval=60): for k in stats: stats[k].update(np.asarray(batch[k])) out = SYT / "assets" / a.repo_id / ASSET_ID normalize.save(out, {k: s.get_statistics() for k, s in stats.items()}) print("wrote", out / "norm_stats.json") def cmd_loader_speed(a): """Steady-state throughput: time --batches batches after the first one; use --batches >= 3x --num-workers, otherwise the workers' prefetched batches make it look free.""" from openpi.training import data_loader as dl cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) it = iter(dl.create_data_loader(cfg, shuffle=True, num_batches=a.batches + 1, skip_norm_stats=True)) t0 = time.time() next(it) t1 = time.time() for _ in range(a.batches): next(it) dt = (time.time() - t1) / a.batches print(f"loader: first batch {t1 - t0:.0f} s, then {dt:.2f} s/batch over {a.batches} batches " f"(batch {a.batch_size}, num_workers {a.num_workers})") def cmd_train(a): import jax from openpi.training import optimizer as _optimizer # fail before the ~15 min warm-up, not after it: optax rejects a decay that ends inside the warmup warmup = _optimizer.CosineDecaySchedule().warmup_steps if (a.decay_steps or a.steps) <= warmup: raise SystemExit(f"decay end {a.decay_steps or a.steps} <= warmup {warmup}; short probes need --decay-steps") if a.batch_size % (a.grad_accum * a.fsdp): raise SystemExit(f"batch {a.batch_size} must divide into {a.grad_accum} micro-batches x {a.fsdp} GPUs") # Claim every visible GPU now (XLA preallocates on first use). Otherwise the GPUs stay empty during the ~15 min # data-loader warm-up and another user's job can take them (run cover_sub_v1 died that way on GPUs 5-7). for d in jax.devices(): jax.device_put(0.0, d).block_until_ready() print(f"claimed {jax.device_count()} GPUs: {[d.id for d in jax.devices()]}", flush=True) train = _load_script("train") base = _optimizer.CosineDecaySchedule() # the official config uses the default schedule cfg = build_config( a, exp_name=a.exp_name, batch_size=a.batch_size, num_workers=a.num_workers, num_train_steps=a.steps, # cosine decay ends at --decay-steps (default: the last step), so a shortened run still anneals fully lr_schedule=_optimizer.CosineDecaySchedule( warmup_steps=base.warmup_steps, peak_lr=base.peak_lr, decay_steps=a.decay_steps or a.steps, decay_lr=base.decay_lr), save_interval=a.save_interval, keep_period=a.save_interval, fsdp_devices=a.fsdp, log_interval=a.log_interval, overwrite=a.overwrite, resume=a.resume, ) if a.fake_data: import dataclasses from openpi.training import config as _config # FakeDataset draws with jax: worker processes would try to grab the already-claimed GPUs cfg = dataclasses.replace(cfg, data=_config.FakeDataConfig(), num_workers=0) print("lr schedule:", cfg.lr_schedule, flush=True) _patch_train_state(train) if a.grad_accum > 1: import grad_accum train.train_step = grad_accum.make_train_step(a.grad_accum) # train.main jits the module-level train_step print(f"gradient accumulation: {a.grad_accum} micro-batches of {a.batch_size // a.grad_accum}", flush=True) train.main(cfg) def cmd_restore_check(a): """Restore the latest checkpoint exactly as train.main does on --resume, skipping the data loader.""" import re import jax import numpy as np from openpi.training import checkpoints as _checkpoints from openpi.training import sharding train = _load_script("train") _patch_train_state(train) cfg = build_config(a, exp_name=a.exp_name, fsdp_devices=a.fsdp, resume=True) mngr, resuming = _checkpoints.initialize_checkpoint_dir(cfg.checkpoint_dir, keep_period=None, overwrite=False, resume=True) assert resuming, f"nothing to resume in {cfg.checkpoint_dir}" print("checkpoints:", mngr.all_steps(), "-> restoring", mngr.latest_step(), flush=True) mesh = sharding.make_mesh(cfg.fsdp_devices) shape, _ = train.init_train_state(cfg, jax.random.key(cfg.seed), mesh, resume=True) state = _checkpoints.restore_state(mngr, shape, None) def norm(tree, pred=lambda path, x: True): leaves = [x for path, x in jax.tree_util.tree_flatten_with_path(tree)[0] if hasattr(x, "ndim") and pred(jax.tree_util.keystr(path), x)] # float64 on host: exact to ~1e-12, so it is the reference for the float32 GPU value in the training log return float(np.sqrt(sum(np.sum(np.square(np.asarray(x, dtype=np.float64))) for x in leaves))) # same filter as train_step's logged param_norm: kernels only kernel = lambda path, x: x.ndim > 1 and not re.search(r"(bias|scale|pos_embedding|input_embedding)", path) print(f"step {int(state.step)} param_norm(kernels) {norm(state.params, kernel):.4f} " f"ema_param_norm(kernels) {norm(state.ema_params, kernel):.4f} opt_state_norm {norm(state.opt_state):.4f}", flush=True) def main(): p = argparse.ArgumentParser() p.add_argument("cmd", choices=["smoke", "norm-stats", "loader-speed", "train", "restore-check"]) p.add_argument("--kind", default="subtask", choices=["subtask", "full"]) p.add_argument("--repo-id", required=True) p.add_argument("--exp-name", default=None) p.add_argument("--batch-size", type=int, default=256) p.add_argument("--num-workers", type=int, default=16) p.add_argument("--max-frames", type=int, default=None) p.add_argument("--batches", type=int, default=20) p.add_argument("--steps", type=int, default=30_000) p.add_argument("--decay-steps", type=int, default=None, help="end of the cosine LR decay (default: --steps)") p.add_argument("--save-interval", type=int, default=10_000) p.add_argument("--log-interval", type=int, default=100) p.add_argument("--fsdp", type=int, default=8) p.add_argument("--grad-accum", type=int, default=1, help="micro-batches per optimizer step (train)") p.add_argument("--fake-data", action="store_true", help="train: memory/speed probe on openpi's fake data") p.add_argument("--overwrite", action="store_true") p.add_argument("--resume", action="store_true") a = p.parse_args() {"smoke": cmd_smoke, "norm-stats": cmd_norm_stats, "loader-speed": cmd_loader_speed, "train": cmd_train, "restore-check": cmd_restore_check}[a.cmd](a) if __name__ == "__main__": main()