Download code/runner/run.py from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 13.6 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/run.py
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/runner/run.py
-
curl -L -o run.py https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/run.py
13.6 kB
| """Entry points for training pi0.5 on our RoboDojo augmentation data on the A100 box, without editing openpi. | |
| Builds the config with aug/train/configs.py (get_config) and hands it to openpi's own scripts. | |
| Run from the openpi dir with the openpi venv, after `source $SYT/env.sh`: | |
| norm-stats compute state/action normalisation stats on the training set | |
| .venv/bin/python $SYT/code/runner/run.py norm-stats --repo-id cover_sub | |
| smoke build the data loader and print one batch's shapes (no norm stats needed) | |
| .venv/bin/python $SYT/code/runner/run.py smoke --repo-id cover_sub_test | |
| loader-speed time N batches of the real training data loader (batch 256) | |
| .venv/bin/python $SYT/code/runner/run.py loader-speed --repo-id cover_sub --batches 20 --num-workers 32 | |
| train openpi training (probe: --steps 100 --save-interval 100000) | |
| .venv/bin/python $SYT/code/runner/run.py train --repo-id cover_sub --exp-name cover_sub_v1 | |
| --grad-accum K: K micro-batches per optimizer step (grad_accum.py), for batches that do not fit in memory | |
| --fake-data: openpi's FakeDataConfig (same tensor shapes, no data loader warm-up): memory/speed probes in minutes | |
| restore-check restore the latest checkpoint of --exp-name the way --resume does (no data loader, ~minutes) and | |
| print its step and norms; proves a run is resumable | |
| .venv/bin/python $SYT/code/runner/run.py restore-check --repo-id cover_sub --exp-name cover_sub_v2 --fsdp 4 | |
| Paths: norm stats $SYT/assets/<repo_id>/arx_x5_sim/norm_stats.json | |
| checkpoints $SYT/checkpoints/robodojo_aug_<kind>/<exp_name>/<step> | |
| """ | |
| import argparse | |
| import importlib.util | |
| import os | |
| import pathlib | |
| import sys | |
| import time | |
| SYT = pathlib.Path(os.environ["SYT"]) | |
| OPENPI = SYT / "code/XPolicyLab/policy/Pi_05/openpi" | |
| ASSET_ID = "arx_x5_sim" # asset_id of the official config; our stats live under our own assets dir | |
| sys.path.insert(0, str(SYT / "code/aug")) | |
| sys.path.insert(0, str(SYT / "code/runner")) | |
| import lazy_masks # noqa: E402 decode target masks only for the sampled frame (see lazy_masks.py) | |
| lazy_masks.install() | |
| def _load_script(name): | |
| spec = importlib.util.spec_from_file_location(f"openpi_{name}", OPENPI / "scripts" / f"{name}.py") | |
| mod = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(mod) | |
| return mod | |
| def _patch_train_state(train): | |
| """Two fixes around openpi's train-state setup (no change to what is trained): | |
| A. Fresh start: init_train_state puts a replicated copy of all pi05_base weights on every GPU and then shards it. | |
| Freeing that copy leaves the GPU allocator fragmented (4x A100-40GB: ~22 GiB free in two ~11 GiB holes, while | |
| one step needs a 14-15 GiB temp buffer -> OOM). Re-place the state from host memory so it is packed at the start. | |
| B. --resume with this orbax (0.11.13), which refuses to restore an array without a target sharding | |
| ("sharding passed to deserialization should be specified ... Got None"): | |
| 1. init_train_state(resume=True) returns jax.eval_shape() structs; attach the FSDP sharding it computes anyway; | |
| 2. restore_state passes bare pytrees (legacy API), from which orbax does not take that sharding; pass explicit | |
| PyTreeRestore args built from it instead. Same items, same split/merge of EMA params as openpi. | |
| Restoring onto the current mesh also reshards, e.g. a FSDP-8 checkpoint into a FSDP-4 run.""" | |
| import jax | |
| import orbax.checkpoint as ocp | |
| from openpi.shared import array_typing as at | |
| from openpi.training import checkpoints as _checkpoints | |
| init_train_state = train.init_train_state | |
| def init_with_sharding(config, init_rng, mesh, *, resume): | |
| state, state_sharding = init_train_state(config, init_rng, mesh, resume=resume) | |
| if resume: | |
| state = jax.tree.map(lambda s, sh: jax.ShapeDtypeStruct(s.shape, s.dtype, sharding=sh), state, state_sharding) | |
| else: | |
| with at.disable_typechecking(): | |
| host = jax.device_get(state) | |
| jax.tree.map(lambda x: x.delete(), state) | |
| state = jax.device_put(host, state_sharding) | |
| return state, state_sharding | |
| def restore_state(checkpoint_manager, state, data_loader, step=None): | |
| del data_loader | |
| with at.disable_typechecking(): | |
| train_state, params = _checkpoints._split_params(state) | |
| items = {"train_state": train_state, "params": {"params": params}} | |
| restored = checkpoint_manager.restore(step, args=ocp.args.Composite(**{ | |
| k: ocp.args.PyTreeRestore(item=v, restore_args=ocp.checkpoint_utils.construct_restore_args(v)) | |
| for k, v in items.items()})) | |
| return _checkpoints._merge_params(restored["train_state"], restored["params"]) | |
| train.init_train_state = init_with_sharding | |
| _checkpoints.restore_state = restore_state # train.main looks it up as _checkpoints.restore_state | |
| def build_config(a, **overrides): | |
| from train.configs import get_config | |
| return get_config( | |
| a.kind, | |
| repo_id=a.repo_id, | |
| norm_assets_dir=str(SYT / "assets" / a.repo_id), | |
| assets_base_dir=str(SYT / "assets"), | |
| checkpoint_base_dir=str(SYT / "checkpoints"), | |
| wandb_enabled=False, | |
| **overrides, | |
| ) | |
| def cmd_smoke(a): | |
| from openpi.training import data_loader as dl | |
| cfg = build_config(a, batch_size=8, num_workers=0) # must divide by the 8 devices | |
| obs, act = next(iter(dl.create_data_loader(cfg, shuffle=True, num_batches=1, skip_norm_stats=True))) | |
| print({k: v.shape for k, v in obs.images.items()}, obs.state.shape, act.shape) | |
| print("prompt tokens:", None if obs.tokenized_prompt is None else obs.tokenized_prompt.shape) | |
| def cmd_norm_stats(a): | |
| """openpi scripts/compute_norm_stats.py (torch-dataset branch), with RemoveStrings from an importable module.""" | |
| import numpy as np | |
| import tqdm | |
| from openpi.shared import normalize | |
| from openpi.training import data_loader as dl | |
| cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) | |
| data_config = cfg.data.create(cfg.assets_dirs, cfg.model) | |
| dataset = dl.create_torch_dataset(data_config, cfg.model.action_horizon, cfg.model) | |
| dataset = dl.TransformedDataset( | |
| dataset, [*data_config.repack_transforms.inputs, *data_config.data_transforms.inputs, lazy_masks.RemoveStrings()] | |
| ) | |
| if a.max_frames is not None and a.max_frames < len(dataset): | |
| n, shuffle = a.max_frames // cfg.batch_size, True | |
| else: | |
| n, shuffle = len(dataset) // cfg.batch_size, False | |
| loader = dl.TorchDataLoader(dataset, local_batch_size=cfg.batch_size, num_workers=cfg.num_workers, | |
| shuffle=shuffle, num_batches=n) | |
| stats = {k: normalize.RunningStats() for k in ("state", "actions")} | |
| for batch in tqdm.tqdm(loader, total=n, desc="norm stats", mininterval=60): | |
| for k in stats: | |
| stats[k].update(np.asarray(batch[k])) | |
| out = SYT / "assets" / a.repo_id / ASSET_ID | |
| normalize.save(out, {k: s.get_statistics() for k, s in stats.items()}) | |
| print("wrote", out / "norm_stats.json") | |
| def cmd_loader_speed(a): | |
| """Steady-state throughput: time --batches batches after the first one; use --batches >= 3x --num-workers, | |
| otherwise the workers' prefetched batches make it look free.""" | |
| from openpi.training import data_loader as dl | |
| cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers) | |
| it = iter(dl.create_data_loader(cfg, shuffle=True, num_batches=a.batches + 1, skip_norm_stats=True)) | |
| t0 = time.time() | |
| next(it) | |
| t1 = time.time() | |
| for _ in range(a.batches): | |
| next(it) | |
| dt = (time.time() - t1) / a.batches | |
| print(f"loader: first batch {t1 - t0:.0f} s, then {dt:.2f} s/batch over {a.batches} batches " | |
| f"(batch {a.batch_size}, num_workers {a.num_workers})") | |
| def cmd_train(a): | |
| import jax | |
| from openpi.training import optimizer as _optimizer | |
| # fail before the ~15 min warm-up, not after it: optax rejects a decay that ends inside the warmup | |
| warmup = _optimizer.CosineDecaySchedule().warmup_steps | |
| if (a.decay_steps or a.steps) <= warmup: | |
| raise SystemExit(f"decay end {a.decay_steps or a.steps} <= warmup {warmup}; short probes need --decay-steps") | |
| if a.batch_size % (a.grad_accum * a.fsdp): | |
| raise SystemExit(f"batch {a.batch_size} must divide into {a.grad_accum} micro-batches x {a.fsdp} GPUs") | |
| # Claim every visible GPU now (XLA preallocates on first use). Otherwise the GPUs stay empty during the ~15 min | |
| # data-loader warm-up and another user's job can take them (run cover_sub_v1 died that way on GPUs 5-7). | |
| for d in jax.devices(): | |
| jax.device_put(0.0, d).block_until_ready() | |
| print(f"claimed {jax.device_count()} GPUs: {[d.id for d in jax.devices()]}", flush=True) | |
| train = _load_script("train") | |
| base = _optimizer.CosineDecaySchedule() # the official config uses the default schedule | |
| cfg = build_config( | |
| a, | |
| exp_name=a.exp_name, | |
| batch_size=a.batch_size, | |
| num_workers=a.num_workers, | |
| num_train_steps=a.steps, | |
| # cosine decay ends at --decay-steps (default: the last step), so a shortened run still anneals fully | |
| lr_schedule=_optimizer.CosineDecaySchedule( | |
| warmup_steps=base.warmup_steps, peak_lr=base.peak_lr, | |
| decay_steps=a.decay_steps or a.steps, decay_lr=base.decay_lr), | |
| save_interval=a.save_interval, | |
| keep_period=a.save_interval, | |
| fsdp_devices=a.fsdp, | |
| log_interval=a.log_interval, | |
| overwrite=a.overwrite, | |
| resume=a.resume, | |
| ) | |
| if a.fake_data: | |
| import dataclasses | |
| from openpi.training import config as _config | |
| # FakeDataset draws with jax: worker processes would try to grab the already-claimed GPUs | |
| cfg = dataclasses.replace(cfg, data=_config.FakeDataConfig(), num_workers=0) | |
| print("lr schedule:", cfg.lr_schedule, flush=True) | |
| _patch_train_state(train) | |
| if a.grad_accum > 1: | |
| import grad_accum | |
| train.train_step = grad_accum.make_train_step(a.grad_accum) # train.main jits the module-level train_step | |
| print(f"gradient accumulation: {a.grad_accum} micro-batches of {a.batch_size // a.grad_accum}", flush=True) | |
| train.main(cfg) | |
| def cmd_restore_check(a): | |
| """Restore the latest checkpoint exactly as train.main does on --resume, skipping the data loader.""" | |
| import re | |
| import jax | |
| import numpy as np | |
| from openpi.training import checkpoints as _checkpoints | |
| from openpi.training import sharding | |
| train = _load_script("train") | |
| _patch_train_state(train) | |
| cfg = build_config(a, exp_name=a.exp_name, fsdp_devices=a.fsdp, resume=True) | |
| mngr, resuming = _checkpoints.initialize_checkpoint_dir(cfg.checkpoint_dir, keep_period=None, overwrite=False, | |
| resume=True) | |
| assert resuming, f"nothing to resume in {cfg.checkpoint_dir}" | |
| print("checkpoints:", mngr.all_steps(), "-> restoring", mngr.latest_step(), flush=True) | |
| mesh = sharding.make_mesh(cfg.fsdp_devices) | |
| shape, _ = train.init_train_state(cfg, jax.random.key(cfg.seed), mesh, resume=True) | |
| state = _checkpoints.restore_state(mngr, shape, None) | |
| def norm(tree, pred=lambda path, x: True): | |
| leaves = [x for path, x in jax.tree_util.tree_flatten_with_path(tree)[0] | |
| if hasattr(x, "ndim") and pred(jax.tree_util.keystr(path), x)] | |
| # float64 on host: exact to ~1e-12, so it is the reference for the float32 GPU value in the training log | |
| return float(np.sqrt(sum(np.sum(np.square(np.asarray(x, dtype=np.float64))) for x in leaves))) | |
| # same filter as train_step's logged param_norm: kernels only | |
| kernel = lambda path, x: x.ndim > 1 and not re.search(r"(bias|scale|pos_embedding|input_embedding)", path) | |
| print(f"step {int(state.step)} param_norm(kernels) {norm(state.params, kernel):.4f} " | |
| f"ema_param_norm(kernels) {norm(state.ema_params, kernel):.4f} opt_state_norm {norm(state.opt_state):.4f}", | |
| flush=True) | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("cmd", choices=["smoke", "norm-stats", "loader-speed", "train", "restore-check"]) | |
| p.add_argument("--kind", default="subtask", choices=["subtask", "full"]) | |
| p.add_argument("--repo-id", required=True) | |
| p.add_argument("--exp-name", default=None) | |
| p.add_argument("--batch-size", type=int, default=256) | |
| p.add_argument("--num-workers", type=int, default=16) | |
| p.add_argument("--max-frames", type=int, default=None) | |
| p.add_argument("--batches", type=int, default=20) | |
| p.add_argument("--steps", type=int, default=30_000) | |
| p.add_argument("--decay-steps", type=int, default=None, help="end of the cosine LR decay (default: --steps)") | |
| p.add_argument("--save-interval", type=int, default=10_000) | |
| p.add_argument("--log-interval", type=int, default=100) | |
| p.add_argument("--fsdp", type=int, default=8) | |
| p.add_argument("--grad-accum", type=int, default=1, help="micro-batches per optimizer step (train)") | |
| p.add_argument("--fake-data", action="store_true", help="train: memory/speed probe on openpi's fake data") | |
| p.add_argument("--overwrite", action="store_true") | |
| p.add_argument("--resume", action="store_true") | |
| a = p.parse_args() | |
| {"smoke": cmd_smoke, "norm-stats": cmd_norm_stats, "loader-speed": cmd_loader_speed, "train": cmd_train, | |
| "restore-check": cmd_restore_check}[a.cmd](a) | |
| if __name__ == "__main__": | |
| main() | |