Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
13.6 kB
"""Entry points for training pi0.5 on our RoboDojo augmentation data on the A100 box, without editing openpi.
Builds the config with aug/train/configs.py (get_config) and hands it to openpi's own scripts.
Run from the openpi dir with the openpi venv, after `source $SYT/env.sh`:
norm-stats compute state/action normalisation stats on the training set
.venv/bin/python $SYT/code/runner/run.py norm-stats --repo-id cover_sub
smoke build the data loader and print one batch's shapes (no norm stats needed)
.venv/bin/python $SYT/code/runner/run.py smoke --repo-id cover_sub_test
loader-speed time N batches of the real training data loader (batch 256)
.venv/bin/python $SYT/code/runner/run.py loader-speed --repo-id cover_sub --batches 20 --num-workers 32
train openpi training (probe: --steps 100 --save-interval 100000)
.venv/bin/python $SYT/code/runner/run.py train --repo-id cover_sub --exp-name cover_sub_v1
--grad-accum K: K micro-batches per optimizer step (grad_accum.py), for batches that do not fit in memory
--fake-data: openpi's FakeDataConfig (same tensor shapes, no data loader warm-up): memory/speed probes in minutes
restore-check restore the latest checkpoint of --exp-name the way --resume does (no data loader, ~minutes) and
print its step and norms; proves a run is resumable
.venv/bin/python $SYT/code/runner/run.py restore-check --repo-id cover_sub --exp-name cover_sub_v2 --fsdp 4
Paths: norm stats $SYT/assets/<repo_id>/arx_x5_sim/norm_stats.json
checkpoints $SYT/checkpoints/robodojo_aug_<kind>/<exp_name>/<step>
"""
import argparse
import importlib.util
import os
import pathlib
import sys
import time
SYT = pathlib.Path(os.environ["SYT"])
OPENPI = SYT / "code/XPolicyLab/policy/Pi_05/openpi"
ASSET_ID = "arx_x5_sim" # asset_id of the official config; our stats live under our own assets dir
sys.path.insert(0, str(SYT / "code/aug"))
sys.path.insert(0, str(SYT / "code/runner"))
import lazy_masks # noqa: E402 decode target masks only for the sampled frame (see lazy_masks.py)
lazy_masks.install()
def _load_script(name):
spec = importlib.util.spec_from_file_location(f"openpi_{name}", OPENPI / "scripts" / f"{name}.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _patch_train_state(train):
"""Two fixes around openpi's train-state setup (no change to what is trained):
A. Fresh start: init_train_state puts a replicated copy of all pi05_base weights on every GPU and then shards it.
Freeing that copy leaves the GPU allocator fragmented (4x A100-40GB: ~22 GiB free in two ~11 GiB holes, while
one step needs a 14-15 GiB temp buffer -> OOM). Re-place the state from host memory so it is packed at the start.
B. --resume with this orbax (0.11.13), which refuses to restore an array without a target sharding
("sharding passed to deserialization should be specified ... Got None"):
1. init_train_state(resume=True) returns jax.eval_shape() structs; attach the FSDP sharding it computes anyway;
2. restore_state passes bare pytrees (legacy API), from which orbax does not take that sharding; pass explicit
PyTreeRestore args built from it instead. Same items, same split/merge of EMA params as openpi.
Restoring onto the current mesh also reshards, e.g. a FSDP-8 checkpoint into a FSDP-4 run."""
import jax
import orbax.checkpoint as ocp
from openpi.shared import array_typing as at
from openpi.training import checkpoints as _checkpoints
init_train_state = train.init_train_state
def init_with_sharding(config, init_rng, mesh, *, resume):
state, state_sharding = init_train_state(config, init_rng, mesh, resume=resume)
if resume:
state = jax.tree.map(lambda s, sh: jax.ShapeDtypeStruct(s.shape, s.dtype, sharding=sh), state, state_sharding)
else:
with at.disable_typechecking():
host = jax.device_get(state)
jax.tree.map(lambda x: x.delete(), state)
state = jax.device_put(host, state_sharding)
return state, state_sharding
def restore_state(checkpoint_manager, state, data_loader, step=None):
del data_loader
with at.disable_typechecking():
train_state, params = _checkpoints._split_params(state)
items = {"train_state": train_state, "params": {"params": params}}
restored = checkpoint_manager.restore(step, args=ocp.args.Composite(**{
k: ocp.args.PyTreeRestore(item=v, restore_args=ocp.checkpoint_utils.construct_restore_args(v))
for k, v in items.items()}))
return _checkpoints._merge_params(restored["train_state"], restored["params"])
train.init_train_state = init_with_sharding
_checkpoints.restore_state = restore_state # train.main looks it up as _checkpoints.restore_state
def build_config(a, **overrides):
from train.configs import get_config
return get_config(
a.kind,
repo_id=a.repo_id,
norm_assets_dir=str(SYT / "assets" / a.repo_id),
assets_base_dir=str(SYT / "assets"),
checkpoint_base_dir=str(SYT / "checkpoints"),
wandb_enabled=False,
**overrides,
)
def cmd_smoke(a):
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=8, num_workers=0) # must divide by the 8 devices
obs, act = next(iter(dl.create_data_loader(cfg, shuffle=True, num_batches=1, skip_norm_stats=True)))
print({k: v.shape for k, v in obs.images.items()}, obs.state.shape, act.shape)
print("prompt tokens:", None if obs.tokenized_prompt is None else obs.tokenized_prompt.shape)
def cmd_norm_stats(a):
"""openpi scripts/compute_norm_stats.py (torch-dataset branch), with RemoveStrings from an importable module."""
import numpy as np
import tqdm
from openpi.shared import normalize
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers)
data_config = cfg.data.create(cfg.assets_dirs, cfg.model)
dataset = dl.create_torch_dataset(data_config, cfg.model.action_horizon, cfg.model)
dataset = dl.TransformedDataset(
dataset, [*data_config.repack_transforms.inputs, *data_config.data_transforms.inputs, lazy_masks.RemoveStrings()]
)
if a.max_frames is not None and a.max_frames < len(dataset):
n, shuffle = a.max_frames // cfg.batch_size, True
else:
n, shuffle = len(dataset) // cfg.batch_size, False
loader = dl.TorchDataLoader(dataset, local_batch_size=cfg.batch_size, num_workers=cfg.num_workers,
shuffle=shuffle, num_batches=n)
stats = {k: normalize.RunningStats() for k in ("state", "actions")}
for batch in tqdm.tqdm(loader, total=n, desc="norm stats", mininterval=60):
for k in stats:
stats[k].update(np.asarray(batch[k]))
out = SYT / "assets" / a.repo_id / ASSET_ID
normalize.save(out, {k: s.get_statistics() for k, s in stats.items()})
print("wrote", out / "norm_stats.json")
def cmd_loader_speed(a):
"""Steady-state throughput: time --batches batches after the first one; use --batches >= 3x --num-workers,
otherwise the workers' prefetched batches make it look free."""
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers)
it = iter(dl.create_data_loader(cfg, shuffle=True, num_batches=a.batches + 1, skip_norm_stats=True))
t0 = time.time()
next(it)
t1 = time.time()
for _ in range(a.batches):
next(it)
dt = (time.time() - t1) / a.batches
print(f"loader: first batch {t1 - t0:.0f} s, then {dt:.2f} s/batch over {a.batches} batches "
f"(batch {a.batch_size}, num_workers {a.num_workers})")
def cmd_train(a):
import jax
from openpi.training import optimizer as _optimizer
# fail before the ~15 min warm-up, not after it: optax rejects a decay that ends inside the warmup
warmup = _optimizer.CosineDecaySchedule().warmup_steps
if (a.decay_steps or a.steps) <= warmup:
raise SystemExit(f"decay end {a.decay_steps or a.steps} <= warmup {warmup}; short probes need --decay-steps")
if a.batch_size % (a.grad_accum * a.fsdp):
raise SystemExit(f"batch {a.batch_size} must divide into {a.grad_accum} micro-batches x {a.fsdp} GPUs")
# Claim every visible GPU now (XLA preallocates on first use). Otherwise the GPUs stay empty during the ~15 min
# data-loader warm-up and another user's job can take them (run cover_sub_v1 died that way on GPUs 5-7).
for d in jax.devices():
jax.device_put(0.0, d).block_until_ready()
print(f"claimed {jax.device_count()} GPUs: {[d.id for d in jax.devices()]}", flush=True)
train = _load_script("train")
base = _optimizer.CosineDecaySchedule() # the official config uses the default schedule
cfg = build_config(
a,
exp_name=a.exp_name,
batch_size=a.batch_size,
num_workers=a.num_workers,
num_train_steps=a.steps,
# cosine decay ends at --decay-steps (default: the last step), so a shortened run still anneals fully
lr_schedule=_optimizer.CosineDecaySchedule(
warmup_steps=base.warmup_steps, peak_lr=base.peak_lr,
decay_steps=a.decay_steps or a.steps, decay_lr=base.decay_lr),
save_interval=a.save_interval,
keep_period=a.save_interval,
fsdp_devices=a.fsdp,
log_interval=a.log_interval,
overwrite=a.overwrite,
resume=a.resume,
)
if a.fake_data:
import dataclasses
from openpi.training import config as _config
# FakeDataset draws with jax: worker processes would try to grab the already-claimed GPUs
cfg = dataclasses.replace(cfg, data=_config.FakeDataConfig(), num_workers=0)
print("lr schedule:", cfg.lr_schedule, flush=True)
_patch_train_state(train)
if a.grad_accum > 1:
import grad_accum
train.train_step = grad_accum.make_train_step(a.grad_accum) # train.main jits the module-level train_step
print(f"gradient accumulation: {a.grad_accum} micro-batches of {a.batch_size // a.grad_accum}", flush=True)
train.main(cfg)
def cmd_restore_check(a):
"""Restore the latest checkpoint exactly as train.main does on --resume, skipping the data loader."""
import re
import jax
import numpy as np
from openpi.training import checkpoints as _checkpoints
from openpi.training import sharding
train = _load_script("train")
_patch_train_state(train)
cfg = build_config(a, exp_name=a.exp_name, fsdp_devices=a.fsdp, resume=True)
mngr, resuming = _checkpoints.initialize_checkpoint_dir(cfg.checkpoint_dir, keep_period=None, overwrite=False,
resume=True)
assert resuming, f"nothing to resume in {cfg.checkpoint_dir}"
print("checkpoints:", mngr.all_steps(), "-> restoring", mngr.latest_step(), flush=True)
mesh = sharding.make_mesh(cfg.fsdp_devices)
shape, _ = train.init_train_state(cfg, jax.random.key(cfg.seed), mesh, resume=True)
state = _checkpoints.restore_state(mngr, shape, None)
def norm(tree, pred=lambda path, x: True):
leaves = [x for path, x in jax.tree_util.tree_flatten_with_path(tree)[0]
if hasattr(x, "ndim") and pred(jax.tree_util.keystr(path), x)]
# float64 on host: exact to ~1e-12, so it is the reference for the float32 GPU value in the training log
return float(np.sqrt(sum(np.sum(np.square(np.asarray(x, dtype=np.float64))) for x in leaves)))
# same filter as train_step's logged param_norm: kernels only
kernel = lambda path, x: x.ndim > 1 and not re.search(r"(bias|scale|pos_embedding|input_embedding)", path)
print(f"step {int(state.step)} param_norm(kernels) {norm(state.params, kernel):.4f} "
f"ema_param_norm(kernels) {norm(state.ema_params, kernel):.4f} opt_state_norm {norm(state.opt_state):.4f}",
flush=True)
def main():
p = argparse.ArgumentParser()
p.add_argument("cmd", choices=["smoke", "norm-stats", "loader-speed", "train", "restore-check"])
p.add_argument("--kind", default="subtask", choices=["subtask", "full"])
p.add_argument("--repo-id", required=True)
p.add_argument("--exp-name", default=None)
p.add_argument("--batch-size", type=int, default=256)
p.add_argument("--num-workers", type=int, default=16)
p.add_argument("--max-frames", type=int, default=None)
p.add_argument("--batches", type=int, default=20)
p.add_argument("--steps", type=int, default=30_000)
p.add_argument("--decay-steps", type=int, default=None, help="end of the cosine LR decay (default: --steps)")
p.add_argument("--save-interval", type=int, default=10_000)
p.add_argument("--log-interval", type=int, default=100)
p.add_argument("--fsdp", type=int, default=8)
p.add_argument("--grad-accum", type=int, default=1, help="micro-batches per optimizer step (train)")
p.add_argument("--fake-data", action="store_true", help="train: memory/speed probe on openpi's fake data")
p.add_argument("--overwrite", action="store_true")
p.add_argument("--resume", action="store_true")
a = p.parse_args()
{"smoke": cmd_smoke, "norm-stats": cmd_norm_stats, "loader-speed": cmd_loader_speed, "train": cmd_train,
"restore-check": cmd_restore_check}[a.cmd](a)
if __name__ == "__main__":
main()