File size: 13,636 Bytes
6a08a28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 | """Entry points for training pi0.5 on our RoboDojo augmentation data on the A100 box, without editing openpi.
Builds the config with aug/train/configs.py (get_config) and hands it to openpi's own scripts.
Run from the openpi dir with the openpi venv, after `source $SYT/env.sh`:
norm-stats compute state/action normalisation stats on the training set
.venv/bin/python $SYT/code/runner/run.py norm-stats --repo-id cover_sub
smoke build the data loader and print one batch's shapes (no norm stats needed)
.venv/bin/python $SYT/code/runner/run.py smoke --repo-id cover_sub_test
loader-speed time N batches of the real training data loader (batch 256)
.venv/bin/python $SYT/code/runner/run.py loader-speed --repo-id cover_sub --batches 20 --num-workers 32
train openpi training (probe: --steps 100 --save-interval 100000)
.venv/bin/python $SYT/code/runner/run.py train --repo-id cover_sub --exp-name cover_sub_v1
--grad-accum K: K micro-batches per optimizer step (grad_accum.py), for batches that do not fit in memory
--fake-data: openpi's FakeDataConfig (same tensor shapes, no data loader warm-up): memory/speed probes in minutes
restore-check restore the latest checkpoint of --exp-name the way --resume does (no data loader, ~minutes) and
print its step and norms; proves a run is resumable
.venv/bin/python $SYT/code/runner/run.py restore-check --repo-id cover_sub --exp-name cover_sub_v2 --fsdp 4
Paths: norm stats $SYT/assets/<repo_id>/arx_x5_sim/norm_stats.json
checkpoints $SYT/checkpoints/robodojo_aug_<kind>/<exp_name>/<step>
"""
import argparse
import importlib.util
import os
import pathlib
import sys
import time
SYT = pathlib.Path(os.environ["SYT"])
OPENPI = SYT / "code/XPolicyLab/policy/Pi_05/openpi"
ASSET_ID = "arx_x5_sim" # asset_id of the official config; our stats live under our own assets dir
sys.path.insert(0, str(SYT / "code/aug"))
sys.path.insert(0, str(SYT / "code/runner"))
import lazy_masks # noqa: E402 decode target masks only for the sampled frame (see lazy_masks.py)
lazy_masks.install()
def _load_script(name):
spec = importlib.util.spec_from_file_location(f"openpi_{name}", OPENPI / "scripts" / f"{name}.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _patch_train_state(train):
"""Two fixes around openpi's train-state setup (no change to what is trained):
A. Fresh start: init_train_state puts a replicated copy of all pi05_base weights on every GPU and then shards it.
Freeing that copy leaves the GPU allocator fragmented (4x A100-40GB: ~22 GiB free in two ~11 GiB holes, while
one step needs a 14-15 GiB temp buffer -> OOM). Re-place the state from host memory so it is packed at the start.
B. --resume with this orbax (0.11.13), which refuses to restore an array without a target sharding
("sharding passed to deserialization should be specified ... Got None"):
1. init_train_state(resume=True) returns jax.eval_shape() structs; attach the FSDP sharding it computes anyway;
2. restore_state passes bare pytrees (legacy API), from which orbax does not take that sharding; pass explicit
PyTreeRestore args built from it instead. Same items, same split/merge of EMA params as openpi.
Restoring onto the current mesh also reshards, e.g. a FSDP-8 checkpoint into a FSDP-4 run."""
import jax
import orbax.checkpoint as ocp
from openpi.shared import array_typing as at
from openpi.training import checkpoints as _checkpoints
init_train_state = train.init_train_state
def init_with_sharding(config, init_rng, mesh, *, resume):
state, state_sharding = init_train_state(config, init_rng, mesh, resume=resume)
if resume:
state = jax.tree.map(lambda s, sh: jax.ShapeDtypeStruct(s.shape, s.dtype, sharding=sh), state, state_sharding)
else:
with at.disable_typechecking():
host = jax.device_get(state)
jax.tree.map(lambda x: x.delete(), state)
state = jax.device_put(host, state_sharding)
return state, state_sharding
def restore_state(checkpoint_manager, state, data_loader, step=None):
del data_loader
with at.disable_typechecking():
train_state, params = _checkpoints._split_params(state)
items = {"train_state": train_state, "params": {"params": params}}
restored = checkpoint_manager.restore(step, args=ocp.args.Composite(**{
k: ocp.args.PyTreeRestore(item=v, restore_args=ocp.checkpoint_utils.construct_restore_args(v))
for k, v in items.items()}))
return _checkpoints._merge_params(restored["train_state"], restored["params"])
train.init_train_state = init_with_sharding
_checkpoints.restore_state = restore_state # train.main looks it up as _checkpoints.restore_state
def build_config(a, **overrides):
from train.configs import get_config
return get_config(
a.kind,
repo_id=a.repo_id,
norm_assets_dir=str(SYT / "assets" / a.repo_id),
assets_base_dir=str(SYT / "assets"),
checkpoint_base_dir=str(SYT / "checkpoints"),
wandb_enabled=False,
**overrides,
)
def cmd_smoke(a):
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=8, num_workers=0) # must divide by the 8 devices
obs, act = next(iter(dl.create_data_loader(cfg, shuffle=True, num_batches=1, skip_norm_stats=True)))
print({k: v.shape for k, v in obs.images.items()}, obs.state.shape, act.shape)
print("prompt tokens:", None if obs.tokenized_prompt is None else obs.tokenized_prompt.shape)
def cmd_norm_stats(a):
"""openpi scripts/compute_norm_stats.py (torch-dataset branch), with RemoveStrings from an importable module."""
import numpy as np
import tqdm
from openpi.shared import normalize
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers)
data_config = cfg.data.create(cfg.assets_dirs, cfg.model)
dataset = dl.create_torch_dataset(data_config, cfg.model.action_horizon, cfg.model)
dataset = dl.TransformedDataset(
dataset, [*data_config.repack_transforms.inputs, *data_config.data_transforms.inputs, lazy_masks.RemoveStrings()]
)
if a.max_frames is not None and a.max_frames < len(dataset):
n, shuffle = a.max_frames // cfg.batch_size, True
else:
n, shuffle = len(dataset) // cfg.batch_size, False
loader = dl.TorchDataLoader(dataset, local_batch_size=cfg.batch_size, num_workers=cfg.num_workers,
shuffle=shuffle, num_batches=n)
stats = {k: normalize.RunningStats() for k in ("state", "actions")}
for batch in tqdm.tqdm(loader, total=n, desc="norm stats", mininterval=60):
for k in stats:
stats[k].update(np.asarray(batch[k]))
out = SYT / "assets" / a.repo_id / ASSET_ID
normalize.save(out, {k: s.get_statistics() for k, s in stats.items()})
print("wrote", out / "norm_stats.json")
def cmd_loader_speed(a):
"""Steady-state throughput: time --batches batches after the first one; use --batches >= 3x --num-workers,
otherwise the workers' prefetched batches make it look free."""
from openpi.training import data_loader as dl
cfg = build_config(a, batch_size=a.batch_size, num_workers=a.num_workers)
it = iter(dl.create_data_loader(cfg, shuffle=True, num_batches=a.batches + 1, skip_norm_stats=True))
t0 = time.time()
next(it)
t1 = time.time()
for _ in range(a.batches):
next(it)
dt = (time.time() - t1) / a.batches
print(f"loader: first batch {t1 - t0:.0f} s, then {dt:.2f} s/batch over {a.batches} batches "
f"(batch {a.batch_size}, num_workers {a.num_workers})")
def cmd_train(a):
import jax
from openpi.training import optimizer as _optimizer
# fail before the ~15 min warm-up, not after it: optax rejects a decay that ends inside the warmup
warmup = _optimizer.CosineDecaySchedule().warmup_steps
if (a.decay_steps or a.steps) <= warmup:
raise SystemExit(f"decay end {a.decay_steps or a.steps} <= warmup {warmup}; short probes need --decay-steps")
if a.batch_size % (a.grad_accum * a.fsdp):
raise SystemExit(f"batch {a.batch_size} must divide into {a.grad_accum} micro-batches x {a.fsdp} GPUs")
# Claim every visible GPU now (XLA preallocates on first use). Otherwise the GPUs stay empty during the ~15 min
# data-loader warm-up and another user's job can take them (run cover_sub_v1 died that way on GPUs 5-7).
for d in jax.devices():
jax.device_put(0.0, d).block_until_ready()
print(f"claimed {jax.device_count()} GPUs: {[d.id for d in jax.devices()]}", flush=True)
train = _load_script("train")
base = _optimizer.CosineDecaySchedule() # the official config uses the default schedule
cfg = build_config(
a,
exp_name=a.exp_name,
batch_size=a.batch_size,
num_workers=a.num_workers,
num_train_steps=a.steps,
# cosine decay ends at --decay-steps (default: the last step), so a shortened run still anneals fully
lr_schedule=_optimizer.CosineDecaySchedule(
warmup_steps=base.warmup_steps, peak_lr=base.peak_lr,
decay_steps=a.decay_steps or a.steps, decay_lr=base.decay_lr),
save_interval=a.save_interval,
keep_period=a.save_interval,
fsdp_devices=a.fsdp,
log_interval=a.log_interval,
overwrite=a.overwrite,
resume=a.resume,
)
if a.fake_data:
import dataclasses
from openpi.training import config as _config
# FakeDataset draws with jax: worker processes would try to grab the already-claimed GPUs
cfg = dataclasses.replace(cfg, data=_config.FakeDataConfig(), num_workers=0)
print("lr schedule:", cfg.lr_schedule, flush=True)
_patch_train_state(train)
if a.grad_accum > 1:
import grad_accum
train.train_step = grad_accum.make_train_step(a.grad_accum) # train.main jits the module-level train_step
print(f"gradient accumulation: {a.grad_accum} micro-batches of {a.batch_size // a.grad_accum}", flush=True)
train.main(cfg)
def cmd_restore_check(a):
"""Restore the latest checkpoint exactly as train.main does on --resume, skipping the data loader."""
import re
import jax
import numpy as np
from openpi.training import checkpoints as _checkpoints
from openpi.training import sharding
train = _load_script("train")
_patch_train_state(train)
cfg = build_config(a, exp_name=a.exp_name, fsdp_devices=a.fsdp, resume=True)
mngr, resuming = _checkpoints.initialize_checkpoint_dir(cfg.checkpoint_dir, keep_period=None, overwrite=False,
resume=True)
assert resuming, f"nothing to resume in {cfg.checkpoint_dir}"
print("checkpoints:", mngr.all_steps(), "-> restoring", mngr.latest_step(), flush=True)
mesh = sharding.make_mesh(cfg.fsdp_devices)
shape, _ = train.init_train_state(cfg, jax.random.key(cfg.seed), mesh, resume=True)
state = _checkpoints.restore_state(mngr, shape, None)
def norm(tree, pred=lambda path, x: True):
leaves = [x for path, x in jax.tree_util.tree_flatten_with_path(tree)[0]
if hasattr(x, "ndim") and pred(jax.tree_util.keystr(path), x)]
# float64 on host: exact to ~1e-12, so it is the reference for the float32 GPU value in the training log
return float(np.sqrt(sum(np.sum(np.square(np.asarray(x, dtype=np.float64))) for x in leaves)))
# same filter as train_step's logged param_norm: kernels only
kernel = lambda path, x: x.ndim > 1 and not re.search(r"(bias|scale|pos_embedding|input_embedding)", path)
print(f"step {int(state.step)} param_norm(kernels) {norm(state.params, kernel):.4f} "
f"ema_param_norm(kernels) {norm(state.ema_params, kernel):.4f} opt_state_norm {norm(state.opt_state):.4f}",
flush=True)
def main():
p = argparse.ArgumentParser()
p.add_argument("cmd", choices=["smoke", "norm-stats", "loader-speed", "train", "restore-check"])
p.add_argument("--kind", default="subtask", choices=["subtask", "full"])
p.add_argument("--repo-id", required=True)
p.add_argument("--exp-name", default=None)
p.add_argument("--batch-size", type=int, default=256)
p.add_argument("--num-workers", type=int, default=16)
p.add_argument("--max-frames", type=int, default=None)
p.add_argument("--batches", type=int, default=20)
p.add_argument("--steps", type=int, default=30_000)
p.add_argument("--decay-steps", type=int, default=None, help="end of the cosine LR decay (default: --steps)")
p.add_argument("--save-interval", type=int, default=10_000)
p.add_argument("--log-interval", type=int, default=100)
p.add_argument("--fsdp", type=int, default=8)
p.add_argument("--grad-accum", type=int, default=1, help="micro-batches per optimizer step (train)")
p.add_argument("--fake-data", action="store_true", help="train: memory/speed probe on openpi's fake data")
p.add_argument("--overwrite", action="store_true")
p.add_argument("--resume", action="store_true")
a = p.parse_args()
{"smoke": cmd_smoke, "norm-stats": cmd_norm_stats, "loader-speed": cmd_loader_speed, "train": cmd_train,
"restore-check": cmd_restore_check}[a.cmd](a)
if __name__ == "__main__":
main()
|