dexhand-train-kit / code /runner /grad_accum.py
Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
4.41 kB
"""Gradient accumulation for openpi's train_step (installed into scripts/train.py by `run.py train --grad-accum K`).
Why: the A100s here have 40 GB. A 4-GPU FSDP run keeps ~12.5 GiB of state per GPU (params, EMA params, AdamW
moments), and one batch-128 step (32 samples per GPU) needs ~28 GiB more, so it does not fit. With K micro-batches
the step computes the gradient of B/K samples K times in a lax.scan (only one micro-batch's activations alive at a
time), averages the K gradients and applies ONE optimizer update, i.e. the batch-B update. The only difference from
a true batch-B step: the flow-matching noise/time draws use one rng per micro-batch (fold_in(step_rng, j)); K=1 is
exactly openpi's step. Everything after the gradient (AdamW, EMA, logged info) is copied unchanged from openpi.
"""
import dataclasses
import flax.nnx as nnx
import jax
import jax.numpy as jnp
import optax
from openpi.shared import nnx_utils
def split_microbatches(tree, k):
"""(B, ...) -> (K, B/K, ...). Micro-batch j = rows j, j+K, j+2K, ...: the batch is sharded over GPUs in contiguous
row blocks, so every micro-batch takes the same number of rows from every GPU (no cross-GPU shuffle)."""
def split(x):
assert x.shape[0] % k == 0, (x.shape, k)
return jnp.swapaxes(x.reshape(x.shape[0] // k, k, *x.shape[1:]), 0, 1)
return jax.tree.map(split, tree)
def value_and_grad_accum(config, model, rng, observation, actions, k):
"""Mean loss and mean gradient over K micro-batches (K=1: openpi's single value_and_grad call, same rng)."""
def loss_fn(model, rng, observation, actions):
return jnp.mean(model.compute_loss(rng, observation, actions, train=True))
diff_state = nnx.DiffState(0, config.trainable_filter)
if k == 1:
return nnx.value_and_grad(loss_fn, argnums=diff_state)(model, rng, observation, actions)
graphdef, model_state = nnx.split(model)
def micro(j, obs, act):
m = nnx.merge(graphdef, model_state) # fresh module per trace level (nnx forbids reusing the outer one)
return nnx.value_and_grad(loss_fn, argnums=diff_state)(m, jax.random.fold_in(rng, j), obs, act)
obs_mb, act_mb = split_microbatches((observation, actions), k)
loss0, grads0 = micro(0, *jax.tree.map(lambda x: x[0], (obs_mb, act_mb)))
def body(carry, xs):
loss_sum, grad_sum = carry
j, obs, act = xs
loss, grads = micro(j, obs, act)
return (loss_sum + loss, jax.tree.map(jnp.add, grad_sum, grads)), None
rest = jax.tree.map(lambda x: x[1:], (obs_mb, act_mb))
(loss_sum, grad_sum), _ = jax.lax.scan(body, (loss0, grads0), (jnp.arange(1, k), *rest))
return loss_sum / k, jax.tree.map(lambda g: g / k, grad_sum)
def make_train_step(k):
"""openpi scripts/train.py::train_step with the value_and_grad call replaced by value_and_grad_accum."""
def train_step(config, rng, state, batch):
model = nnx.merge(state.model_def, state.params)
model.train()
train_rng = jax.random.fold_in(rng, state.step)
observation, actions = batch
loss, grads = value_and_grad_accum(config, model, train_rng, observation, actions, k)
# ---- unchanged from openpi ----
params = state.params.filter(config.trainable_filter)
updates, new_opt_state = state.tx.update(grads, state.opt_state, params)
new_params = optax.apply_updates(params, updates)
nnx.update(model, new_params)
new_params = nnx.state(model)
new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state)
if state.ema_decay is not None:
new_state = dataclasses.replace(
new_state,
ema_params=jax.tree.map(
lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params
),
)
kernel_params = nnx.state(
model,
nnx.All(
nnx.Param,
nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")),
lambda _, x: x.value.ndim > 1,
),
)
info = {
"loss": loss,
"grad_norm": optax.global_norm(grads),
"param_norm": optax.global_norm(kernel_params),
}
return new_state, info
return train_step