"""Gradient accumulation for openpi's train_step (installed into scripts/train.py by `run.py train --grad-accum K`). Why: the A100s here have 40 GB. A 4-GPU FSDP run keeps ~12.5 GiB of state per GPU (params, EMA params, AdamW moments), and one batch-128 step (32 samples per GPU) needs ~28 GiB more, so it does not fit. With K micro-batches the step computes the gradient of B/K samples K times in a lax.scan (only one micro-batch's activations alive at a time), averages the K gradients and applies ONE optimizer update, i.e. the batch-B update. The only difference from a true batch-B step: the flow-matching noise/time draws use one rng per micro-batch (fold_in(step_rng, j)); K=1 is exactly openpi's step. Everything after the gradient (AdamW, EMA, logged info) is copied unchanged from openpi. """ import dataclasses import flax.nnx as nnx import jax import jax.numpy as jnp import optax from openpi.shared import nnx_utils def split_microbatches(tree, k): """(B, ...) -> (K, B/K, ...). Micro-batch j = rows j, j+K, j+2K, ...: the batch is sharded over GPUs in contiguous row blocks, so every micro-batch takes the same number of rows from every GPU (no cross-GPU shuffle).""" def split(x): assert x.shape[0] % k == 0, (x.shape, k) return jnp.swapaxes(x.reshape(x.shape[0] // k, k, *x.shape[1:]), 0, 1) return jax.tree.map(split, tree) def value_and_grad_accum(config, model, rng, observation, actions, k): """Mean loss and mean gradient over K micro-batches (K=1: openpi's single value_and_grad call, same rng).""" def loss_fn(model, rng, observation, actions): return jnp.mean(model.compute_loss(rng, observation, actions, train=True)) diff_state = nnx.DiffState(0, config.trainable_filter) if k == 1: return nnx.value_and_grad(loss_fn, argnums=diff_state)(model, rng, observation, actions) graphdef, model_state = nnx.split(model) def micro(j, obs, act): m = nnx.merge(graphdef, model_state) # fresh module per trace level (nnx forbids reusing the outer one) return nnx.value_and_grad(loss_fn, argnums=diff_state)(m, jax.random.fold_in(rng, j), obs, act) obs_mb, act_mb = split_microbatches((observation, actions), k) loss0, grads0 = micro(0, *jax.tree.map(lambda x: x[0], (obs_mb, act_mb))) def body(carry, xs): loss_sum, grad_sum = carry j, obs, act = xs loss, grads = micro(j, obs, act) return (loss_sum + loss, jax.tree.map(jnp.add, grad_sum, grads)), None rest = jax.tree.map(lambda x: x[1:], (obs_mb, act_mb)) (loss_sum, grad_sum), _ = jax.lax.scan(body, (loss0, grads0), (jnp.arange(1, k), *rest)) return loss_sum / k, jax.tree.map(lambda g: g / k, grad_sum) def make_train_step(k): """openpi scripts/train.py::train_step with the value_and_grad call replaced by value_and_grad_accum.""" def train_step(config, rng, state, batch): model = nnx.merge(state.model_def, state.params) model.train() train_rng = jax.random.fold_in(rng, state.step) observation, actions = batch loss, grads = value_and_grad_accum(config, model, train_rng, observation, actions, k) # ---- unchanged from openpi ---- params = state.params.filter(config.trainable_filter) updates, new_opt_state = state.tx.update(grads, state.opt_state, params) new_params = optax.apply_updates(params, updates) nnx.update(model, new_params) new_params = nnx.state(model) new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state) if state.ema_decay is not None: new_state = dataclasses.replace( new_state, ema_params=jax.tree.map( lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params ), ) kernel_params = nnx.state( model, nnx.All( nnx.Param, nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")), lambda _, x: x.value.ndim > 1, ), ) info = { "loss": loss, "grad_norm": optax.global_norm(grads), "param_norm": optax.global_norm(kernel_params), } return new_state, info return train_step