Download code/runner/grad_accum.py from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 4.41 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/grad_accum.py
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/runner/grad_accum.py
-
curl -L -o grad_accum.py https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/grad_accum.py
4.41 kB
| """Gradient accumulation for openpi's train_step (installed into scripts/train.py by `run.py train --grad-accum K`). | |
| Why: the A100s here have 40 GB. A 4-GPU FSDP run keeps ~12.5 GiB of state per GPU (params, EMA params, AdamW | |
| moments), and one batch-128 step (32 samples per GPU) needs ~28 GiB more, so it does not fit. With K micro-batches | |
| the step computes the gradient of B/K samples K times in a lax.scan (only one micro-batch's activations alive at a | |
| time), averages the K gradients and applies ONE optimizer update, i.e. the batch-B update. The only difference from | |
| a true batch-B step: the flow-matching noise/time draws use one rng per micro-batch (fold_in(step_rng, j)); K=1 is | |
| exactly openpi's step. Everything after the gradient (AdamW, EMA, logged info) is copied unchanged from openpi. | |
| """ | |
| import dataclasses | |
| import flax.nnx as nnx | |
| import jax | |
| import jax.numpy as jnp | |
| import optax | |
| from openpi.shared import nnx_utils | |
| def split_microbatches(tree, k): | |
| """(B, ...) -> (K, B/K, ...). Micro-batch j = rows j, j+K, j+2K, ...: the batch is sharded over GPUs in contiguous | |
| row blocks, so every micro-batch takes the same number of rows from every GPU (no cross-GPU shuffle).""" | |
| def split(x): | |
| assert x.shape[0] % k == 0, (x.shape, k) | |
| return jnp.swapaxes(x.reshape(x.shape[0] // k, k, *x.shape[1:]), 0, 1) | |
| return jax.tree.map(split, tree) | |
| def value_and_grad_accum(config, model, rng, observation, actions, k): | |
| """Mean loss and mean gradient over K micro-batches (K=1: openpi's single value_and_grad call, same rng).""" | |
| def loss_fn(model, rng, observation, actions): | |
| return jnp.mean(model.compute_loss(rng, observation, actions, train=True)) | |
| diff_state = nnx.DiffState(0, config.trainable_filter) | |
| if k == 1: | |
| return nnx.value_and_grad(loss_fn, argnums=diff_state)(model, rng, observation, actions) | |
| graphdef, model_state = nnx.split(model) | |
| def micro(j, obs, act): | |
| m = nnx.merge(graphdef, model_state) # fresh module per trace level (nnx forbids reusing the outer one) | |
| return nnx.value_and_grad(loss_fn, argnums=diff_state)(m, jax.random.fold_in(rng, j), obs, act) | |
| obs_mb, act_mb = split_microbatches((observation, actions), k) | |
| loss0, grads0 = micro(0, *jax.tree.map(lambda x: x[0], (obs_mb, act_mb))) | |
| def body(carry, xs): | |
| loss_sum, grad_sum = carry | |
| j, obs, act = xs | |
| loss, grads = micro(j, obs, act) | |
| return (loss_sum + loss, jax.tree.map(jnp.add, grad_sum, grads)), None | |
| rest = jax.tree.map(lambda x: x[1:], (obs_mb, act_mb)) | |
| (loss_sum, grad_sum), _ = jax.lax.scan(body, (loss0, grads0), (jnp.arange(1, k), *rest)) | |
| return loss_sum / k, jax.tree.map(lambda g: g / k, grad_sum) | |
| def make_train_step(k): | |
| """openpi scripts/train.py::train_step with the value_and_grad call replaced by value_and_grad_accum.""" | |
| def train_step(config, rng, state, batch): | |
| model = nnx.merge(state.model_def, state.params) | |
| model.train() | |
| train_rng = jax.random.fold_in(rng, state.step) | |
| observation, actions = batch | |
| loss, grads = value_and_grad_accum(config, model, train_rng, observation, actions, k) | |
| # ---- unchanged from openpi ---- | |
| params = state.params.filter(config.trainable_filter) | |
| updates, new_opt_state = state.tx.update(grads, state.opt_state, params) | |
| new_params = optax.apply_updates(params, updates) | |
| nnx.update(model, new_params) | |
| new_params = nnx.state(model) | |
| new_state = dataclasses.replace(state, step=state.step + 1, params=new_params, opt_state=new_opt_state) | |
| if state.ema_decay is not None: | |
| new_state = dataclasses.replace( | |
| new_state, | |
| ema_params=jax.tree.map( | |
| lambda old, new: state.ema_decay * old + (1 - state.ema_decay) * new, state.ema_params, new_params | |
| ), | |
| ) | |
| kernel_params = nnx.state( | |
| model, | |
| nnx.All( | |
| nnx.Param, | |
| nnx.Not(nnx_utils.PathRegex(".*/(bias|scale|pos_embedding|input_embedding)")), | |
| lambda _, x: x.value.ndim > 1, | |
| ), | |
| ) | |
| info = { | |
| "loss": loss, | |
| "grad_norm": optax.global_norm(grads), | |
| "param_norm": optax.global_norm(kernel_params), | |
| } | |
| return new_state, info | |
| return train_step | |