File size: 3,533 Bytes
6a08a28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
"""Check grad_accum.value_and_grad_accum (K=2) against openpi's own value_and_grad, on CPU with 4 virtual devices
(FSDP 4, same sharding code path as training) and the pi0.5 architecture with openpi's "dummy" Gemma sizes and random
init: the accumulation logic does not depend on model size, and the full model's gradients do not fit next to the
training run.

reference = mean over j=0,1 of openpi's loss/grad on rows j::2 with rng fold_in(rng, j)  (what K=2 must equal)
Pass: relative gradient error far below the "half batch" gap (grad on rows 0::2 alone vs the reference), which shows
how large a real mistake (wrong rows, missing 1/K, one micro-batch dropped) would be.
Run from the openpi dir:  .venv/bin/python $SYT/code/runner/check_grad_accum.py   (sets JAX to CPU itself)
"""

import argparse
import dataclasses
import os
import sys

os.environ["JAX_PLATFORMS"] = "cpu"
os.environ["XLA_FLAGS"] = os.environ.get("XLA_FLAGS", "") + " --xla_force_host_platform_device_count=4"
sys.path.insert(0, os.path.join(os.environ["SYT"], "code/runner"))

import flax.nnx as nnx  # noqa: E402
import jax  # noqa: E402
import jax.numpy as jnp  # noqa: E402
import optax  # noqa: E402
from openpi.training import sharding  # noqa: E402
from openpi.training import weight_loaders  # noqa: E402

import grad_accum as ga  # noqa: E402
import run  # noqa: E402

B, K = 8, 2
cfg = run.build_config(argparse.Namespace(kind="subtask", repo_id="cover_sub"), batch_size=B, fsdp_devices=4)
cfg = dataclasses.replace(cfg, weight_loader=weight_loaders.NoOpWeightLoader(),
                          model=dataclasses.replace(cfg.model, paligemma_variant="dummy", action_expert_variant="dummy"))
train = run._load_script("train")
mesh = sharding.make_mesh(4)
state, _ = train.init_train_state(cfg, jax.random.key(0), mesh, resume=False)

# non-trivial inputs: random images / state / actions in the model's input ranges
obs, act = cfg.model.fake_obs(B), cfg.model.fake_act(B)
keys = iter(jax.random.split(jax.random.key(1), 16))
obs = dataclasses.replace(
    obs, images={k: jax.random.uniform(next(keys), v.shape, minval=-1, maxval=1) for k, v in obs.images.items()},
    state=jax.random.normal(next(keys), obs.state.shape))
act = jax.random.normal(next(keys), act.shape)
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.DATA_AXIS))
obs, act = jax.device_put((obs, act), data_sharding)
rng = jax.random.key(2)


def vg(state, rng, obs, act, k):
    model = nnx.merge(state.model_def, state.params)
    model.train()
    return ga.value_and_grad_accum(cfg, model, rng, obs, act, k)


def compare(state, rng, obs, act):
    """Everything in one jit, returning scalars only."""
    loss_k, g_k = vg(state, rng, obs, act, K)
    parts = [vg(state, jax.random.fold_in(rng, j), *jax.tree.map(lambda x: x[j::K], (obs, act)), 1) for j in range(K)]
    loss_ref = sum(p[0] for p in parts) / K
    g_ref = jax.tree.map(lambda *g: sum(g) / K, *[p[1] for p in parts])
    rel = lambda a, b: optax.global_norm(jax.tree.map(jnp.subtract, a, b)) / optax.global_norm(b)  # noqa: E731
    return loss_k, loss_ref, rel(g_k, g_ref), rel(parts[0][1], g_ref)


with sharding.set_mesh(mesh):
    loss_k, loss_ref, err, gap = map(float, jax.jit(compare)(state, rng, obs, act))
print(f"loss  K={K}: {loss_k:.6f}   reference: {loss_ref:.6f}")
print(f"grad  |g_K - g_ref| / |g_ref| = {err:.2e}   (half-batch gap for scale: {gap:.2e})")
print("GRAD_ACCUM_OK" if err < 0.01 * gap and abs(loss_k - loss_ref) < 1e-3 else "GRAD_ACCUM_MISMATCH")