File size: 2,933 Bytes
6a08a28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Per-GPU memory of one training step, from XLA's compiled memory analysis (no step is executed).
Builds the train state exactly like train.main (pi05_base weights, FSDP over the visible GPUs), then compiles openpi's
train step (K=1) or grad_accum's (K>1) for a fake batch and prints argument / temp sizes next to the allocator's view.
Run from the openpi dir:
CUDA_VISIBLE_DEVICES=0,1,2,3 .venv/bin/python $SYT/code/runner/check_memory.py --batch 128 --accum 1 2 4 [--impl openpi]
"""
import argparse
import functools
import os
import sys
sys.path.insert(0, os.path.join(os.environ["SYT"], "code/runner"))
import jax # noqa: E402
from openpi.shared import array_typing as at # noqa: E402
from openpi.training import sharding # noqa: E402
import grad_accum # noqa: E402
import run # noqa: E402
p = argparse.ArgumentParser()
p.add_argument("--batch", type=int, default=128)
p.add_argument("--accum", type=int, nargs="+", default=[1])
p.add_argument("--impl", choices=["openpi", "ours"], default="ours",
help="K=1 step: openpi's train_step, or grad_accum.make_train_step(1) (same math)")
a = p.parse_args()
GiB = 2**30
n = jax.device_count()
cfg = run.build_config(argparse.Namespace(kind="subtask", repo_id="cover_sub"), batch_size=a.batch, fsdp_devices=n)
train = run._load_script("train")
mesh = sharding.make_mesh(n)
state, state_sharding = train.init_train_state(cfg, jax.random.key(0), mesh, resume=False)
jax.block_until_ready(state)
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.DATA_AXIS))
replicated = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
batch = jax.device_put((cfg.model.fake_obs(a.batch), cfg.model.fake_act(a.batch)), data_sharding)
s = jax.devices()[0].memory_stats()
print(f"after init: in use {s['bytes_in_use'] / GiB:.2f} GiB, limit {s['bytes_limit'] / GiB:.2f} GiB, "
f"largest free block {s.get('largest_free_block_bytes', 0) / GiB:.2f} GiB", flush=True)
for k in a.accum:
step = train.train_step if (k == 1 and a.impl == "openpi") else grad_accum.make_train_step(k)
# disable_typechecking: lowering rebuilds TrainState with ArgInfo leaves, which jaxtyping rejects
with sharding.set_mesh(mesh), at.disable_typechecking():
compiled = jax.jit(
functools.partial(step, cfg),
in_shardings=(replicated, state_sharding, data_sharding),
out_shardings=(state_sharding, replicated),
donate_argnums=(1,),
).lower(jax.random.key(0), state, batch).compile()
m = compiled.memory_analysis()
print(f"[{a.impl if k == 1 else 'ours'}] K={k} (micro-batch {a.batch // k}, {a.batch // k // n}/GPU): arguments {m.argument_size_in_bytes / GiB:.2f} "
f"GiB, temp {m.temp_size_in_bytes / GiB:.2f} GiB, outputs {m.output_size_in_bytes / GiB:.2f} GiB "
f"(aliased with donated args {m.alias_size_in_bytes / GiB:.2f} GiB)", flush=True)
|