Download code/runner/check_memory.py from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 2.93 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/check_memory.py
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/runner/check_memory.py
-
curl -L -o check_memory.py https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/runner/check_memory.py
2.93 kB
| """Per-GPU memory of one training step, from XLA's compiled memory analysis (no step is executed). | |
| Builds the train state exactly like train.main (pi05_base weights, FSDP over the visible GPUs), then compiles openpi's | |
| train step (K=1) or grad_accum's (K>1) for a fake batch and prints argument / temp sizes next to the allocator's view. | |
| Run from the openpi dir: | |
| CUDA_VISIBLE_DEVICES=0,1,2,3 .venv/bin/python $SYT/code/runner/check_memory.py --batch 128 --accum 1 2 4 [--impl openpi] | |
| """ | |
| import argparse | |
| import functools | |
| import os | |
| import sys | |
| sys.path.insert(0, os.path.join(os.environ["SYT"], "code/runner")) | |
| import jax # noqa: E402 | |
| from openpi.shared import array_typing as at # noqa: E402 | |
| from openpi.training import sharding # noqa: E402 | |
| import grad_accum # noqa: E402 | |
| import run # noqa: E402 | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--batch", type=int, default=128) | |
| p.add_argument("--accum", type=int, nargs="+", default=[1]) | |
| p.add_argument("--impl", choices=["openpi", "ours"], default="ours", | |
| help="K=1 step: openpi's train_step, or grad_accum.make_train_step(1) (same math)") | |
| a = p.parse_args() | |
| GiB = 2**30 | |
| n = jax.device_count() | |
| cfg = run.build_config(argparse.Namespace(kind="subtask", repo_id="cover_sub"), batch_size=a.batch, fsdp_devices=n) | |
| train = run._load_script("train") | |
| mesh = sharding.make_mesh(n) | |
| state, state_sharding = train.init_train_state(cfg, jax.random.key(0), mesh, resume=False) | |
| jax.block_until_ready(state) | |
| data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.DATA_AXIS)) | |
| replicated = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec()) | |
| batch = jax.device_put((cfg.model.fake_obs(a.batch), cfg.model.fake_act(a.batch)), data_sharding) | |
| s = jax.devices()[0].memory_stats() | |
| print(f"after init: in use {s['bytes_in_use'] / GiB:.2f} GiB, limit {s['bytes_limit'] / GiB:.2f} GiB, " | |
| f"largest free block {s.get('largest_free_block_bytes', 0) / GiB:.2f} GiB", flush=True) | |
| for k in a.accum: | |
| step = train.train_step if (k == 1 and a.impl == "openpi") else grad_accum.make_train_step(k) | |
| # disable_typechecking: lowering rebuilds TrainState with ArgInfo leaves, which jaxtyping rejects | |
| with sharding.set_mesh(mesh), at.disable_typechecking(): | |
| compiled = jax.jit( | |
| functools.partial(step, cfg), | |
| in_shardings=(replicated, state_sharding, data_sharding), | |
| out_shardings=(state_sharding, replicated), | |
| donate_argnums=(1,), | |
| ).lower(jax.random.key(0), state, batch).compile() | |
| m = compiled.memory_analysis() | |
| print(f"[{a.impl if k == 1 else 'ours'}] K={k} (micro-batch {a.batch // k}, {a.batch // k // n}/GPU): arguments {m.argument_size_in_bytes / GiB:.2f} " | |
| f"GiB, temp {m.temp_size_in_bytes / GiB:.2f} GiB, outputs {m.output_size_in_bytes / GiB:.2f} GiB " | |
| f"(aliased with donated args {m.alias_size_in_bytes / GiB:.2f} GiB)", flush=True) | |