File size: 2,933 Bytes
6a08a28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""Per-GPU memory of one training step, from XLA's compiled memory analysis (no step is executed).

Builds the train state exactly like train.main (pi05_base weights, FSDP over the visible GPUs), then compiles openpi's
train step (K=1) or grad_accum's (K>1) for a fake batch and prints argument / temp sizes next to the allocator's view.
Run from the openpi dir:
  CUDA_VISIBLE_DEVICES=0,1,2,3 .venv/bin/python $SYT/code/runner/check_memory.py --batch 128 --accum 1 2 4 [--impl openpi]
"""

import argparse
import functools
import os
import sys

sys.path.insert(0, os.path.join(os.environ["SYT"], "code/runner"))

import jax  # noqa: E402
from openpi.shared import array_typing as at  # noqa: E402
from openpi.training import sharding  # noqa: E402

import grad_accum  # noqa: E402
import run  # noqa: E402

p = argparse.ArgumentParser()
p.add_argument("--batch", type=int, default=128)
p.add_argument("--accum", type=int, nargs="+", default=[1])
p.add_argument("--impl", choices=["openpi", "ours"], default="ours",
               help="K=1 step: openpi's train_step, or grad_accum.make_train_step(1) (same math)")
a = p.parse_args()
GiB = 2**30
n = jax.device_count()

cfg = run.build_config(argparse.Namespace(kind="subtask", repo_id="cover_sub"), batch_size=a.batch, fsdp_devices=n)
train = run._load_script("train")
mesh = sharding.make_mesh(n)
state, state_sharding = train.init_train_state(cfg, jax.random.key(0), mesh, resume=False)
jax.block_until_ready(state)
data_sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(sharding.DATA_AXIS))
replicated = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec())
batch = jax.device_put((cfg.model.fake_obs(a.batch), cfg.model.fake_act(a.batch)), data_sharding)
s = jax.devices()[0].memory_stats()
print(f"after init: in use {s['bytes_in_use'] / GiB:.2f} GiB, limit {s['bytes_limit'] / GiB:.2f} GiB, "
      f"largest free block {s.get('largest_free_block_bytes', 0) / GiB:.2f} GiB", flush=True)

for k in a.accum:
    step = train.train_step if (k == 1 and a.impl == "openpi") else grad_accum.make_train_step(k)
    # disable_typechecking: lowering rebuilds TrainState with ArgInfo leaves, which jaxtyping rejects
    with sharding.set_mesh(mesh), at.disable_typechecking():
        compiled = jax.jit(
            functools.partial(step, cfg),
            in_shardings=(replicated, state_sharding, data_sharding),
            out_shardings=(state_sharding, replicated),
            donate_argnums=(1,),
        ).lower(jax.random.key(0), state, batch).compile()
    m = compiled.memory_analysis()
    print(f"[{a.impl if k == 1 else 'ours'}] K={k} (micro-batch {a.batch // k}, {a.batch // k // n}/GPU): arguments {m.argument_size_in_bytes / GiB:.2f} "
          f"GiB, temp {m.temp_size_in_bytes / GiB:.2f} GiB, outputs {m.output_size_in_bytes / GiB:.2f} GiB "
          f"(aliased with donated args {m.alias_size_in_bytes / GiB:.2f} GiB)", flush=True)