devin-lai's picture
Release v0.4.0: selective MLX model downloads
3da88e3 verified
Raw History Blame Contribute Delete
3.93 kB
"""Test native reduction matching for full-width RMSNorm + AdaLN fusion."""
import argparse
import copy
import json
import types
from pathlib import Path
import mlx.core as mx
from experiments.r3.common import paired
from experiments.r3.fusions import norm_native_reduce
from h3_mlx.benchmark import error
from h3_mlx.config import SIGMAS
from h3_mlx.dit import DiT
from h3_mlx.layout import image_positions, rope_cache
from h3_mlx.ops import gated_add, modulate
from h3_mlx.runtime import configure_memory, environment
def main(a):
configure_memory(wired_gb=14)
data = mx.load(a.trace)
model = DiT(a.model, kernel="metal-exact", tune_matmul=True)
model.prepare_modulation(SIGMAS)
text = model.refine(data["conditioning"])
x = mx.concatenate(
(
text,
model._linear("video_patch_proj")(data["initial_rows"]).astype(mx.bfloat16),
),
axis=0,
)
rope = rope_cache(
image_positions(len(text), 128, 128), model._constant("rope.inv_freq")
)
indices = mx.concatenate((data["text_tags"], mx.zeros(4096, mx.int32)))
params = tuple(model.modulation["block.0"][0, :, j] for j in range(6))
block = model._block("blocks.0")
report = {"environment": environment(), "trials": []}
ref = modulate(x, block.n1, params[0], params[1], indices, block.config.norm_eps)
mx.eval(ref)
for threads in (256, 512, 1024):
def fn(threads=threads):
return norm_native_reduce(
x,
block.n1,
params[0],
params[1],
indices,
block.config.norm_eps,
threads,
)
def native():
return modulate(
x, block.n1, params[0], params[1], indices, block.config.norm_eps
)
row = {
"group": "norm",
"threads": threads,
**error(ref, fn()),
**paired(native, fn, a.repeats),
}
report["trials"].append(row)
print(json.dumps(row), flush=True)
native = mx.compile(
lambda x, params, indices, rope: block(
x,
params=params,
indices=indices,
rope=rope,
mlp_chunk=0,
kernel="metal-exact",
)
)
reference = native(x, params, indices, rope)
mx.eval(reference)
def forward(self, xx, *, params, indices, rope, **kwargs):
sa, ca, ga, sm, cm, gm = params
h = norm_native_reduce(xx, self.n1, sa, ca, indices, self.config.norm_eps)
xx = gated_add(xx, self.attn(h, rope, 0, "metal-exact"), ga, indices)
h = norm_native_reduce(xx, self.n2, sm, cm, indices, self.config.norm_eps)
return gated_add(xx, self.mlp(h, 0), gm, indices)
# __call__ is looked up on the class, so use an explicit bound method.
fused = copy.copy(block)
fused.forward = types.MethodType(forward, fused)
candidate = mx.compile(
lambda x, params, indices, rope: fused.forward(
x, params=params, indices=indices, rope=rope
)
)
row = {
"group": "block",
**error(reference, candidate(x, params, indices, rope)),
**paired(
lambda: native(x, params, indices, rope),
lambda: candidate(x, params, indices, rope),
a.repeats,
),
}
report["trials"].append(row)
print(json.dumps(row), flush=True)
Path(a.output).write_text(json.dumps(report, indent=2) + "\n")
model.clear()
if __name__ == "__main__":
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--model", required=True)
p.add_argument(
"--trace", default="outputs/suite/product/native-3.trace.safetensors"
)
p.add_argument("--repeats", type=int, default=10)
p.add_argument("--output", default="reports/optimization-r3/norm.json")
main(p.parse_args())