jburtoft's picture
v1.0.0: NKI kernels for NewBERT (gte) encoders on trn2
25e8424
Raw History Blame Contribute Delete
13.4 kB
"""Fused NewBERT gated MLP + residual + post-LayerNorm (replaces NewGatedMLP + mlp_ln).
h = x @ Wup^T, gt = x @ Wgate^T (up_gate_proj, no bias; split [up | gate])
m = (gelu_tanh(gt) * h) @ Wd^T + bd (down_proj)
y = LayerNorm(x + m) * gamma + beta (mlp_ln, residual = MLP input)
Shapes: x [R, K] bf16 (K = hidden), wupT / wgT [K, I] bf16 (I = intermediate), wdT [I, K] bf16,
bd / gamma / beta pre-broadcast to [128, K] (fp32 or bf16). Returns y [R, K] bf16.
Two kernels with the same inputs; pick with mlp_weights_fit(K, I):
gated_mlp_ln_kernel weights resident in SBUF (~108 KiB per partition at K=768, I=3072)
gated_mlp_ln_stream_kernel weights streamed in I chunks (K=1024, I=4096 needs 192 KiB resident: too big)
Resident layout: K-tiles x I for up/gate, I-tiles x K for down. Rows are processed in blocks of up to
512 (the matmul moving free size). Per block:
* x rows are loaded once (row layout, kept for the residual) and transposed to x^T (K on
partitions) by DMA transpose.
* up/gate for each 128-wide I tile land in PSUM [128 (I), rows]; gelu_tanh on Scalar, multiply
on Vector -> g [I tile, rows] bf16, already in the layout the down-projection needs.
* down: per 128-row subtile, PSUM [128 rows, K] accumulated over the I tiles.
* epilogue: s = psum + x + bd (fp32), bn_stats/bn_aggr, y = (s - mean) * gamma * rstd + beta.
R must be a multiple of 128 * LNC; rows are split across LNC programs. Inference only.
"""
import nki
import nki.isa as nisa
import nki.language as nl
P = 128
FMAX = 512
def _cdiv(a, b):
return (a + b - 1) // b
@nki.jit
def gated_mlp_ln_kernel(x_hbm, wupT_hbm, wgT_hbm, wdT_hbm, bd_hbm, g_hbm, b_hbm, eps=1e-12):
R, K = x_hbm.shape
K1, I = wupT_hbm.shape
assert K1 == K
assert K % P == 0
assert I % P == 0
n_prog = nl.num_programs(0)
pid = nl.program_id(0)
assert R % (P * n_prog) == 0
rows_per = R // n_prog
r_base = pid * rows_per
n_k = K // P
n_i = I // P
n_ko = _cdiv(K, FMAX)
n_blk = _cdiv(rows_per, FMAX)
fmax = nl.tile_size.bn_stats_fmax
n_ch = _cdiv(K, fmax)
csz = K // n_ch
dt = x_hbm.dtype
y_hbm = nl.ndarray((R, K), dtype=dt, buffer=nl.shared_hbm)
# ---- resident weights ----
wu = []
wg = []
for kt in range(n_k):
tu = nl.ndarray((P, I), dtype=wupT_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=tu, src=wupT_hbm[kt * P:(kt + 1) * P, 0:I])
wu.append(tu)
tg = nl.ndarray((P, I), dtype=wgT_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=tg, src=wgT_hbm[kt * P:(kt + 1) * P, 0:I])
wg.append(tg)
wd = []
for it in range(n_i):
td = nl.ndarray((P, K), dtype=wdT_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=td, src=wdT_hbm[it * P:(it + 1) * P, 0:K])
wd.append(td)
bd_sb = nl.ndarray((P, K), dtype=bd_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=bd_sb, src=bd_hbm[0:P, 0:K])
ga_sb = nl.ndarray((P, K), dtype=g_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=ga_sb, src=g_hbm[0:P, 0:K])
be_sb = nl.ndarray((P, K), dtype=b_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=be_sb, src=b_hbm[0:P, 0:K])
eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.memset(dst=eps_sb, value=eps)
for blk in range(n_blk):
q0 = blk * FMAX
rw = min(FMAX, rows_per - q0)
n_rs = rw // P
rb = r_base + q0
# ---- x rows (kept for the residual) and x^T ----
xr = []
xt = []
for kt in range(n_k):
xt.append(nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf))
for rs in range(n_rs):
t = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.dma_copy(dst=t, src=x_hbm[nl.ds(rb + rs * P, P), 0:K])
xr.append(t)
for kt in range(n_k):
nisa.dma_transpose(dst=xt[kt][0:P, rs * P:(rs + 1) * P], src=t[0:P, kt * P:(kt + 1) * P])
# ---- up / gate -> g [I tile, rows] ----
gl = []
for it in range(n_i):
i0 = it * P
pu = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=pu[0:P, 0:rw], stationary=wu[kt][0:P, i0:i0 + P], moving=xt[kt][0:P, 0:rw])
pg = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=pg[0:P, 0:rw], stationary=wg[kt][0:P, i0:i0 + P], moving=xt[kt][0:P, 0:rw])
ge = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=ge[0:P, 0:rw], op=nl.gelu_apprx_tanh, data=pg[0:P, 0:rw])
g = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf)
nisa.tensor_tensor(dst=g[0:P, 0:rw], data1=pu[0:P, 0:rw], data2=ge[0:P, 0:rw], op=nl.multiply)
gl.append(g)
# ---- down + residual + LN, per 128-row subtile ----
for rs in range(n_rs):
s_sb = nl.ndarray((P, K), dtype=nl.float32, buffer=nl.sbuf)
xb = nl.ndarray((P, K), dtype=nl.float32, buffer=nl.sbuf)
nisa.tensor_tensor(dst=xb, data1=xr[rs], data2=bd_sb, op=nl.add, engine=nisa.engine.gpsimd)
for ko in range(n_ko):
k0 = ko * FMAX
nk = min(FMAX, K - k0)
pd = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for it in nl.affine_range(n_i):
nisa.nc_matmul(dst=pd[0:P, 0:nk], stationary=gl[it][0:P, rs * P:(rs + 1) * P],
moving=wd[it][0:P, k0:k0 + nk])
nisa.tensor_tensor(dst=s_sb[0:P, k0:k0 + nk], data1=pd[0:P, 0:nk], data2=xb[0:P, k0:k0 + nk],
op=nl.add)
stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf)
for c in range(n_ch):
nisa.bn_stats(dst=stats[0:P, c * 6:(c + 1) * 6], data=s_sb[0:P, c * csz:(c + 1) * csz])
mv = nl.ndarray((P, 2), dtype=nl.float32, buffer=nl.sbuf)
nisa.bn_aggr(dst=mv, data=stats)
rstd = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.activation(dst=rstd, op=nl.rsqrt, data=mv[0:P, 1:2], bias=eps_sb)
u_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mv[0:P, 0:1],
op1=nl.multiply, operand1=ga_sb)
v_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd)
nisa.dma_compute(dst=y_hbm[nl.ds(rb + rs * P, P), 0:K], srcs=[v_sb, be_sb], reduce_op=nl.add)
return y_hbm
# ---------------------------------------------------------------- weight-streaming variant (large K * I)
RESIDENT_BUDGET = 128 * 1024 # bytes per partition the resident kernel may spend on weights
STREAM_IC = 1024 # intermediate columns per streamed weight chunk
STREAM_SB = 1024 # rows per super-block (rows that share one pass over the weights)
def mlp_weights_fit(K, I, itemsize=2):
"""True if gated_mlp_ln_kernel can keep all three weight matrices resident in SBUF."""
return (2 * (K // P) * I + (I // P) * K) * itemsize <= RESIDENT_BUDGET
@nki.jit
def gated_mlp_ln_stream_kernel(x_hbm, wupT_hbm, wgT_hbm, wdT_hbm, bd_hbm, g_hbm, b_hbm, eps=1e-12):
"""Same math and inputs as gated_mlp_ln_kernel, for weights too large to keep resident
(e.g. K=1024, I=4096). Weights are streamed in chunks of STREAM_IC intermediate columns, once per
super-block of up to STREAM_SB rows; the down-projection partials accumulate in fp32 SBUF, seeded
with x + bd, so the residual costs nothing extra."""
R, K = x_hbm.shape
K1, I = wupT_hbm.shape
assert K1 == K
assert K % P == 0
assert I % P == 0
n_prog = nl.num_programs(0)
pid = nl.program_id(0)
assert R % (P * n_prog) == 0
rows_per = R // n_prog
r_base = pid * rows_per
IC = STREAM_IC
if I % IC != 0:
IC = P
n_c = I // IC
n_ic = IC // P
n_k = K // P
n_ko = _cdiv(K, FMAX)
n_sb = _cdiv(rows_per, STREAM_SB)
fmax = nl.tile_size.bn_stats_fmax
n_ch = _cdiv(K, fmax)
csz = K // n_ch
dt = x_hbm.dtype
y_hbm = nl.ndarray((R, K), dtype=dt, buffer=nl.shared_hbm)
bd_sb = nl.ndarray((P, K), dtype=bd_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=bd_sb, src=bd_hbm[0:P, 0:K])
ga_sb = nl.ndarray((P, K), dtype=g_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=ga_sb, src=g_hbm[0:P, 0:K])
be_sb = nl.ndarray((P, K), dtype=b_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=be_sb, src=b_hbm[0:P, 0:K])
eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.memset(dst=eps_sb, value=eps)
for sb in range(n_sb):
s0 = sb * STREAM_SB
sw = min(STREAM_SB, rows_per - s0)
n_sub = sw // P
rb = r_base + s0
# ---- x^T for the super-block; accumulator seeded with x + bd ----
xT = nl.ndarray((P, n_k, STREAM_SB), dtype=dt, buffer=nl.sbuf)
acc = nl.ndarray((P, n_sub, K), dtype=nl.float32, buffer=nl.sbuf)
for rs in range(n_sub):
t = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.dma_copy(dst=t, src=x_hbm[nl.ds(rb + rs * P, P), 0:K])
nisa.tensor_tensor(dst=acc[:, rs, :], data1=t, data2=bd_sb, op=nl.add, engine=nisa.engine.gpsimd)
for kt in range(n_k):
nisa.dma_transpose(dst=xT[:, kt, rs * P:(rs + 1) * P], src=t[:, kt * P:(kt + 1) * P])
for c in range(n_c):
c0 = c * IC
wu = nl.ndarray((P, n_k, IC), dtype=wupT_hbm.dtype, buffer=nl.sbuf)
wg = nl.ndarray((P, n_k, IC), dtype=wgT_hbm.dtype, buffer=nl.sbuf)
wd = nl.ndarray((P, n_ic, K), dtype=wdT_hbm.dtype, buffer=nl.sbuf)
for kt in range(n_k):
nisa.dma_copy(dst=wu[:, kt, :], src=wupT_hbm[kt * P:(kt + 1) * P, c0:c0 + IC])
nisa.dma_copy(dst=wg[:, kt, :], src=wgT_hbm[kt * P:(kt + 1) * P, c0:c0 + IC])
for it in range(n_ic):
nisa.dma_copy(dst=wd[:, it, :], src=wdT_hbm[c0 + it * P:c0 + (it + 1) * P, 0:K])
for blk in range(_cdiv(sw, FMAX)):
q0 = blk * FMAX
rw = min(FMAX, sw - q0)
# up / gate for this chunk -> g [I tile, rows]
gl = nl.ndarray((P, n_ic, FMAX), dtype=dt, buffer=nl.sbuf)
for it in range(n_ic):
pu = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=pu[:, 0:rw], stationary=wu[:, kt, it * P:(it + 1) * P],
moving=xT[:, kt, q0:q0 + rw])
pg = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=pg[:, 0:rw], stationary=wg[:, kt, it * P:(it + 1) * P],
moving=xT[:, kt, q0:q0 + rw])
ge = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=ge[:, 0:rw], op=nl.gelu_apprx_tanh, data=pg[:, 0:rw])
nisa.tensor_tensor(dst=gl[:, it, 0:rw], data1=pu[:, 0:rw], data2=ge[:, 0:rw], op=nl.multiply)
# down partial for this chunk, accumulated into acc
for rs2 in range(rw // P):
rs = q0 // P + rs2
for ko in range(n_ko):
k0 = ko * FMAX
nk = min(FMAX, K - k0)
pd = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for it in nl.affine_range(n_ic):
nisa.nc_matmul(dst=pd[:, 0:nk], stationary=gl[:, it, rs2 * P:(rs2 + 1) * P],
moving=wd[:, it, k0:k0 + nk])
nisa.tensor_tensor(dst=acc[:, rs, k0:k0 + nk], data1=pd[:, 0:nk],
data2=acc[:, rs, k0:k0 + nk], op=nl.add)
# ---- LayerNorm epilogue ----
for rs in range(n_sub):
s_sb = acc[:, rs, :]
stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf)
for ch in range(n_ch):
nisa.bn_stats(dst=stats[:, ch * 6:(ch + 1) * 6], data=s_sb[:, ch * csz:(ch + 1) * csz])
mv = nl.ndarray((P, 2), dtype=nl.float32, buffer=nl.sbuf)
nisa.bn_aggr(dst=mv, data=stats)
rstd = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.activation(dst=rstd, op=nl.rsqrt, data=mv[:, 1:2], bias=eps_sb)
u_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mv[:, 0:1],
op1=nl.multiply, operand1=ga_sb)
v_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd)
nisa.dma_compute(dst=y_hbm[nl.ds(rb + rs * P, P), 0:K], srcs=[v_sb, be_sb], reduce_op=nl.add)
return y_hbm