Download build/torch-neuron/gated_mlp_kernel.py from jburtoft/gte-encoder-neuron-kernels: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/gated_mlp_kernel.py
- Command line
-
hf download hf://jburtoft/gte-encoder-neuron-kernels/build/torch-neuron/gated_mlp_kernel.py
-
curl -L -o gated_mlp_kernel.py https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/gated_mlp_kernel.py
13.4 kB
| """Fused NewBERT gated MLP + residual + post-LayerNorm (replaces NewGatedMLP + mlp_ln). | |
| h = x @ Wup^T, gt = x @ Wgate^T (up_gate_proj, no bias; split [up | gate]) | |
| m = (gelu_tanh(gt) * h) @ Wd^T + bd (down_proj) | |
| y = LayerNorm(x + m) * gamma + beta (mlp_ln, residual = MLP input) | |
| Shapes: x [R, K] bf16 (K = hidden), wupT / wgT [K, I] bf16 (I = intermediate), wdT [I, K] bf16, | |
| bd / gamma / beta pre-broadcast to [128, K] (fp32 or bf16). Returns y [R, K] bf16. | |
| Two kernels with the same inputs; pick with mlp_weights_fit(K, I): | |
| gated_mlp_ln_kernel weights resident in SBUF (~108 KiB per partition at K=768, I=3072) | |
| gated_mlp_ln_stream_kernel weights streamed in I chunks (K=1024, I=4096 needs 192 KiB resident: too big) | |
| Resident layout: K-tiles x I for up/gate, I-tiles x K for down. Rows are processed in blocks of up to | |
| 512 (the matmul moving free size). Per block: | |
| * x rows are loaded once (row layout, kept for the residual) and transposed to x^T (K on | |
| partitions) by DMA transpose. | |
| * up/gate for each 128-wide I tile land in PSUM [128 (I), rows]; gelu_tanh on Scalar, multiply | |
| on Vector -> g [I tile, rows] bf16, already in the layout the down-projection needs. | |
| * down: per 128-row subtile, PSUM [128 rows, K] accumulated over the I tiles. | |
| * epilogue: s = psum + x + bd (fp32), bn_stats/bn_aggr, y = (s - mean) * gamma * rstd + beta. | |
| R must be a multiple of 128 * LNC; rows are split across LNC programs. Inference only. | |
| """ | |
| import nki | |
| import nki.isa as nisa | |
| import nki.language as nl | |
| P = 128 | |
| FMAX = 512 | |
| def _cdiv(a, b): | |
| return (a + b - 1) // b | |
| def gated_mlp_ln_kernel(x_hbm, wupT_hbm, wgT_hbm, wdT_hbm, bd_hbm, g_hbm, b_hbm, eps=1e-12): | |
| R, K = x_hbm.shape | |
| K1, I = wupT_hbm.shape | |
| assert K1 == K | |
| assert K % P == 0 | |
| assert I % P == 0 | |
| n_prog = nl.num_programs(0) | |
| pid = nl.program_id(0) | |
| assert R % (P * n_prog) == 0 | |
| rows_per = R // n_prog | |
| r_base = pid * rows_per | |
| n_k = K // P | |
| n_i = I // P | |
| n_ko = _cdiv(K, FMAX) | |
| n_blk = _cdiv(rows_per, FMAX) | |
| fmax = nl.tile_size.bn_stats_fmax | |
| n_ch = _cdiv(K, fmax) | |
| csz = K // n_ch | |
| dt = x_hbm.dtype | |
| y_hbm = nl.ndarray((R, K), dtype=dt, buffer=nl.shared_hbm) | |
| # ---- resident weights ---- | |
| wu = [] | |
| wg = [] | |
| for kt in range(n_k): | |
| tu = nl.ndarray((P, I), dtype=wupT_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=tu, src=wupT_hbm[kt * P:(kt + 1) * P, 0:I]) | |
| wu.append(tu) | |
| tg = nl.ndarray((P, I), dtype=wgT_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=tg, src=wgT_hbm[kt * P:(kt + 1) * P, 0:I]) | |
| wg.append(tg) | |
| wd = [] | |
| for it in range(n_i): | |
| td = nl.ndarray((P, K), dtype=wdT_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=td, src=wdT_hbm[it * P:(it + 1) * P, 0:K]) | |
| wd.append(td) | |
| bd_sb = nl.ndarray((P, K), dtype=bd_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=bd_sb, src=bd_hbm[0:P, 0:K]) | |
| ga_sb = nl.ndarray((P, K), dtype=g_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=ga_sb, src=g_hbm[0:P, 0:K]) | |
| be_sb = nl.ndarray((P, K), dtype=b_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=be_sb, src=b_hbm[0:P, 0:K]) | |
| eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.memset(dst=eps_sb, value=eps) | |
| for blk in range(n_blk): | |
| q0 = blk * FMAX | |
| rw = min(FMAX, rows_per - q0) | |
| n_rs = rw // P | |
| rb = r_base + q0 | |
| # ---- x rows (kept for the residual) and x^T ---- | |
| xr = [] | |
| xt = [] | |
| for kt in range(n_k): | |
| xt.append(nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf)) | |
| for rs in range(n_rs): | |
| t = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=t, src=x_hbm[nl.ds(rb + rs * P, P), 0:K]) | |
| xr.append(t) | |
| for kt in range(n_k): | |
| nisa.dma_transpose(dst=xt[kt][0:P, rs * P:(rs + 1) * P], src=t[0:P, kt * P:(kt + 1) * P]) | |
| # ---- up / gate -> g [I tile, rows] ---- | |
| gl = [] | |
| for it in range(n_i): | |
| i0 = it * P | |
| pu = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=pu[0:P, 0:rw], stationary=wu[kt][0:P, i0:i0 + P], moving=xt[kt][0:P, 0:rw]) | |
| pg = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=pg[0:P, 0:rw], stationary=wg[kt][0:P, i0:i0 + P], moving=xt[kt][0:P, 0:rw]) | |
| ge = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=ge[0:P, 0:rw], op=nl.gelu_apprx_tanh, data=pg[0:P, 0:rw]) | |
| g = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf) | |
| nisa.tensor_tensor(dst=g[0:P, 0:rw], data1=pu[0:P, 0:rw], data2=ge[0:P, 0:rw], op=nl.multiply) | |
| gl.append(g) | |
| # ---- down + residual + LN, per 128-row subtile ---- | |
| for rs in range(n_rs): | |
| s_sb = nl.ndarray((P, K), dtype=nl.float32, buffer=nl.sbuf) | |
| xb = nl.ndarray((P, K), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.tensor_tensor(dst=xb, data1=xr[rs], data2=bd_sb, op=nl.add, engine=nisa.engine.gpsimd) | |
| for ko in range(n_ko): | |
| k0 = ko * FMAX | |
| nk = min(FMAX, K - k0) | |
| pd = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for it in nl.affine_range(n_i): | |
| nisa.nc_matmul(dst=pd[0:P, 0:nk], stationary=gl[it][0:P, rs * P:(rs + 1) * P], | |
| moving=wd[it][0:P, k0:k0 + nk]) | |
| nisa.tensor_tensor(dst=s_sb[0:P, k0:k0 + nk], data1=pd[0:P, 0:nk], data2=xb[0:P, k0:k0 + nk], | |
| op=nl.add) | |
| stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf) | |
| for c in range(n_ch): | |
| nisa.bn_stats(dst=stats[0:P, c * 6:(c + 1) * 6], data=s_sb[0:P, c * csz:(c + 1) * csz]) | |
| mv = nl.ndarray((P, 2), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.bn_aggr(dst=mv, data=stats) | |
| rstd = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.activation(dst=rstd, op=nl.rsqrt, data=mv[0:P, 1:2], bias=eps_sb) | |
| u_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mv[0:P, 0:1], | |
| op1=nl.multiply, operand1=ga_sb) | |
| v_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd) | |
| nisa.dma_compute(dst=y_hbm[nl.ds(rb + rs * P, P), 0:K], srcs=[v_sb, be_sb], reduce_op=nl.add) | |
| return y_hbm | |
| # ---------------------------------------------------------------- weight-streaming variant (large K * I) | |
| RESIDENT_BUDGET = 128 * 1024 # bytes per partition the resident kernel may spend on weights | |
| STREAM_IC = 1024 # intermediate columns per streamed weight chunk | |
| STREAM_SB = 1024 # rows per super-block (rows that share one pass over the weights) | |
| def mlp_weights_fit(K, I, itemsize=2): | |
| """True if gated_mlp_ln_kernel can keep all three weight matrices resident in SBUF.""" | |
| return (2 * (K // P) * I + (I // P) * K) * itemsize <= RESIDENT_BUDGET | |
| def gated_mlp_ln_stream_kernel(x_hbm, wupT_hbm, wgT_hbm, wdT_hbm, bd_hbm, g_hbm, b_hbm, eps=1e-12): | |
| """Same math and inputs as gated_mlp_ln_kernel, for weights too large to keep resident | |
| (e.g. K=1024, I=4096). Weights are streamed in chunks of STREAM_IC intermediate columns, once per | |
| super-block of up to STREAM_SB rows; the down-projection partials accumulate in fp32 SBUF, seeded | |
| with x + bd, so the residual costs nothing extra.""" | |
| R, K = x_hbm.shape | |
| K1, I = wupT_hbm.shape | |
| assert K1 == K | |
| assert K % P == 0 | |
| assert I % P == 0 | |
| n_prog = nl.num_programs(0) | |
| pid = nl.program_id(0) | |
| assert R % (P * n_prog) == 0 | |
| rows_per = R // n_prog | |
| r_base = pid * rows_per | |
| IC = STREAM_IC | |
| if I % IC != 0: | |
| IC = P | |
| n_c = I // IC | |
| n_ic = IC // P | |
| n_k = K // P | |
| n_ko = _cdiv(K, FMAX) | |
| n_sb = _cdiv(rows_per, STREAM_SB) | |
| fmax = nl.tile_size.bn_stats_fmax | |
| n_ch = _cdiv(K, fmax) | |
| csz = K // n_ch | |
| dt = x_hbm.dtype | |
| y_hbm = nl.ndarray((R, K), dtype=dt, buffer=nl.shared_hbm) | |
| bd_sb = nl.ndarray((P, K), dtype=bd_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=bd_sb, src=bd_hbm[0:P, 0:K]) | |
| ga_sb = nl.ndarray((P, K), dtype=g_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=ga_sb, src=g_hbm[0:P, 0:K]) | |
| be_sb = nl.ndarray((P, K), dtype=b_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=be_sb, src=b_hbm[0:P, 0:K]) | |
| eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.memset(dst=eps_sb, value=eps) | |
| for sb in range(n_sb): | |
| s0 = sb * STREAM_SB | |
| sw = min(STREAM_SB, rows_per - s0) | |
| n_sub = sw // P | |
| rb = r_base + s0 | |
| # ---- x^T for the super-block; accumulator seeded with x + bd ---- | |
| xT = nl.ndarray((P, n_k, STREAM_SB), dtype=dt, buffer=nl.sbuf) | |
| acc = nl.ndarray((P, n_sub, K), dtype=nl.float32, buffer=nl.sbuf) | |
| for rs in range(n_sub): | |
| t = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=t, src=x_hbm[nl.ds(rb + rs * P, P), 0:K]) | |
| nisa.tensor_tensor(dst=acc[:, rs, :], data1=t, data2=bd_sb, op=nl.add, engine=nisa.engine.gpsimd) | |
| for kt in range(n_k): | |
| nisa.dma_transpose(dst=xT[:, kt, rs * P:(rs + 1) * P], src=t[:, kt * P:(kt + 1) * P]) | |
| for c in range(n_c): | |
| c0 = c * IC | |
| wu = nl.ndarray((P, n_k, IC), dtype=wupT_hbm.dtype, buffer=nl.sbuf) | |
| wg = nl.ndarray((P, n_k, IC), dtype=wgT_hbm.dtype, buffer=nl.sbuf) | |
| wd = nl.ndarray((P, n_ic, K), dtype=wdT_hbm.dtype, buffer=nl.sbuf) | |
| for kt in range(n_k): | |
| nisa.dma_copy(dst=wu[:, kt, :], src=wupT_hbm[kt * P:(kt + 1) * P, c0:c0 + IC]) | |
| nisa.dma_copy(dst=wg[:, kt, :], src=wgT_hbm[kt * P:(kt + 1) * P, c0:c0 + IC]) | |
| for it in range(n_ic): | |
| nisa.dma_copy(dst=wd[:, it, :], src=wdT_hbm[c0 + it * P:c0 + (it + 1) * P, 0:K]) | |
| for blk in range(_cdiv(sw, FMAX)): | |
| q0 = blk * FMAX | |
| rw = min(FMAX, sw - q0) | |
| # up / gate for this chunk -> g [I tile, rows] | |
| gl = nl.ndarray((P, n_ic, FMAX), dtype=dt, buffer=nl.sbuf) | |
| for it in range(n_ic): | |
| pu = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=pu[:, 0:rw], stationary=wu[:, kt, it * P:(it + 1) * P], | |
| moving=xT[:, kt, q0:q0 + rw]) | |
| pg = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=pg[:, 0:rw], stationary=wg[:, kt, it * P:(it + 1) * P], | |
| moving=xT[:, kt, q0:q0 + rw]) | |
| ge = nl.ndarray((P, FMAX), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=ge[:, 0:rw], op=nl.gelu_apprx_tanh, data=pg[:, 0:rw]) | |
| nisa.tensor_tensor(dst=gl[:, it, 0:rw], data1=pu[:, 0:rw], data2=ge[:, 0:rw], op=nl.multiply) | |
| # down partial for this chunk, accumulated into acc | |
| for rs2 in range(rw // P): | |
| rs = q0 // P + rs2 | |
| for ko in range(n_ko): | |
| k0 = ko * FMAX | |
| nk = min(FMAX, K - k0) | |
| pd = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for it in nl.affine_range(n_ic): | |
| nisa.nc_matmul(dst=pd[:, 0:nk], stationary=gl[:, it, rs2 * P:(rs2 + 1) * P], | |
| moving=wd[:, it, k0:k0 + nk]) | |
| nisa.tensor_tensor(dst=acc[:, rs, k0:k0 + nk], data1=pd[:, 0:nk], | |
| data2=acc[:, rs, k0:k0 + nk], op=nl.add) | |
| # ---- LayerNorm epilogue ---- | |
| for rs in range(n_sub): | |
| s_sb = acc[:, rs, :] | |
| stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf) | |
| for ch in range(n_ch): | |
| nisa.bn_stats(dst=stats[:, ch * 6:(ch + 1) * 6], data=s_sb[:, ch * csz:(ch + 1) * csz]) | |
| mv = nl.ndarray((P, 2), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.bn_aggr(dst=mv, data=stats) | |
| rstd = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.activation(dst=rstd, op=nl.rsqrt, data=mv[:, 1:2], bias=eps_sb) | |
| u_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mv[:, 0:1], | |
| op1=nl.multiply, operand1=ga_sb) | |
| v_sb = nl.ndarray((P, K), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd) | |
| nisa.dma_compute(dst=y_hbm[nl.ds(rb + rs * P, P), 0:K], srcs=[v_sb, be_sb], reduce_op=nl.add) | |
| return y_hbm | |