Download build/torch-neuron/attn_block_kernel.py from jburtoft/gte-encoder-neuron-kernels: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/attn_block_kernel.py
- Command line
-
hf download hf://jburtoft/gte-encoder-neuron-kernels/build/torch-neuron/attn_block_kernel.py
-
curl -L -o attn_block_kernel.py https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/attn_block_kernel.py
18.4 kB
| """Fused NewBERT attention block: y = attn_ln(x + o_proj(attention(rope(q), rope(k), v))) | |
| Replaces qkv_proj + RoPE + masked softmax attention + o_proj + residual + attn_ln of one layer. | |
| Inputs (bf16 unless noted), all weights host-packed once (see attn_block_pack.pack): | |
| x [B*L, H] layer input, token-major (head_dim D = 64, NH = H / 64 heads, NH % 4 == 0) | |
| wqk [H, 2*H] Q and K projection, transposed, LANE-PERMUTED, Q pre-scaled by 1/sqrt(D) | |
| bqk [128, 2*H/128] Q/K bias, same permutation, one column per 128-lane group (fp32) | |
| wv [H, H] V projection transposed; bv [128, H] broadcast (fp32) | |
| wo [H, H] o_proj transposed; bo [128, H] broadcast (fp32) | |
| g, b [128, H] attn_ln weight / bias broadcast (fp32) | |
| cs [2, 128, L] RoPE cos / sin per lane (fp32); lane l uses frequency l % 32 | |
| kb [B, L] additive key bias (0 valid, -30000 padded), in already-scaled score units | |
| Output y [B*L, H]. | |
| Lane permutation (RoPE without transposes): 128-lane group (j, par) holds lanes b*32+i = feature | |
| (4j+b)*D + par*32 + i of heads 4j..4j+3. Group par=0 holds the low halves, par=1 the high halves, so | |
| rotate-half RoPE is elementwise between the two groups: | |
| lo' = lo*cos - hi*sin ; hi' = hi*cos + lo*sin | |
| and the results are written per head into qa/ka[0:64, h, :] (rows 0-31 lo', 32-63 hi'). Q and K | |
| use the same order, so the dot product is unchanged. | |
| Per projection pass (sequences split across LNC programs; up to 4 sequences packed per pass so the | |
| projection moving width is 512): | |
| x rows -> x^T (DMA transpose); Q/K projections (feature-major) + bias + RoPE -> qa/ka [65, NH, L] | |
| with a ones row (qa) / key-bias row (ka) at partition 64, so one K=65 matmul gives S = QK^T + kb. | |
| V projection (token-major) + bias -> va [128, L/128, NH, 65] with a ones column, so the PV matmul | |
| also produces the softmax row sums. | |
| Per 128-query tile: S -> exp(S - max) -> P^T (one DMA transpose per head group) -> PV | |
| -> normalise -> o^T (PE transposes) -> o_proj -> + bo + x -> bn_stats. | |
| The LayerNorm epilogue (rsqrt + affine) is deferred until all query tiles of the pass are done, so | |
| the Scalar engine does not alternate between the exp and rsqrt activation tables. | |
| Query split: at LNC > 1 with a batch not divisible by the core count, every core computes K/V for | |
| every sequence but only its own share of the query tiles (needs (L/128) % LNC == 0). No cross-core | |
| reduction is needed. | |
| L must be a multiple of 128 and <= 1024. Above 512 keys (the matmul moving-size limit) the projection and QK^T | |
| matmuls are issued in 512-column chunks; one head's score row (<= 1024 fp32 = 2 PSUM banks) still fits in PSUM, so | |
| the softmax stays exact (no running max). Right padding only. Inference only. | |
| """ | |
| import nki | |
| import nki.isa as nisa | |
| import nki.language as nl | |
| P = 128 | |
| FMAX = 512 | |
| LMAX = 1024 | |
| PSUM_BANK = 2048 # bytes per partition per PSUM bank | |
| def _cdiv(a, b): | |
| return (a + b - 1) // b | |
| def _group(NH, L, D=64): | |
| """Heads per softmax group: S [128, G, L] fp32 spans at most 2 banks (G * L <= 1024), and the PV | |
| output O [128, G, D + 1] fp32 must stay inside one bank (a matmul cannot write across banks).""" | |
| best = 1 | |
| for g in range(1, NH + 1): | |
| if NH % g == 0 and g * L <= 1024 and g * (D + 1) * 4 <= PSUM_BANK: | |
| best = g | |
| return best | |
| def _ln_out(s_sb, mean, rstd, ga, be, y_hbm, row0, P_, H, dt): | |
| u_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf) | |
| nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mean, | |
| op1=nl.multiply, operand1=ga) | |
| v_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd) | |
| nisa.dma_compute(dst=y_hbm[nl.ds(row0, P_), 0:H], srcs=[v_sb, be], reduce_op=nl.add) | |
| def attn_block_kernel(x_hbm, wqk_hbm, bqk_hbm, wv_hbm, bv_hbm, wo_hbm, bo_hbm, g_hbm, b_hbm, cs_hbm, kb_hbm, | |
| eps=1e-12): | |
| R, H = x_hbm.shape | |
| B, L = kb_hbm.shape | |
| D = 64 | |
| NH = H // D | |
| half = D // 2 | |
| quad = P // half # heads per 128-lane group (4) | |
| NQ = NH // quad # lane groups per parity | |
| assert NH * D == H | |
| assert R == B * L | |
| assert L % P == 0 | |
| assert L <= LMAX | |
| assert H % P == 0 | |
| assert NH % quad == 0 | |
| n_prog = nl.num_programs(0) | |
| pid = nl.program_id(0) | |
| nt0 = L // P | |
| QS = 0 | |
| if n_prog > 1 and B % n_prog != 0: | |
| QS = 1 | |
| if QS == 1: | |
| assert nt0 % n_prog == 0 | |
| Bp = B | |
| else: | |
| assert B % n_prog == 0 | |
| Bp = B // n_prog | |
| n_k = H // P # contraction tiles over H | |
| nt = L // P # token tiles per sequence | |
| G = _group(NH, L) | |
| NG = NH // G | |
| n_ko = _cdiv(H, FMAX) | |
| fmax = nl.tile_size.bn_stats_fmax | |
| n_ch = _cdiv(H, fmax) | |
| csz = H // n_ch | |
| dt = x_hbm.dtype | |
| y_hbm = nl.ndarray((R, H), dtype=dt, buffer=nl.shared_hbm) | |
| # ---------------- resident weights / constants ---------------- | |
| wqk = nl.ndarray((P, n_k, 2 * H), dtype=dt, buffer=nl.sbuf) | |
| wv = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf) | |
| wo = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf) | |
| for kt in range(n_k): | |
| nisa.dma_copy(dst=wqk[:, kt, :], src=wqk_hbm[kt * P:(kt + 1) * P, 0:2 * H]) | |
| nisa.dma_copy(dst=wv[:, kt, :], src=wv_hbm[kt * P:(kt + 1) * P, 0:H]) | |
| nisa.dma_copy(dst=wo[:, kt, :], src=wo_hbm[kt * P:(kt + 1) * P, 0:H]) | |
| bqk = nl.ndarray((P, 2 * H // P), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=bqk, src=bqk_hbm[0:P, 0:2 * H // P]) | |
| bv = nl.ndarray((P, H), dtype=bv_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=bv, src=bv_hbm[0:P, 0:H]) | |
| bo = nl.ndarray((P, H), dtype=bo_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=bo, src=bo_hbm[0:P, 0:H]) | |
| ga = nl.ndarray((P, H), dtype=g_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=ga, src=g_hbm[0:P, 0:H]) | |
| be = nl.ndarray((P, H), dtype=b_hbm.dtype, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=be, src=b_hbm[0:P, 0:H]) | |
| cos_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf) | |
| sin_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=cos_t, src=cs_hbm[0, 0:P, 0:L]) | |
| nisa.dma_copy(dst=sin_t, src=cs_hbm[1, 0:P, 0:L]) | |
| eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.memset(dst=eps_sb, value=eps) | |
| # sequences per projection pass: 4 at L=128, 2 at L=256, 1 at L>=384 (or if Bp does not divide) | |
| SP = 1 | |
| if QS == 0: | |
| SP = FMAX // L | |
| if SP > 4: | |
| SP = 4 | |
| if SP < 1: | |
| SP = 1 | |
| if Bp % SP != 0: | |
| SP = 1 | |
| LW = L * SP # projection moving width | |
| # Long sequences (L > 512): x + bo is formed per query tile instead of for the whole sequence, and the deferred | |
| # LayerNorm epilogue runs per group of LN_GROUP query tiles, to bound SBUF use. For L <= 512 a pass has at most | |
| # 4 query tiles, so the epilogue is deferred over the whole pass as before. | |
| LONG = 0 | |
| if L > FMAX: | |
| LONG = 1 | |
| LN_GROUP = 4 | |
| cosw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| sinw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| for sp in range(SP): | |
| nisa.tensor_copy(dst=cosw[:, sp * L:(sp + 1) * L], src=cos_t) | |
| nisa.tensor_copy(dst=sinw[:, sp * L:(sp + 1) * L], src=sin_t) | |
| for bi in nl.affine_range(Bp // SP): | |
| bseq = pid * Bp + bi * SP | |
| if QS == 1: | |
| bseq = bi | |
| r0 = bseq * L | |
| ntw = nt * SP # token tiles in this pass | |
| # ---------------- x rows, x + bo, x^T ---------------- | |
| xr = nl.ndarray((P, ntw, H), dtype=dt, buffer=nl.sbuf) | |
| if LONG == 0: | |
| xb = nl.ndarray((P, ntw, H), dtype=nl.float32, buffer=nl.sbuf) | |
| xT = nl.ndarray((P, n_k, LW), dtype=dt, buffer=nl.sbuf) | |
| for t in range(ntw): | |
| nisa.dma_copy(dst=xr[:, t, :], src=x_hbm[nl.ds(r0 + t * P, P), 0:H]) | |
| if LONG == 0: | |
| nisa.tensor_tensor(dst=xb[:, t, :], data1=xr[:, t, :], data2=bo, op=nl.add, | |
| engine=nisa.engine.gpsimd) | |
| for kt in range(n_k): | |
| nisa.dma_transpose(dst=xT[:, kt, t * P:(t + 1) * P], src=xr[:, t, kt * P:(kt + 1) * P]) | |
| # ---------------- Q/K projection + bias + RoPE -> qa / ka ---------------- | |
| qa = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf) | |
| ka = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf) | |
| nisa.memset(dst=qa[D:D + 1, :, :], value=1.0, engine=nisa.engine.gpsimd) | |
| for sp in range(SP): | |
| nisa.dma_copy(dst=ka[D:D + 1, :, sp * L:(sp + 1) * L], | |
| src=kb_hbm.ap(pattern=[[L, 1], [0, NH], [1, L]], offset=(bseq + sp) * L)) | |
| for qk in range(2): | |
| dst_t = qa | |
| if qk == 1: | |
| dst_t = ka | |
| for j in range(NQ): | |
| pr = [] | |
| for par in range(2): | |
| gi = (qk * NQ + j) * 2 + par | |
| ps = nl.ndarray((P, LW), dtype=nl.float32, buffer=nl.psum) | |
| if LW <= FMAX: | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=ps, stationary=wqk[:, kt, gi * P:(gi + 1) * P], moving=xT[:, kt, :]) | |
| else: | |
| for c0w in range(0, LW, FMAX): | |
| cw = min(FMAX, LW - c0w) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=ps[:, c0w:c0w + cw], stationary=wqk[:, kt, gi * P:(gi + 1) * P], | |
| moving=xT[:, kt, c0w:c0w + cw]) | |
| pt = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| nisa.activation(dst=pt, op=nl.copy, data=ps, bias=bqk[:, gi:gi + 1]) | |
| pr.append(pt) | |
| lo = pr[0] | |
| hi = pr[1] | |
| a0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| t0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| a1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| t1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| nisa.tensor_tensor(dst=a0, data1=lo, data2=cosw, op=nl.multiply) | |
| nisa.tensor_tensor(dst=t0, data1=hi, data2=sinw, op=nl.multiply, engine=nisa.engine.vector) | |
| nisa.tensor_tensor(dst=a1, data1=hi, data2=cosw, op=nl.multiply) | |
| nisa.tensor_tensor(dst=t1, data1=lo, data2=sinw, op=nl.multiply, engine=nisa.engine.vector) | |
| r_lo = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| r_hi = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) | |
| nisa.tensor_tensor(dst=r_lo, data1=a0, data2=t0, op=nl.subtract) | |
| nisa.tensor_tensor(dst=r_hi, data1=a1, data2=t1, op=nl.add) | |
| # Per-head copies into the head-major qa/ka layout. A single strided DMA spanning several | |
| # 32-partition blocks matches in nki.simulate but copies only the first block on trn2, so | |
| # engine copies are used here. | |
| for bq in range(quad): | |
| h = j * quad + bq | |
| rr = bq * half | |
| nisa.tensor_copy(dst=dst_t[0:half, h, :], src=r_lo[rr:rr + half, :], engine=nisa.engine.scalar) | |
| nisa.tensor_copy(dst=dst_t[half:D, h, :], src=r_hi[rr:rr + half, :], engine=nisa.engine.unknown) | |
| # ---------------- V projection (token-major) + bias -> va ---------------- | |
| va = nl.ndarray((P, ntw, NH, D + 1), dtype=dt, buffer=nl.sbuf) | |
| nisa.memset(dst=va[:, :, :, D:D + 1], value=1.0, engine=nisa.engine.gpsimd) | |
| for t in range(ntw): | |
| for ko in range(n_ko): | |
| k0 = ko * FMAX | |
| nk = min(FMAX, H - k0) | |
| hv = nk // D | |
| pv = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=pv[:, 0:nk], stationary=xT[:, kt, t * P:(t + 1) * P], | |
| moving=wv[:, kt, k0:k0 + nk]) | |
| h0 = k0 // D | |
| nisa.tensor_tensor(dst=va[:, t, h0:h0 + hv, 0:D], data1=pv[:, 0:nk], data2=bv[:, k0:k0 + nk], | |
| op=nl.add) | |
| # ---------------- attention + o_proj + residual + LN stats, per query tile ---------------- | |
| nq = ntw | |
| if QS == 1: | |
| nq = ntw // n_prog | |
| qbase = pid * nq # SYMBOLIC: index arithmetic only | |
| xbq = nl.ndarray((P, nq, H), dtype=nl.float32, buffer=nl.sbuf) | |
| for i in range(nq): | |
| xq = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf) | |
| nisa.dma_copy(dst=xq, src=x_hbm[nl.ds(r0 + (qbase + i) * P, P), 0:H]) | |
| nisa.tensor_tensor(dst=xbq[:, i, :], data1=xq, data2=bo, op=nl.add, engine=nisa.engine.gpsimd) | |
| ng = nq | |
| if ng > LN_GROUP: | |
| ng = LN_GROUP | |
| s_all = nl.ndarray((P, ng, H), dtype=nl.float32, buffer=nl.sbuf) | |
| mv_all = nl.ndarray((P, ng, 2), dtype=nl.float32, buffer=nl.sbuf) | |
| for qi in range(nq): | |
| gi_ = qi % ng # slot in the current LayerNorm group | |
| sp = qi // nt | |
| qt = qi % nt | |
| c0 = sp * L # column offset of this sequence in qa/ka | |
| t_off = sp * nt # token-tile offset of this sequence in va / xb | |
| qcol = c0 + qt * P # query column start in qa | |
| if QS == 1: | |
| qcol = (qbase + qi) * P | |
| xres = xbq[:, qi, :] | |
| elif LONG == 1: | |
| xres = nl.ndarray((P, H), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.tensor_tensor(dst=xres, data1=xr[:, t_off + qt, :], data2=bo, op=nl.add, | |
| engine=nisa.engine.gpsimd) | |
| else: | |
| xres = xb[:, t_off + qt, :] | |
| o_sb = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf) | |
| for gg in range(NG): | |
| S = nl.ndarray((P, G, L), dtype=nl.float32, buffer=nl.psum) | |
| for hl in range(G): | |
| h = gg * G + hl | |
| if L <= FMAX: | |
| nisa.nc_matmul(dst=S[:, hl, :], stationary=qa[0:D + 1, h, nl.ds(qcol, P)], | |
| moving=ka[0:D + 1, h, c0:c0 + L]) | |
| else: | |
| for k0c in range(0, L, FMAX): | |
| kw = min(FMAX, L - k0c) | |
| nisa.nc_matmul(dst=S[:, hl, k0c:k0c + kw], stationary=qa[0:D + 1, h, nl.ds(qcol, P)], | |
| moving=ka[0:D + 1, h, c0 + k0c:c0 + k0c + kw]) | |
| negmax = nl.ndarray((P, G), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.tensor_reduce(dst=negmax, op=nl.maximum, data=S, axis=2, negate=True) | |
| Pm = nl.ndarray((P, G, L), dtype=dt, buffer=nl.sbuf) | |
| for hl in range(G): | |
| nisa.activation(dst=Pm[:, hl, :], op=nl.exp, data=S[:, hl, :], bias=negmax[:, hl:hl + 1]) | |
| PT = nl.ndarray((P, G, nt, P), dtype=dt, buffer=nl.sbuf) | |
| nisa.dma_transpose(dst=PT.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0), | |
| src=Pm.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0)) | |
| O = nl.ndarray((P, G, D + 1), dtype=nl.float32, buffer=nl.psum) | |
| for hl in range(G): | |
| h = gg * G + hl | |
| for kt in nl.affine_range(nt): | |
| nisa.nc_matmul(dst=O[:, hl, :], stationary=PT[:, hl, kt, :], moving=va[:, t_off + kt, h, :]) | |
| rs = nl.ndarray((P, G, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.reciprocal(dst=rs, data=O[:, :, D:D + 1]) | |
| o3 = o_sb[:, gg * G * D:(gg + 1) * G * D].reshape((P, G, D)) | |
| nisa.tensor_tensor(dst=o3, data1=O[:, :, 0:D], | |
| data2=rs.ap(pattern=[[G, P], [1, G], [0, D]], offset=0), op=nl.multiply) | |
| # o^T for the output projection | |
| oTp = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.psum) | |
| for kt in range(n_k): | |
| nisa.nc_transpose(dst=oTp[:, kt, :], data=o_sb[:, kt * P:(kt + 1) * P], engine=nisa.engine.tensor) | |
| oT = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.sbuf) | |
| nisa.tensor_copy(dst=oT, src=oTp, engine=nisa.engine.scalar) | |
| s_sb = s_all[:, gi_, :] | |
| for ko in range(n_ko): | |
| k0 = ko * FMAX | |
| nk = min(FMAX, H - k0) | |
| po = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) | |
| for kt in nl.affine_range(n_k): | |
| nisa.nc_matmul(dst=po[:, 0:nk], stationary=oT[:, kt, :], moving=wo[:, kt, k0:k0 + nk]) | |
| nisa.tensor_tensor(dst=s_sb[:, k0:k0 + nk], data1=po[:, 0:nk], data2=xres[:, k0:k0 + nk], | |
| op=nl.add) | |
| stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf) | |
| for c in range(n_ch): | |
| nisa.bn_stats(dst=stats[:, c * 6:(c + 1) * 6], data=s_sb[:, c * csz:(c + 1) * csz]) | |
| nisa.bn_aggr(dst=mv_all[:, gi_, :], data=stats) | |
| # ---------------- deferred LayerNorm epilogue (once per group of query tiles) ---------------- | |
| if gi_ == ng - 1 or qi == nq - 1: | |
| g0 = qi - gi_ # first query tile of this group | |
| gn = gi_ + 1 | |
| rstd_all = nl.ndarray((P, ng, 1), dtype=nl.float32, buffer=nl.sbuf) | |
| nisa.activation(dst=rstd_all[:, 0:gn, :], op=nl.rsqrt, data=mv_all[:, 0:gn, 1:2], bias=eps_sb) | |
| for qj in range(gn): | |
| yr = r0 + (g0 + qj) * P | |
| if QS == 1: | |
| yr = r0 + (qbase + g0 + qj) * P | |
| _ln_out(s_all[:, qj, :], mv_all[:, qj, 0:1], rstd_all[:, qj, :], ga, be, y_hbm, | |
| yr, P, H, dt) | |
| return y_hbm | |