"""Fused NewBERT attention block: y = attn_ln(x + o_proj(attention(rope(q), rope(k), v))) Replaces qkv_proj + RoPE + masked softmax attention + o_proj + residual + attn_ln of one layer. Inputs (bf16 unless noted), all weights host-packed once (see attn_block_pack.pack): x [B*L, H] layer input, token-major (head_dim D = 64, NH = H / 64 heads, NH % 4 == 0) wqk [H, 2*H] Q and K projection, transposed, LANE-PERMUTED, Q pre-scaled by 1/sqrt(D) bqk [128, 2*H/128] Q/K bias, same permutation, one column per 128-lane group (fp32) wv [H, H] V projection transposed; bv [128, H] broadcast (fp32) wo [H, H] o_proj transposed; bo [128, H] broadcast (fp32) g, b [128, H] attn_ln weight / bias broadcast (fp32) cs [2, 128, L] RoPE cos / sin per lane (fp32); lane l uses frequency l % 32 kb [B, L] additive key bias (0 valid, -30000 padded), in already-scaled score units Output y [B*L, H]. Lane permutation (RoPE without transposes): 128-lane group (j, par) holds lanes b*32+i = feature (4j+b)*D + par*32 + i of heads 4j..4j+3. Group par=0 holds the low halves, par=1 the high halves, so rotate-half RoPE is elementwise between the two groups: lo' = lo*cos - hi*sin ; hi' = hi*cos + lo*sin and the results are written per head into qa/ka[0:64, h, :] (rows 0-31 lo', 32-63 hi'). Q and K use the same order, so the dot product is unchanged. Per projection pass (sequences split across LNC programs; up to 4 sequences packed per pass so the projection moving width is 512): x rows -> x^T (DMA transpose); Q/K projections (feature-major) + bias + RoPE -> qa/ka [65, NH, L] with a ones row (qa) / key-bias row (ka) at partition 64, so one K=65 matmul gives S = QK^T + kb. V projection (token-major) + bias -> va [128, L/128, NH, 65] with a ones column, so the PV matmul also produces the softmax row sums. Per 128-query tile: S -> exp(S - max) -> P^T (one DMA transpose per head group) -> PV -> normalise -> o^T (PE transposes) -> o_proj -> + bo + x -> bn_stats. The LayerNorm epilogue (rsqrt + affine) is deferred until all query tiles of the pass are done, so the Scalar engine does not alternate between the exp and rsqrt activation tables. Query split: at LNC > 1 with a batch not divisible by the core count, every core computes K/V for every sequence but only its own share of the query tiles (needs (L/128) % LNC == 0). No cross-core reduction is needed. L must be a multiple of 128 and <= 1024. Above 512 keys (the matmul moving-size limit) the projection and QK^T matmuls are issued in 512-column chunks; one head's score row (<= 1024 fp32 = 2 PSUM banks) still fits in PSUM, so the softmax stays exact (no running max). Right padding only. Inference only. """ import nki import nki.isa as nisa import nki.language as nl P = 128 FMAX = 512 LMAX = 1024 PSUM_BANK = 2048 # bytes per partition per PSUM bank def _cdiv(a, b): return (a + b - 1) // b def _group(NH, L, D=64): """Heads per softmax group: S [128, G, L] fp32 spans at most 2 banks (G * L <= 1024), and the PV output O [128, G, D + 1] fp32 must stay inside one bank (a matmul cannot write across banks).""" best = 1 for g in range(1, NH + 1): if NH % g == 0 and g * L <= 1024 and g * (D + 1) * 4 <= PSUM_BANK: best = g return best def _ln_out(s_sb, mean, rstd, ga, be, y_hbm, row0, P_, H, dt): u_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf) nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mean, op1=nl.multiply, operand1=ga) v_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf) nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd) nisa.dma_compute(dst=y_hbm[nl.ds(row0, P_), 0:H], srcs=[v_sb, be], reduce_op=nl.add) @nki.jit def attn_block_kernel(x_hbm, wqk_hbm, bqk_hbm, wv_hbm, bv_hbm, wo_hbm, bo_hbm, g_hbm, b_hbm, cs_hbm, kb_hbm, eps=1e-12): R, H = x_hbm.shape B, L = kb_hbm.shape D = 64 NH = H // D half = D // 2 quad = P // half # heads per 128-lane group (4) NQ = NH // quad # lane groups per parity assert NH * D == H assert R == B * L assert L % P == 0 assert L <= LMAX assert H % P == 0 assert NH % quad == 0 n_prog = nl.num_programs(0) pid = nl.program_id(0) nt0 = L // P QS = 0 if n_prog > 1 and B % n_prog != 0: QS = 1 if QS == 1: assert nt0 % n_prog == 0 Bp = B else: assert B % n_prog == 0 Bp = B // n_prog n_k = H // P # contraction tiles over H nt = L // P # token tiles per sequence G = _group(NH, L) NG = NH // G n_ko = _cdiv(H, FMAX) fmax = nl.tile_size.bn_stats_fmax n_ch = _cdiv(H, fmax) csz = H // n_ch dt = x_hbm.dtype y_hbm = nl.ndarray((R, H), dtype=dt, buffer=nl.shared_hbm) # ---------------- resident weights / constants ---------------- wqk = nl.ndarray((P, n_k, 2 * H), dtype=dt, buffer=nl.sbuf) wv = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf) wo = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf) for kt in range(n_k): nisa.dma_copy(dst=wqk[:, kt, :], src=wqk_hbm[kt * P:(kt + 1) * P, 0:2 * H]) nisa.dma_copy(dst=wv[:, kt, :], src=wv_hbm[kt * P:(kt + 1) * P, 0:H]) nisa.dma_copy(dst=wo[:, kt, :], src=wo_hbm[kt * P:(kt + 1) * P, 0:H]) bqk = nl.ndarray((P, 2 * H // P), dtype=nl.float32, buffer=nl.sbuf) nisa.dma_copy(dst=bqk, src=bqk_hbm[0:P, 0:2 * H // P]) bv = nl.ndarray((P, H), dtype=bv_hbm.dtype, buffer=nl.sbuf) nisa.dma_copy(dst=bv, src=bv_hbm[0:P, 0:H]) bo = nl.ndarray((P, H), dtype=bo_hbm.dtype, buffer=nl.sbuf) nisa.dma_copy(dst=bo, src=bo_hbm[0:P, 0:H]) ga = nl.ndarray((P, H), dtype=g_hbm.dtype, buffer=nl.sbuf) nisa.dma_copy(dst=ga, src=g_hbm[0:P, 0:H]) be = nl.ndarray((P, H), dtype=b_hbm.dtype, buffer=nl.sbuf) nisa.dma_copy(dst=be, src=b_hbm[0:P, 0:H]) cos_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf) sin_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf) nisa.dma_copy(dst=cos_t, src=cs_hbm[0, 0:P, 0:L]) nisa.dma_copy(dst=sin_t, src=cs_hbm[1, 0:P, 0:L]) eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf) nisa.memset(dst=eps_sb, value=eps) # sequences per projection pass: 4 at L=128, 2 at L=256, 1 at L>=384 (or if Bp does not divide) SP = 1 if QS == 0: SP = FMAX // L if SP > 4: SP = 4 if SP < 1: SP = 1 if Bp % SP != 0: SP = 1 LW = L * SP # projection moving width # Long sequences (L > 512): x + bo is formed per query tile instead of for the whole sequence, and the deferred # LayerNorm epilogue runs per group of LN_GROUP query tiles, to bound SBUF use. For L <= 512 a pass has at most # 4 query tiles, so the epilogue is deferred over the whole pass as before. LONG = 0 if L > FMAX: LONG = 1 LN_GROUP = 4 cosw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) sinw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) for sp in range(SP): nisa.tensor_copy(dst=cosw[:, sp * L:(sp + 1) * L], src=cos_t) nisa.tensor_copy(dst=sinw[:, sp * L:(sp + 1) * L], src=sin_t) for bi in nl.affine_range(Bp // SP): bseq = pid * Bp + bi * SP if QS == 1: bseq = bi r0 = bseq * L ntw = nt * SP # token tiles in this pass # ---------------- x rows, x + bo, x^T ---------------- xr = nl.ndarray((P, ntw, H), dtype=dt, buffer=nl.sbuf) if LONG == 0: xb = nl.ndarray((P, ntw, H), dtype=nl.float32, buffer=nl.sbuf) xT = nl.ndarray((P, n_k, LW), dtype=dt, buffer=nl.sbuf) for t in range(ntw): nisa.dma_copy(dst=xr[:, t, :], src=x_hbm[nl.ds(r0 + t * P, P), 0:H]) if LONG == 0: nisa.tensor_tensor(dst=xb[:, t, :], data1=xr[:, t, :], data2=bo, op=nl.add, engine=nisa.engine.gpsimd) for kt in range(n_k): nisa.dma_transpose(dst=xT[:, kt, t * P:(t + 1) * P], src=xr[:, t, kt * P:(kt + 1) * P]) # ---------------- Q/K projection + bias + RoPE -> qa / ka ---------------- qa = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf) ka = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf) nisa.memset(dst=qa[D:D + 1, :, :], value=1.0, engine=nisa.engine.gpsimd) for sp in range(SP): nisa.dma_copy(dst=ka[D:D + 1, :, sp * L:(sp + 1) * L], src=kb_hbm.ap(pattern=[[L, 1], [0, NH], [1, L]], offset=(bseq + sp) * L)) for qk in range(2): dst_t = qa if qk == 1: dst_t = ka for j in range(NQ): pr = [] for par in range(2): gi = (qk * NQ + j) * 2 + par ps = nl.ndarray((P, LW), dtype=nl.float32, buffer=nl.psum) if LW <= FMAX: for kt in nl.affine_range(n_k): nisa.nc_matmul(dst=ps, stationary=wqk[:, kt, gi * P:(gi + 1) * P], moving=xT[:, kt, :]) else: for c0w in range(0, LW, FMAX): cw = min(FMAX, LW - c0w) for kt in nl.affine_range(n_k): nisa.nc_matmul(dst=ps[:, c0w:c0w + cw], stationary=wqk[:, kt, gi * P:(gi + 1) * P], moving=xT[:, kt, c0w:c0w + cw]) pt = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) nisa.activation(dst=pt, op=nl.copy, data=ps, bias=bqk[:, gi:gi + 1]) pr.append(pt) lo = pr[0] hi = pr[1] a0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) t0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) a1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) t1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) nisa.tensor_tensor(dst=a0, data1=lo, data2=cosw, op=nl.multiply) nisa.tensor_tensor(dst=t0, data1=hi, data2=sinw, op=nl.multiply, engine=nisa.engine.vector) nisa.tensor_tensor(dst=a1, data1=hi, data2=cosw, op=nl.multiply) nisa.tensor_tensor(dst=t1, data1=lo, data2=sinw, op=nl.multiply, engine=nisa.engine.vector) r_lo = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) r_hi = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf) nisa.tensor_tensor(dst=r_lo, data1=a0, data2=t0, op=nl.subtract) nisa.tensor_tensor(dst=r_hi, data1=a1, data2=t1, op=nl.add) # Per-head copies into the head-major qa/ka layout. A single strided DMA spanning several # 32-partition blocks matches in nki.simulate but copies only the first block on trn2, so # engine copies are used here. for bq in range(quad): h = j * quad + bq rr = bq * half nisa.tensor_copy(dst=dst_t[0:half, h, :], src=r_lo[rr:rr + half, :], engine=nisa.engine.scalar) nisa.tensor_copy(dst=dst_t[half:D, h, :], src=r_hi[rr:rr + half, :], engine=nisa.engine.unknown) # ---------------- V projection (token-major) + bias -> va ---------------- va = nl.ndarray((P, ntw, NH, D + 1), dtype=dt, buffer=nl.sbuf) nisa.memset(dst=va[:, :, :, D:D + 1], value=1.0, engine=nisa.engine.gpsimd) for t in range(ntw): for ko in range(n_ko): k0 = ko * FMAX nk = min(FMAX, H - k0) hv = nk // D pv = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) for kt in nl.affine_range(n_k): nisa.nc_matmul(dst=pv[:, 0:nk], stationary=xT[:, kt, t * P:(t + 1) * P], moving=wv[:, kt, k0:k0 + nk]) h0 = k0 // D nisa.tensor_tensor(dst=va[:, t, h0:h0 + hv, 0:D], data1=pv[:, 0:nk], data2=bv[:, k0:k0 + nk], op=nl.add) # ---------------- attention + o_proj + residual + LN stats, per query tile ---------------- nq = ntw if QS == 1: nq = ntw // n_prog qbase = pid * nq # SYMBOLIC: index arithmetic only xbq = nl.ndarray((P, nq, H), dtype=nl.float32, buffer=nl.sbuf) for i in range(nq): xq = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf) nisa.dma_copy(dst=xq, src=x_hbm[nl.ds(r0 + (qbase + i) * P, P), 0:H]) nisa.tensor_tensor(dst=xbq[:, i, :], data1=xq, data2=bo, op=nl.add, engine=nisa.engine.gpsimd) ng = nq if ng > LN_GROUP: ng = LN_GROUP s_all = nl.ndarray((P, ng, H), dtype=nl.float32, buffer=nl.sbuf) mv_all = nl.ndarray((P, ng, 2), dtype=nl.float32, buffer=nl.sbuf) for qi in range(nq): gi_ = qi % ng # slot in the current LayerNorm group sp = qi // nt qt = qi % nt c0 = sp * L # column offset of this sequence in qa/ka t_off = sp * nt # token-tile offset of this sequence in va / xb qcol = c0 + qt * P # query column start in qa if QS == 1: qcol = (qbase + qi) * P xres = xbq[:, qi, :] elif LONG == 1: xres = nl.ndarray((P, H), dtype=nl.float32, buffer=nl.sbuf) nisa.tensor_tensor(dst=xres, data1=xr[:, t_off + qt, :], data2=bo, op=nl.add, engine=nisa.engine.gpsimd) else: xres = xb[:, t_off + qt, :] o_sb = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf) for gg in range(NG): S = nl.ndarray((P, G, L), dtype=nl.float32, buffer=nl.psum) for hl in range(G): h = gg * G + hl if L <= FMAX: nisa.nc_matmul(dst=S[:, hl, :], stationary=qa[0:D + 1, h, nl.ds(qcol, P)], moving=ka[0:D + 1, h, c0:c0 + L]) else: for k0c in range(0, L, FMAX): kw = min(FMAX, L - k0c) nisa.nc_matmul(dst=S[:, hl, k0c:k0c + kw], stationary=qa[0:D + 1, h, nl.ds(qcol, P)], moving=ka[0:D + 1, h, c0 + k0c:c0 + k0c + kw]) negmax = nl.ndarray((P, G), dtype=nl.float32, buffer=nl.sbuf) nisa.tensor_reduce(dst=negmax, op=nl.maximum, data=S, axis=2, negate=True) Pm = nl.ndarray((P, G, L), dtype=dt, buffer=nl.sbuf) for hl in range(G): nisa.activation(dst=Pm[:, hl, :], op=nl.exp, data=S[:, hl, :], bias=negmax[:, hl:hl + 1]) PT = nl.ndarray((P, G, nt, P), dtype=dt, buffer=nl.sbuf) nisa.dma_transpose(dst=PT.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0), src=Pm.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0)) O = nl.ndarray((P, G, D + 1), dtype=nl.float32, buffer=nl.psum) for hl in range(G): h = gg * G + hl for kt in nl.affine_range(nt): nisa.nc_matmul(dst=O[:, hl, :], stationary=PT[:, hl, kt, :], moving=va[:, t_off + kt, h, :]) rs = nl.ndarray((P, G, 1), dtype=nl.float32, buffer=nl.sbuf) nisa.reciprocal(dst=rs, data=O[:, :, D:D + 1]) o3 = o_sb[:, gg * G * D:(gg + 1) * G * D].reshape((P, G, D)) nisa.tensor_tensor(dst=o3, data1=O[:, :, 0:D], data2=rs.ap(pattern=[[G, P], [1, G], [0, D]], offset=0), op=nl.multiply) # o^T for the output projection oTp = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.psum) for kt in range(n_k): nisa.nc_transpose(dst=oTp[:, kt, :], data=o_sb[:, kt * P:(kt + 1) * P], engine=nisa.engine.tensor) oT = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.sbuf) nisa.tensor_copy(dst=oT, src=oTp, engine=nisa.engine.scalar) s_sb = s_all[:, gi_, :] for ko in range(n_ko): k0 = ko * FMAX nk = min(FMAX, H - k0) po = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum) for kt in nl.affine_range(n_k): nisa.nc_matmul(dst=po[:, 0:nk], stationary=oT[:, kt, :], moving=wo[:, kt, k0:k0 + nk]) nisa.tensor_tensor(dst=s_sb[:, k0:k0 + nk], data1=po[:, 0:nk], data2=xres[:, k0:k0 + nk], op=nl.add) stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf) for c in range(n_ch): nisa.bn_stats(dst=stats[:, c * 6:(c + 1) * 6], data=s_sb[:, c * csz:(c + 1) * csz]) nisa.bn_aggr(dst=mv_all[:, gi_, :], data=stats) # ---------------- deferred LayerNorm epilogue (once per group of query tiles) ---------------- if gi_ == ng - 1 or qi == nq - 1: g0 = qi - gi_ # first query tile of this group gn = gi_ + 1 rstd_all = nl.ndarray((P, ng, 1), dtype=nl.float32, buffer=nl.sbuf) nisa.activation(dst=rstd_all[:, 0:gn, :], op=nl.rsqrt, data=mv_all[:, 0:gn, 1:2], bias=eps_sb) for qj in range(gn): yr = r0 + (g0 + qj) * P if QS == 1: yr = r0 + (qbase + g0 + qj) * P _ln_out(s_all[:, qj, :], mv_all[:, qj, 0:1], rstd_all[:, qj, :], ga, be, y_hbm, yr, P, H, dt) return y_hbm