jburtoft's picture
v1.1.0: fused attention block for L = 768 / 1024
fb58c6c
Raw History Blame Contribute Delete
18.4 kB
"""Fused NewBERT attention block: y = attn_ln(x + o_proj(attention(rope(q), rope(k), v)))
Replaces qkv_proj + RoPE + masked softmax attention + o_proj + residual + attn_ln of one layer.
Inputs (bf16 unless noted), all weights host-packed once (see attn_block_pack.pack):
x [B*L, H] layer input, token-major (head_dim D = 64, NH = H / 64 heads, NH % 4 == 0)
wqk [H, 2*H] Q and K projection, transposed, LANE-PERMUTED, Q pre-scaled by 1/sqrt(D)
bqk [128, 2*H/128] Q/K bias, same permutation, one column per 128-lane group (fp32)
wv [H, H] V projection transposed; bv [128, H] broadcast (fp32)
wo [H, H] o_proj transposed; bo [128, H] broadcast (fp32)
g, b [128, H] attn_ln weight / bias broadcast (fp32)
cs [2, 128, L] RoPE cos / sin per lane (fp32); lane l uses frequency l % 32
kb [B, L] additive key bias (0 valid, -30000 padded), in already-scaled score units
Output y [B*L, H].
Lane permutation (RoPE without transposes): 128-lane group (j, par) holds lanes b*32+i = feature
(4j+b)*D + par*32 + i of heads 4j..4j+3. Group par=0 holds the low halves, par=1 the high halves, so
rotate-half RoPE is elementwise between the two groups:
lo' = lo*cos - hi*sin ; hi' = hi*cos + lo*sin
and the results are written per head into qa/ka[0:64, h, :] (rows 0-31 lo', 32-63 hi'). Q and K
use the same order, so the dot product is unchanged.
Per projection pass (sequences split across LNC programs; up to 4 sequences packed per pass so the
projection moving width is 512):
x rows -> x^T (DMA transpose); Q/K projections (feature-major) + bias + RoPE -> qa/ka [65, NH, L]
with a ones row (qa) / key-bias row (ka) at partition 64, so one K=65 matmul gives S = QK^T + kb.
V projection (token-major) + bias -> va [128, L/128, NH, 65] with a ones column, so the PV matmul
also produces the softmax row sums.
Per 128-query tile: S -> exp(S - max) -> P^T (one DMA transpose per head group) -> PV
-> normalise -> o^T (PE transposes) -> o_proj -> + bo + x -> bn_stats.
The LayerNorm epilogue (rsqrt + affine) is deferred until all query tiles of the pass are done, so
the Scalar engine does not alternate between the exp and rsqrt activation tables.
Query split: at LNC > 1 with a batch not divisible by the core count, every core computes K/V for
every sequence but only its own share of the query tiles (needs (L/128) % LNC == 0). No cross-core
reduction is needed.
L must be a multiple of 128 and <= 1024. Above 512 keys (the matmul moving-size limit) the projection and QK^T
matmuls are issued in 512-column chunks; one head's score row (<= 1024 fp32 = 2 PSUM banks) still fits in PSUM, so
the softmax stays exact (no running max). Right padding only. Inference only.
"""
import nki
import nki.isa as nisa
import nki.language as nl
P = 128
FMAX = 512
LMAX = 1024
PSUM_BANK = 2048 # bytes per partition per PSUM bank
def _cdiv(a, b):
return (a + b - 1) // b
def _group(NH, L, D=64):
"""Heads per softmax group: S [128, G, L] fp32 spans at most 2 banks (G * L <= 1024), and the PV
output O [128, G, D + 1] fp32 must stay inside one bank (a matmul cannot write across banks)."""
best = 1
for g in range(1, NH + 1):
if NH % g == 0 and g * L <= 1024 and g * (D + 1) * 4 <= PSUM_BANK:
best = g
return best
def _ln_out(s_sb, mean, rstd, ga, be, y_hbm, row0, P_, H, dt):
u_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf)
nisa.scalar_tensor_tensor(dst=u_sb, data=s_sb, op0=nl.subtract, operand0=mean,
op1=nl.multiply, operand1=ga)
v_sb = nl.ndarray((P_, H), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=v_sb, op=nl.copy, data=u_sb, scale=rstd)
nisa.dma_compute(dst=y_hbm[nl.ds(row0, P_), 0:H], srcs=[v_sb, be], reduce_op=nl.add)
@nki.jit
def attn_block_kernel(x_hbm, wqk_hbm, bqk_hbm, wv_hbm, bv_hbm, wo_hbm, bo_hbm, g_hbm, b_hbm, cs_hbm, kb_hbm,
eps=1e-12):
R, H = x_hbm.shape
B, L = kb_hbm.shape
D = 64
NH = H // D
half = D // 2
quad = P // half # heads per 128-lane group (4)
NQ = NH // quad # lane groups per parity
assert NH * D == H
assert R == B * L
assert L % P == 0
assert L <= LMAX
assert H % P == 0
assert NH % quad == 0
n_prog = nl.num_programs(0)
pid = nl.program_id(0)
nt0 = L // P
QS = 0
if n_prog > 1 and B % n_prog != 0:
QS = 1
if QS == 1:
assert nt0 % n_prog == 0
Bp = B
else:
assert B % n_prog == 0
Bp = B // n_prog
n_k = H // P # contraction tiles over H
nt = L // P # token tiles per sequence
G = _group(NH, L)
NG = NH // G
n_ko = _cdiv(H, FMAX)
fmax = nl.tile_size.bn_stats_fmax
n_ch = _cdiv(H, fmax)
csz = H // n_ch
dt = x_hbm.dtype
y_hbm = nl.ndarray((R, H), dtype=dt, buffer=nl.shared_hbm)
# ---------------- resident weights / constants ----------------
wqk = nl.ndarray((P, n_k, 2 * H), dtype=dt, buffer=nl.sbuf)
wv = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf)
wo = nl.ndarray((P, n_k, H), dtype=dt, buffer=nl.sbuf)
for kt in range(n_k):
nisa.dma_copy(dst=wqk[:, kt, :], src=wqk_hbm[kt * P:(kt + 1) * P, 0:2 * H])
nisa.dma_copy(dst=wv[:, kt, :], src=wv_hbm[kt * P:(kt + 1) * P, 0:H])
nisa.dma_copy(dst=wo[:, kt, :], src=wo_hbm[kt * P:(kt + 1) * P, 0:H])
bqk = nl.ndarray((P, 2 * H // P), dtype=nl.float32, buffer=nl.sbuf)
nisa.dma_copy(dst=bqk, src=bqk_hbm[0:P, 0:2 * H // P])
bv = nl.ndarray((P, H), dtype=bv_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=bv, src=bv_hbm[0:P, 0:H])
bo = nl.ndarray((P, H), dtype=bo_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=bo, src=bo_hbm[0:P, 0:H])
ga = nl.ndarray((P, H), dtype=g_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=ga, src=g_hbm[0:P, 0:H])
be = nl.ndarray((P, H), dtype=b_hbm.dtype, buffer=nl.sbuf)
nisa.dma_copy(dst=be, src=b_hbm[0:P, 0:H])
cos_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf)
sin_t = nl.ndarray((P, L), dtype=nl.float32, buffer=nl.sbuf)
nisa.dma_copy(dst=cos_t, src=cs_hbm[0, 0:P, 0:L])
nisa.dma_copy(dst=sin_t, src=cs_hbm[1, 0:P, 0:L])
eps_sb = nl.ndarray((P, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.memset(dst=eps_sb, value=eps)
# sequences per projection pass: 4 at L=128, 2 at L=256, 1 at L>=384 (or if Bp does not divide)
SP = 1
if QS == 0:
SP = FMAX // L
if SP > 4:
SP = 4
if SP < 1:
SP = 1
if Bp % SP != 0:
SP = 1
LW = L * SP # projection moving width
# Long sequences (L > 512): x + bo is formed per query tile instead of for the whole sequence, and the deferred
# LayerNorm epilogue runs per group of LN_GROUP query tiles, to bound SBUF use. For L <= 512 a pass has at most
# 4 query tiles, so the epilogue is deferred over the whole pass as before.
LONG = 0
if L > FMAX:
LONG = 1
LN_GROUP = 4
cosw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
sinw = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
for sp in range(SP):
nisa.tensor_copy(dst=cosw[:, sp * L:(sp + 1) * L], src=cos_t)
nisa.tensor_copy(dst=sinw[:, sp * L:(sp + 1) * L], src=sin_t)
for bi in nl.affine_range(Bp // SP):
bseq = pid * Bp + bi * SP
if QS == 1:
bseq = bi
r0 = bseq * L
ntw = nt * SP # token tiles in this pass
# ---------------- x rows, x + bo, x^T ----------------
xr = nl.ndarray((P, ntw, H), dtype=dt, buffer=nl.sbuf)
if LONG == 0:
xb = nl.ndarray((P, ntw, H), dtype=nl.float32, buffer=nl.sbuf)
xT = nl.ndarray((P, n_k, LW), dtype=dt, buffer=nl.sbuf)
for t in range(ntw):
nisa.dma_copy(dst=xr[:, t, :], src=x_hbm[nl.ds(r0 + t * P, P), 0:H])
if LONG == 0:
nisa.tensor_tensor(dst=xb[:, t, :], data1=xr[:, t, :], data2=bo, op=nl.add,
engine=nisa.engine.gpsimd)
for kt in range(n_k):
nisa.dma_transpose(dst=xT[:, kt, t * P:(t + 1) * P], src=xr[:, t, kt * P:(kt + 1) * P])
# ---------------- Q/K projection + bias + RoPE -> qa / ka ----------------
qa = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf)
ka = nl.ndarray((D + 1, NH, LW), dtype=dt, buffer=nl.sbuf)
nisa.memset(dst=qa[D:D + 1, :, :], value=1.0, engine=nisa.engine.gpsimd)
for sp in range(SP):
nisa.dma_copy(dst=ka[D:D + 1, :, sp * L:(sp + 1) * L],
src=kb_hbm.ap(pattern=[[L, 1], [0, NH], [1, L]], offset=(bseq + sp) * L))
for qk in range(2):
dst_t = qa
if qk == 1:
dst_t = ka
for j in range(NQ):
pr = []
for par in range(2):
gi = (qk * NQ + j) * 2 + par
ps = nl.ndarray((P, LW), dtype=nl.float32, buffer=nl.psum)
if LW <= FMAX:
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=ps, stationary=wqk[:, kt, gi * P:(gi + 1) * P], moving=xT[:, kt, :])
else:
for c0w in range(0, LW, FMAX):
cw = min(FMAX, LW - c0w)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=ps[:, c0w:c0w + cw], stationary=wqk[:, kt, gi * P:(gi + 1) * P],
moving=xT[:, kt, c0w:c0w + cw])
pt = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
nisa.activation(dst=pt, op=nl.copy, data=ps, bias=bqk[:, gi:gi + 1])
pr.append(pt)
lo = pr[0]
hi = pr[1]
a0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
t0 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
a1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
t1 = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
nisa.tensor_tensor(dst=a0, data1=lo, data2=cosw, op=nl.multiply)
nisa.tensor_tensor(dst=t0, data1=hi, data2=sinw, op=nl.multiply, engine=nisa.engine.vector)
nisa.tensor_tensor(dst=a1, data1=hi, data2=cosw, op=nl.multiply)
nisa.tensor_tensor(dst=t1, data1=lo, data2=sinw, op=nl.multiply, engine=nisa.engine.vector)
r_lo = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
r_hi = nl.ndarray((P, LW), dtype=dt, buffer=nl.sbuf)
nisa.tensor_tensor(dst=r_lo, data1=a0, data2=t0, op=nl.subtract)
nisa.tensor_tensor(dst=r_hi, data1=a1, data2=t1, op=nl.add)
# Per-head copies into the head-major qa/ka layout. A single strided DMA spanning several
# 32-partition blocks matches in nki.simulate but copies only the first block on trn2, so
# engine copies are used here.
for bq in range(quad):
h = j * quad + bq
rr = bq * half
nisa.tensor_copy(dst=dst_t[0:half, h, :], src=r_lo[rr:rr + half, :], engine=nisa.engine.scalar)
nisa.tensor_copy(dst=dst_t[half:D, h, :], src=r_hi[rr:rr + half, :], engine=nisa.engine.unknown)
# ---------------- V projection (token-major) + bias -> va ----------------
va = nl.ndarray((P, ntw, NH, D + 1), dtype=dt, buffer=nl.sbuf)
nisa.memset(dst=va[:, :, :, D:D + 1], value=1.0, engine=nisa.engine.gpsimd)
for t in range(ntw):
for ko in range(n_ko):
k0 = ko * FMAX
nk = min(FMAX, H - k0)
hv = nk // D
pv = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=pv[:, 0:nk], stationary=xT[:, kt, t * P:(t + 1) * P],
moving=wv[:, kt, k0:k0 + nk])
h0 = k0 // D
nisa.tensor_tensor(dst=va[:, t, h0:h0 + hv, 0:D], data1=pv[:, 0:nk], data2=bv[:, k0:k0 + nk],
op=nl.add)
# ---------------- attention + o_proj + residual + LN stats, per query tile ----------------
nq = ntw
if QS == 1:
nq = ntw // n_prog
qbase = pid * nq # SYMBOLIC: index arithmetic only
xbq = nl.ndarray((P, nq, H), dtype=nl.float32, buffer=nl.sbuf)
for i in range(nq):
xq = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf)
nisa.dma_copy(dst=xq, src=x_hbm[nl.ds(r0 + (qbase + i) * P, P), 0:H])
nisa.tensor_tensor(dst=xbq[:, i, :], data1=xq, data2=bo, op=nl.add, engine=nisa.engine.gpsimd)
ng = nq
if ng > LN_GROUP:
ng = LN_GROUP
s_all = nl.ndarray((P, ng, H), dtype=nl.float32, buffer=nl.sbuf)
mv_all = nl.ndarray((P, ng, 2), dtype=nl.float32, buffer=nl.sbuf)
for qi in range(nq):
gi_ = qi % ng # slot in the current LayerNorm group
sp = qi // nt
qt = qi % nt
c0 = sp * L # column offset of this sequence in qa/ka
t_off = sp * nt # token-tile offset of this sequence in va / xb
qcol = c0 + qt * P # query column start in qa
if QS == 1:
qcol = (qbase + qi) * P
xres = xbq[:, qi, :]
elif LONG == 1:
xres = nl.ndarray((P, H), dtype=nl.float32, buffer=nl.sbuf)
nisa.tensor_tensor(dst=xres, data1=xr[:, t_off + qt, :], data2=bo, op=nl.add,
engine=nisa.engine.gpsimd)
else:
xres = xb[:, t_off + qt, :]
o_sb = nl.ndarray((P, H), dtype=dt, buffer=nl.sbuf)
for gg in range(NG):
S = nl.ndarray((P, G, L), dtype=nl.float32, buffer=nl.psum)
for hl in range(G):
h = gg * G + hl
if L <= FMAX:
nisa.nc_matmul(dst=S[:, hl, :], stationary=qa[0:D + 1, h, nl.ds(qcol, P)],
moving=ka[0:D + 1, h, c0:c0 + L])
else:
for k0c in range(0, L, FMAX):
kw = min(FMAX, L - k0c)
nisa.nc_matmul(dst=S[:, hl, k0c:k0c + kw], stationary=qa[0:D + 1, h, nl.ds(qcol, P)],
moving=ka[0:D + 1, h, c0 + k0c:c0 + k0c + kw])
negmax = nl.ndarray((P, G), dtype=nl.float32, buffer=nl.sbuf)
nisa.tensor_reduce(dst=negmax, op=nl.maximum, data=S, axis=2, negate=True)
Pm = nl.ndarray((P, G, L), dtype=dt, buffer=nl.sbuf)
for hl in range(G):
nisa.activation(dst=Pm[:, hl, :], op=nl.exp, data=S[:, hl, :], bias=negmax[:, hl:hl + 1])
PT = nl.ndarray((P, G, nt, P), dtype=dt, buffer=nl.sbuf)
nisa.dma_transpose(dst=PT.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0),
src=Pm.ap(pattern=[[G * L, P], [1, 1], [P, G * nt], [1, P]], offset=0))
O = nl.ndarray((P, G, D + 1), dtype=nl.float32, buffer=nl.psum)
for hl in range(G):
h = gg * G + hl
for kt in nl.affine_range(nt):
nisa.nc_matmul(dst=O[:, hl, :], stationary=PT[:, hl, kt, :], moving=va[:, t_off + kt, h, :])
rs = nl.ndarray((P, G, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.reciprocal(dst=rs, data=O[:, :, D:D + 1])
o3 = o_sb[:, gg * G * D:(gg + 1) * G * D].reshape((P, G, D))
nisa.tensor_tensor(dst=o3, data1=O[:, :, 0:D],
data2=rs.ap(pattern=[[G, P], [1, G], [0, D]], offset=0), op=nl.multiply)
# o^T for the output projection
oTp = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.psum)
for kt in range(n_k):
nisa.nc_transpose(dst=oTp[:, kt, :], data=o_sb[:, kt * P:(kt + 1) * P], engine=nisa.engine.tensor)
oT = nl.ndarray((P, n_k, P), dtype=dt, buffer=nl.sbuf)
nisa.tensor_copy(dst=oT, src=oTp, engine=nisa.engine.scalar)
s_sb = s_all[:, gi_, :]
for ko in range(n_ko):
k0 = ko * FMAX
nk = min(FMAX, H - k0)
po = nl.ndarray((P, FMAX), dtype=nl.float32, buffer=nl.psum)
for kt in nl.affine_range(n_k):
nisa.nc_matmul(dst=po[:, 0:nk], stationary=oT[:, kt, :], moving=wo[:, kt, k0:k0 + nk])
nisa.tensor_tensor(dst=s_sb[:, k0:k0 + nk], data1=po[:, 0:nk], data2=xres[:, k0:k0 + nk],
op=nl.add)
stats = nl.ndarray((P, 6 * n_ch), dtype=nl.float32, buffer=nl.sbuf)
for c in range(n_ch):
nisa.bn_stats(dst=stats[:, c * 6:(c + 1) * 6], data=s_sb[:, c * csz:(c + 1) * csz])
nisa.bn_aggr(dst=mv_all[:, gi_, :], data=stats)
# ---------------- deferred LayerNorm epilogue (once per group of query tiles) ----------------
if gi_ == ng - 1 or qi == nq - 1:
g0 = qi - gi_ # first query tile of this group
gn = gi_ + 1
rstd_all = nl.ndarray((P, ng, 1), dtype=nl.float32, buffer=nl.sbuf)
nisa.activation(dst=rstd_all[:, 0:gn, :], op=nl.rsqrt, data=mv_all[:, 0:gn, 1:2], bias=eps_sb)
for qj in range(gn):
yr = r0 + (g0 + qj) * P
if QS == 1:
yr = r0 + (qbase + g0 + qj) * P
_ln_out(s_all[:, qj, :], mv_all[:, qj, 0:1], rstd_all[:, qj, :], ga, be, y_hbm,
yr, P, H, dt)
return y_hbm