jburtoft's picture
v1.0.0: NKI kernels for NewBERT (gte) encoders on trn2
25e8424
Raw History Blame Contribute Delete
1.79 kB
"""Host-side weight packing for attn_block_kernel (pure NumPy; no torch or NKI import)."""
import numpy as np
P = 128
D = 64
HALF = 32
QUAD = P // HALF
def lane_perm(NH):
"""perm[g*128 + l] = feature index (within one projection) for lane group g = j*2 + par."""
NQ = NH // QUAD
perm = np.zeros(NH * D, np.int64)
for j in range(NQ):
for par in range(2):
g = j * 2 + par
for bq in range(QUAD):
h = j * QUAD + bq
for i in range(HALF):
perm[g * P + bq * HALF + i] = h * D + par * HALF + i
return perm
def pack(wqkv, bqkv, wo, bo, g, b, cos, sin, H, scale):
"""wqkv [3H, H], bqkv [3H] (qkv_proj, rows q|k|v), wo [H, H], bo [H] (o_proj), g/b [H] (attn_ln),
cos/sin [L, D] (RoPE table rows for positions 0..L-1), scale = 1/sqrt(head_dim), folded into Wq/bq.
Returns a dict of kernel inputs as fp32 NumPy arrays (cast wqk / wv / wo to the model dtype)."""
NH = H // D
perm = lane_perm(NH)
wq, wk, wv = wqkv[:H], wqkv[H:2 * H], wqkv[2 * H:]
bq, bk, bv = bqkv[:H], bqkv[H:2 * H], bqkv[2 * H:]
wq_p, bq_p = wq[perm] * scale, bq[perm] * scale
wk_p, bk_p = wk[perm], bk[perm]
wqk = np.concatenate([wq_p, wk_p], 0).T.copy() # [H, 2H]
bqk = np.concatenate([bq_p, bk_p]).reshape(-1, P).T.copy() # [128, 2H/128]
lane_freq = np.arange(P) % HALF # lane l -> freq index
cs = np.stack([cos[:, lane_freq].T, sin[:, lane_freq].T]).astype(np.float32) # [2, 128, L]
bc = lambda v: np.tile(v.astype(np.float32), (P, 1))
return dict(wqk=wqk, bqk=bqk.astype(np.float32), wv=wv.T.copy(), bv=bc(bv), wo=wo.T.copy(), bo=bc(bo),
g=bc(g), b=bc(b), cs=cs)