Download build/torch-neuron/attn_block_pack.py from jburtoft/gte-encoder-neuron-kernels: direct link, hf CLI and curl.
- Browser
- Download file 1.79 kB
-
https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/attn_block_pack.py
- Command line
-
hf download hf://jburtoft/gte-encoder-neuron-kernels/build/torch-neuron/attn_block_pack.py
-
curl -L -o attn_block_pack.py https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/attn_block_pack.py
1.79 kB
| """Host-side weight packing for attn_block_kernel (pure NumPy; no torch or NKI import).""" | |
| import numpy as np | |
| P = 128 | |
| D = 64 | |
| HALF = 32 | |
| QUAD = P // HALF | |
| def lane_perm(NH): | |
| """perm[g*128 + l] = feature index (within one projection) for lane group g = j*2 + par.""" | |
| NQ = NH // QUAD | |
| perm = np.zeros(NH * D, np.int64) | |
| for j in range(NQ): | |
| for par in range(2): | |
| g = j * 2 + par | |
| for bq in range(QUAD): | |
| h = j * QUAD + bq | |
| for i in range(HALF): | |
| perm[g * P + bq * HALF + i] = h * D + par * HALF + i | |
| return perm | |
| def pack(wqkv, bqkv, wo, bo, g, b, cos, sin, H, scale): | |
| """wqkv [3H, H], bqkv [3H] (qkv_proj, rows q|k|v), wo [H, H], bo [H] (o_proj), g/b [H] (attn_ln), | |
| cos/sin [L, D] (RoPE table rows for positions 0..L-1), scale = 1/sqrt(head_dim), folded into Wq/bq. | |
| Returns a dict of kernel inputs as fp32 NumPy arrays (cast wqk / wv / wo to the model dtype).""" | |
| NH = H // D | |
| perm = lane_perm(NH) | |
| wq, wk, wv = wqkv[:H], wqkv[H:2 * H], wqkv[2 * H:] | |
| bq, bk, bv = bqkv[:H], bqkv[H:2 * H], bqkv[2 * H:] | |
| wq_p, bq_p = wq[perm] * scale, bq[perm] * scale | |
| wk_p, bk_p = wk[perm], bk[perm] | |
| wqk = np.concatenate([wq_p, wk_p], 0).T.copy() # [H, 2H] | |
| bqk = np.concatenate([bq_p, bk_p]).reshape(-1, P).T.copy() # [128, 2H/128] | |
| lane_freq = np.arange(P) % HALF # lane l -> freq index | |
| cs = np.stack([cos[:, lane_freq].T, sin[:, lane_freq].T]).astype(np.float32) # [2, 128, L] | |
| bc = lambda v: np.tile(v.astype(np.float32), (P, 1)) | |
| return dict(wqk=wqk, bqk=bqk.astype(np.float32), wv=wv.T.copy(), bv=bc(bv), wo=wo.T.copy(), bo=bc(bo), | |
| g=bc(g), b=bc(b), cs=cs) | |