Download build/torch-neuron/torch_ops.py from jburtoft/gte-encoder-neuron-kernels: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/torch_ops.py
- Command line
-
hf download hf://jburtoft/gte-encoder-neuron-kernels/build/torch-neuron/torch_ops.py
-
curl -L -o torch_ops.py https://huggingface.co/kernels/jburtoft/gte-encoder-neuron-kernels/resolve/main/build/torch-neuron/torch_ops.py
10.7 kB
| """PyTorch Native wrappers and model patching for NewBERT encoders (torch.compile(backend="neuron")). | |
| Inference only. LNC is read once, at import, from NEURON_LOGICAL_NC_CONFIG (default 2, the trn2 | |
| default); set it before importing this package. Shapes the kernels do not support fall back to the | |
| stock modules at trace time (shapes are static under torch.compile), so patching is always safe. | |
| """ | |
| import math | |
| import os | |
| import types | |
| import torch | |
| from torch_neuronx import nki_op | |
| from .attn_block_kernel import attn_block_kernel | |
| from .attn_block_pack import pack as _pack_attn | |
| from .gated_mlp_kernel import gated_mlp_ln_kernel, gated_mlp_ln_stream_kernel, mlp_weights_fit | |
| from .res_ln_kernel import res_ln_kernel | |
| LNC = int(os.environ.get("NEURON_LOGICAL_NC_CONFIG", "2")) | |
| # Fused attention block: supported sequence lengths (multiples of 128, <= 1024). | |
| ATTN_LENGTHS = (128, 256, 384, 512, 768, 1024) | |
| # Lengths not in ATTN_LENGTHS from this length up use nkilib attention_cte (key padding as sequence bounds) | |
| # for the attention core; below it the compiler's own attention is faster. | |
| CTE_MIN_LEN = 768 | |
| # Key-bias clamp: padded keys get this additive bias (exp underflows to 0 in fp32 and bf16). | |
| MASK_BIAS = -30000.0 | |
| # ---------------------------------------------------------------- ops | |
| def res_ln(x: torch.Tensor, r: torch.Tensor, g: torch.Tensor, b: torch.Tensor, eps: float) -> torch.Tensor: | |
| """LayerNorm(x + r) * g + b. x, r [R, H]; g, b [128, H] (pre-broadcast). R % (128 * LNC) == 0.""" | |
| return res_ln_kernel[LNC](x, r, g, b, eps, True) | |
| def ln(x: torch.Tensor, g: torch.Tensor, b: torch.Tensor, eps: float) -> torch.Tensor: | |
| """LayerNorm(x) * g + b (no residual). Same constraints as res_ln.""" | |
| return res_ln_kernel[LNC](x, x, g, b, eps, False) | |
| def gated_mlp_ln(x: torch.Tensor, wupT: torch.Tensor, wgT: torch.Tensor, wdT: torch.Tensor, | |
| bd: torch.Tensor, g: torch.Tensor, b: torch.Tensor, eps: float) -> torch.Tensor: | |
| """LayerNorm(x + (gelu_tanh(x Wg^T) * x Wu^T) Wd^T + bd) * g + b. R % (128 * LNC) == 0. | |
| Weights stay resident in SBUF when they fit (hidden 768 / intermediate 3072), else they are streamed.""" | |
| K, I = wupT.shape | |
| if mlp_weights_fit(K, I, wupT.element_size()): | |
| return gated_mlp_ln_kernel[LNC](x, wupT, wgT, wdT, bd, g, b, eps) | |
| return gated_mlp_ln_stream_kernel[LNC](x, wupT, wgT, wdT, bd, g, b, eps) | |
| def attn_block(x: torch.Tensor, wqk: torch.Tensor, bqk: torch.Tensor, wv: torch.Tensor, bv: torch.Tensor, | |
| wo: torch.Tensor, bo: torch.Tensor, g: torch.Tensor, b: torch.Tensor, cs: torch.Tensor, | |
| kb: torch.Tensor, eps: float) -> torch.Tensor: | |
| """attn_ln(x + o_proj(attention(rope(q), rope(k), v))) for packed weights (attn_block_pack.pack).""" | |
| return attn_block_kernel[LNC](x, wqk, bqk, wv, bv, wo, bo, g, b, cs, kb, eps) | |
| def _bcast(v, H): | |
| return v.detach().float().reshape(1, H).expand(128, H).contiguous() | |
| # ---------------------------------------------------------------- modules | |
| class NkiLN(torch.nn.Module): | |
| """Drop-in for nn.LayerNorm(H) over the last dim, with an optional fused residual.""" | |
| def __init__(self, ln_mod: torch.nn.LayerNorm): | |
| super().__init__() | |
| H = ln_mod.normalized_shape[0] | |
| self.H, self.eps = H, float(ln_mod.eps) | |
| self.register_buffer("g", _bcast(ln_mod.weight, H)) | |
| self.register_buffer("b", _bcast(ln_mod.bias, H)) | |
| def forward(self, x, residual=None): | |
| shp = x.shape | |
| x2 = x.reshape(-1, self.H) | |
| if x2.shape[0] % (128 * LNC) != 0: | |
| s = x if residual is None else x + residual | |
| return torch.nn.functional.layer_norm(s, (self.H,), self.g[0].to(s.dtype), | |
| self.b[0].to(s.dtype), self.eps) | |
| if residual is None: | |
| y = ln(x2, self.g, self.b, self.eps) | |
| else: | |
| y = res_ln(x2, residual.reshape(-1, self.H), self.g, self.b, self.eps) | |
| return y.reshape(shp) | |
| class NkiMlpLN(torch.nn.Module): | |
| """Replaces NewGatedMLP + mlp_ln: y = LN(x + down(gelu_tanh(gate(x)) * up(x))).""" | |
| def __init__(self, mlp, ln_mod): | |
| super().__init__() | |
| I = mlp.intermediate_size | |
| w = mlp.up_gate_proj.weight.detach() # [2I, H]: rows [0:I] up, [I:2I] gate | |
| self.register_buffer("wupT", w[:I].t().contiguous()) | |
| self.register_buffer("wgT", w[I:].t().contiguous()) | |
| self.register_buffer("wdT", mlp.down_proj.weight.detach().t().contiguous()) | |
| H = self.wupT.shape[0] | |
| self.register_buffer("bd", _bcast(mlp.down_proj.bias, H)) | |
| self.register_buffer("g", _bcast(ln_mod.weight, H)) | |
| self.register_buffer("b", _bcast(ln_mod.bias, H)) | |
| self.eps = float(ln_mod.eps) | |
| self.H = H | |
| self._mlp, self._ln = [mlp], [ln_mod] # stock fallback (not registered as children) | |
| def forward(self, x): | |
| shp = x.shape | |
| x2 = x.reshape(-1, self.H) | |
| if x2.shape[0] % (128 * LNC) != 0: | |
| return self._ln[0](x + self._mlp[0](x)) | |
| return gated_mlp_ln(x2, self.wupT, self.wgT, self.wdT, self.bd, self.g, self.b, self.eps).reshape(shp) | |
| class NkiAttnBlock(torch.nn.Module): | |
| """attn_ln(x + attention(x)) for one NewLayer; RoPE tables packed per supported length.""" | |
| def __init__(self, attn, ln_mod, rotary, lengths=ATTN_LENGTHS): | |
| super().__init__() | |
| H = attn.hidden_size | |
| self.H, self.eps = H, float(ln_mod.eps) | |
| sc = 1.0 / math.sqrt(attn.attention_head_size) | |
| f = lambda t: t.detach().float().cpu().numpy() | |
| cos, sin = rotary(torch.zeros(1, dtype=torch.float32), seq_len=max(lengths)) | |
| pk = _pack_attn(f(attn.qkv_proj.weight), f(attn.qkv_proj.bias), f(attn.o_proj.weight), | |
| f(attn.o_proj.bias), f(ln_mod.weight), f(ln_mod.bias), f(cos), f(sin), H, sc) | |
| dt = attn.qkv_proj.weight.dtype | |
| for k in ("wqk", "wv", "wo"): | |
| self.register_buffer(k, torch.from_numpy(pk[k]).to(dt)) | |
| for k in ("bqk", "bv", "bo", "g", "b"): | |
| self.register_buffer(k, torch.from_numpy(pk[k]).float()) | |
| for L in lengths: | |
| self.register_buffer(f"cs{L}", torch.from_numpy(pk["cs"][:, :, :L].copy()).float()) | |
| self.lengths = set(lengths) | |
| self._attn = [attn] | |
| self.fallback_ln = NkiLN(ln_mod) # unsupported shapes: stock (or cte) attention + NKI LN | |
| def forward(self, x, attention_bias, rope_embeds): | |
| B, L, H = x.shape | |
| qsplit_ok = (L // 128) % LNC == 0 | |
| if L not in self.lengths or (B % LNC and not qsplit_ok): | |
| a = self._attn[0](x, attention_bias, rope_embeds)[0] | |
| return self.fallback_ln(a, x) | |
| kb = attention_bias.reshape(B, L).float().clamp(min=MASK_BIAS).to(x.dtype) | |
| y = attn_block(x.reshape(B * L, H), self.wqk, self.bqk, self.wv, self.bv, self.wo, self.bo, | |
| self.g, self.b, getattr(self, f"cs{L}"), kb, self.eps) | |
| return y.reshape(B, L, H) | |
| # ---------------------------------------------------------------- nkilib attention_cte for long sequences | |
| _cte = None | |
| def _cte_attention(self, q, k, v, attention_bias, head_mask): | |
| """q/k/v [B, L, nh, hd]; attention_bias [B, 1, 1, L] additive (0 valid, min for pad, right-padded).""" | |
| global _cte | |
| B, L, nh, hd = q.shape | |
| if L < CTE_MIN_LEN or head_mask is not None or (self.training and self.dropout.p > 0): | |
| return self._stock_attention(q, k, v, attention_bias, head_mask) | |
| if _cte is None: | |
| from torch_neuronx import wrap_nki | |
| from nkilib.core.attention.attention_cte import attention_cte | |
| _cte = wrap_nki(attention_cte) | |
| scale = 1.0 / math.sqrt(hd) | |
| qh = (q * scale).permute(0, 2, 1, 3).reshape(B * nh, L, hd) # bounds path needs scale=1 | |
| kh = k.permute(0, 2, 1, 3).reshape(B * nh, L, hd) | |
| vh = v.permute(0, 2, 1, 3).reshape(B * nh, L, hd) | |
| vl = (attention_bias.reshape(B, L) == 0).sum(-1).to(torch.float32) | |
| bmax = vl.repeat_interleave(nh)[:, None, None].expand(B * nh, L, 1).contiguous() | |
| bmin = torch.zeros_like(bmax) | |
| o = _cte[LNC](qh, kh, vh, scale=1.0, causal_mask=False, tp_q=True, tp_k=True, | |
| bound_min=bmin, bound_max=bmax) | |
| return o.reshape(B, nh, L, hd).permute(0, 2, 1, 3).contiguous(), None | |
| # ---------------------------------------------------------------- patching | |
| def _layer_forward(self, hidden_states, attention_bias, rope_embeds=None, padding_inputs=None, | |
| attention_scale=None, subset_indices=None, head_mask=None, | |
| output_attentions=False, qkv_inputs=None): | |
| hidden_states = self.attn_block(hidden_states, attention_bias, rope_embeds) | |
| hidden_states = self.mlp_block(hidden_states) | |
| return (hidden_states,) | |
| def patch_model(model, long_attention=True, attn_lengths=ATTN_LENGTHS): | |
| """Patch every NewLayer / NewEmbeddings in a NewBERT (Alibaba-NLP "new-impl") model, in place: | |
| * embedding LayerNorm -> NKI LayerNorm | |
| * attention half of each layer -> fused attention block (L in attn_lengths, default ATTN_LENGTHS) | |
| * gated MLP + mlp_ln -> fused gated-MLP + residual + LayerNorm | |
| * attention core at L >= CTE_MIN_LEN -> nkilib attention_cte (if long_attention) | |
| Call after model.to(torch.bfloat16) and before model.to("neuron"). Inference only (eval mode). | |
| Returns the number of layers patched. | |
| """ | |
| assert not model.training, "patch_model is inference only; call model.eval() first" | |
| rot = None | |
| for mod in model.modules(): | |
| if type(mod).__name__ == "NewEmbeddings": | |
| rot = mod.rotary_emb | |
| mod.LayerNorm = NkiLN(mod.LayerNorm) | |
| assert rot is not None, "no NewEmbeddings module found (is this a NewBERT model?)" | |
| if long_attention: | |
| for mod in model.modules(): | |
| if type(mod).__name__ == "NewAttention": | |
| mod._stock_attention = mod._attention | |
| mod._attention = types.MethodType(_cte_attention, mod) | |
| n = 0 | |
| for mod in model.modules(): | |
| if type(mod).__name__ == "NewLayer": | |
| assert "Gated" in type(mod.mlp).__name__, "only the gated (NewGatedMLP) MLP is supported" | |
| mod.mlp_block = NkiMlpLN(mod.mlp, mod.mlp_ln) | |
| mod.attn_block = NkiAttnBlock(mod.attention, mod.attn_ln, rot, lengths=attn_lengths) | |
| mod.forward = types.MethodType(_layer_forward, mod) | |
| n += 1 | |
| return n | |