File size: 1,795 Bytes
25e8424 fb58c6c 25e8424 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """NKI kernels for NewBERT (Alibaba-NLP "new-impl") encoders on AWS Trainium2, PyTorch Native.
Covers the gte family built on the NewBERT modeling code (post-LN, rotate-half NTK RoPE, gated GELU MLP),
for example gte-multilingual-reranker-base, gte-multilingual-base, gte-base-en-v1.5, gte-large-en-v1.5.
Entry point:
patch_model(model) -- in-place patch of every layer + the embedding LayerNorm (inference only)
Ops (torch custom ops, namespace gte_enc::):
res_ln, ln -- (residual +) LayerNorm with bn_stats / bn_aggr
gated_mlp_ln -- gated GELU MLP + residual + LayerNorm
attn_block -- QKV + RoPE + masked softmax attention + o_proj + residual + LayerNorm, L <= 512
Raw NKI kernels (for nki.simulate or other frameworks): res_ln_kernel, gated_mlp_ln_kernel,
gated_mlp_ln_stream_kernel (weights too large for SBUF), attn_block_kernel, plus the host-side packer pack_attn_block.
Set NEURON_LOGICAL_NC_CONFIG before importing; LNC is read once at import.
"""
from .attn_block_kernel import attn_block_kernel
from .attn_block_pack import lane_perm
from .attn_block_pack import pack as pack_attn_block
from .gated_mlp_kernel import gated_mlp_ln_kernel, gated_mlp_ln_stream_kernel, mlp_weights_fit
from .res_ln_kernel import res_ln_kernel
from .torch_ops import (
ATTN_LENGTHS,
CTE_MIN_LEN,
LNC,
NkiAttnBlock,
NkiLN,
NkiMlpLN,
attn_block,
gated_mlp_ln,
ln,
patch_model,
res_ln,
)
__version__ = "1.1.0"
__all__ = [
"patch_model",
"res_ln", "ln", "gated_mlp_ln", "attn_block",
"NkiLN", "NkiMlpLN", "NkiAttnBlock",
"res_ln_kernel", "gated_mlp_ln_kernel", "gated_mlp_ln_stream_kernel", "mlp_weights_fit", "attn_block_kernel", "pack_attn_block", "lane_perm",
"LNC", "ATTN_LENGTHS", "CTE_MIN_LEN",
]
|