nuri-base-embed / encoder.py
gaon12's picture
Release completed Nuri research checkpoint and measured evaluation
36fdf91 verified
Raw History Blame Contribute Delete
5.02 kB
"""Bidirectional input encoder producing the read-only memory X (ARCHITECTURE.md §4)."""
from __future__ import annotations
import torch
from torch import nn
from .config import NuriConfig
from .layers import Attention, RMSNorm, rope_cache, run_checkpointed
from .moe import build_ffn
class RopeTables(nn.Module):
"""Lazily grown cos/sin tables shared by every RoPE user in one stack."""
def __init__(self, head_dim: int, theta: float):
super().__init__()
self.head_dim, self.theta = head_dim, theta
self._tables = None
def forward(self, length: int, device) -> tuple[torch.Tensor, torch.Tensor]:
if self._tables is None or self._tables[0].shape[0] < length or self._tables[0].device != torch.device(device):
self._tables = rope_cache(max(length, 64), self.head_dim, self.theta, device)
return self._tables
def local_mask(valid: torch.Tensor, radius: int) -> torch.Tensor:
"""[B, N, N] mask: query i may attend to valid key j when |i - j| <= radius.
Every position may also attend to itself. Without that, a padding query far
from any real token would have no key at all, and SDPA returns NaN for
such a row, which would then leak into later layers.
"""
n = valid.shape[1]
positions = torch.arange(n, device=valid.device)
band = (positions[:, None] - positions[None, :]).abs() <= radius
itself = torch.eye(n, dtype=torch.bool, device=valid.device)
return (band[None] & valid[:, None, :]) | itself[None]
def segment_mask(segments: torch.Tensor, valid: torch.Tensor) -> torch.Tensor:
"""[B, N, N] mask for packed rows: a token sees only valid tokens of its own document.
segments: [B, N] document number of each token (padding may hold any value).
Every position may also see itself, so no attention row is ever empty.
"""
same = segments[:, :, None] == segments[:, None, :]
itself = torch.eye(segments.shape[1], dtype=torch.bool, device=segments.device)
return (same & valid[:, None, :]) | itself[None]
class EncoderBlock(nn.Module):
def __init__(self, config: NuriConfig, layer_number: int):
super().__init__()
schedule = config.attention_schedule
self.local = bool(schedule) and schedule[layer_number - 1] == "L"
self.attn_norm = RMSNorm(config.hidden_size, config.norm_eps)
self.attn = Attention(config.hidden_size, config.num_heads, config.qk_norm, config.norm_eps, config.num_kv_heads, config.kv_latent_dim)
self.ffn_norm = RMSNorm(config.hidden_size, config.norm_eps)
self.ffn = build_ffn(config, "input", layer_number)
def forward(self, h, valid, rope, local_keys=None, global_keys=None):
"""global_keys: optional per-query mask (packed rows); otherwise the padding mask is used."""
keys = local_keys if self.local else (global_keys if global_keys is not None else valid)
h = h + self.attn(self.attn_norm(h), key_mask=keys, rope=rope)
return h + self.ffn(self.ffn_norm(h), valid)
class InputEncoder(nn.Module):
"""X = E_phi(tokens).
It takes already embedded tokens. The token embedding lives in the model,
which shares it with the canvas input and the vocabulary head; keeping it
out of this module means the weight has exactly one name in checkpoints.
"""
def __init__(self, config: NuriConfig):
super().__init__()
self.local_radius = config.local_radius
self.gradient_checkpointing = False # set by NuriModel.set_gradient_checkpointing
self.rope = RopeTables(config.head_dim, config.rope_theta)
self.blocks = nn.ModuleList(EncoderBlock(config, i + 1) for i in range(config.input_layers))
self.final_norm = RMSNorm(config.hidden_size, config.norm_eps)
def forward(self, embedded: torch.Tensor, valid: torch.Tensor,
segments: torch.Tensor | None = None) -> torch.Tensor:
"""embedded: [B, N, D] token embeddings; valid: [B, N] real-token mask.
segments: optional [B, N] document numbers for packed rows (several
documents in one row). Tokens then attend only within their own
document. RoPE is relative, so a document's offset in the row does
not matter.
"""
h = embedded
rope = self.rope(h.shape[1], h.device)
# Masks are built once per forward and shared by all layers.
global_keys = segment_mask(segments, valid) if segments is not None else None
local_keys = None
if any(b.local for b in self.blocks):
local_keys = local_mask(valid, self.local_radius)
if global_keys is not None:
local_keys = local_keys & global_keys
for block in self.blocks:
if self.gradient_checkpointing and torch.is_grad_enabled():
h = run_checkpointed(block, h, valid, rope, local_keys, global_keys)
else:
h = block(h, valid, rope, local_keys, global_keys)
return self.final_norm(h)