import math import torch import torch.nn as nn import torch.nn.functional as F from torch.utils.checkpoint import checkpoint from dataclasses import dataclass @dataclass class ModelConfig: """Configuration class for ViuResonance100M.""" vocab_size: int = 64000 d_model: int = 1024 n_layers: int = 12 n_heads: int = 16 dropout: float = 0.1 chunk_size: int = 512 class RMSNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-6): super().__init__() self.weight = nn.Parameter(torch.ones(dim)) self.eps = eps def forward(self, x: torch.Tensor) -> torch.Tensor: # Enforce float32 accumulation for stability, then cast back to original dtype norm_x = x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.weight.float() return norm_x.to(x.dtype) class ResonanceLayerKaggle(nn.Module): def __init__(self, config: ModelConfig): super().__init__() if config.d_model % config.n_heads != 0: raise ValueError(f"d_model ({config.d_model}) must be divisible by n_heads ({config.n_heads})") self.n_heads = config.n_heads self.head_dim = config.d_model // config.n_heads self.chunk_size = config.chunk_size self.to_amp = nn.Linear(config.d_model, config.d_model, bias=False) self.to_freq = nn.Linear(config.d_model, config.n_heads, bias=False) self.to_phase = nn.Linear(config.d_model, config.n_heads, bias=False) self.to_v = nn.Linear(config.d_model, config.d_model, bias=False) self.to_out = nn.Linear(config.d_model, config.d_model, bias=False) self.norm = RMSNorm(config.d_model) def forward(self, x: torch.Tensor) -> torch.Tensor: B, T, C = x.shape h = self.norm(x) A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim) V = self.to_v(h).view(B, T, self.n_heads, self.head_dim) freq = torch.tanh(self.to_freq(h)) * 2.0 # B,T,H phase = torch.tanh(self.to_phase(h)) * math.pi # Use torch.long for absolute positions to avoid float precision issues during subtraction pos = torch.arange(T, device=x.device, dtype=torch.long) out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype) for i in range(0, T, self.chunk_size): end = min(i+self.chunk_size, T) phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1 phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T # Robust distance calculation in torch.long to prevent overflow/dtype issues row_idx = torch.arange(i, end, device=x.device, dtype=torch.long)[:, None] col_idx = pos[None, :] dist_long = (row_idx - col_idx).clamp(min=0) dist = dist_long.to(freq.dtype).view(1, 1, end-i, T) freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1) angle = (phase_i - phase_j) + freq_i * dist * 0.05 # Pure Cosine similarity. We apply the sqrt(d) scaling during the gating phase below score = torch.cos(angle) # Boolean causal mask for precision and stability causal_bool = (row_idx >= col_idx).view(1,1,end-i,T) # Mask out future tokens score = score.masked_fill(~causal_bool, 0.0) score = score / math.sqrt(self.head_dim) amp = A.norm(dim=-1).permute(0,2,1) # B,H,T amp_i = amp[:,:,i:end].unsqueeze(-1) amp_j = amp.unsqueeze(-2) score = score * torch.sigmoid(amp_i * amp_j) V_t = V.permute(0,2,1,3).contiguous() # B,H,T,D Bh = B * self.n_heads assert C == self.n_heads * self.head_dim, f"C ({C}) != n_heads*head_dim ({self.n_heads*self.head_dim})" s = score.reshape(Bh, end-i, T) v = V_t.reshape(Bh, T, self.head_dim) o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C) out[:, i:end] = o return self.to_out(out) class ViuResonance100M(nn.Module): def __init__(self, config: ModelConfig): super().__init__() self.config = config self.emb = nn.Embedding(config.vocab_size, config.d_model) self.layers = nn.ModuleList([ResonanceLayerKaggle(config) for _ in range(config.n_layers)]) self.norm = RMSNorm(config.d_model) self.head = nn.Linear(config.d_model, config.vocab_size, bias=False) self.resid_dropout = nn.Dropout(config.dropout) # Proper initialization sequence to preserve weight tying! # First randomly initialize everything... self.apply(self._init_weights) # THEN explicitly tie the weights self.head.weight = self.emb.weight def _init_weights(self, module: nn.Module): if isinstance(module, nn.Linear): torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) elif isinstance(module, nn.Embedding): torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) elif isinstance(module, RMSNorm): torch.nn.init.ones_(module.weight) def forward(self, idx: torch.Tensor, targets: torch.Tensor = None, use_checkpoint: bool = True) -> tuple[torch.Tensor, torch.Tensor | None]: x = self.emb(idx) for layer in self.layers: if self.training and use_checkpoint: x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False)) else: x = x + self.resid_dropout(layer(x)) logits = self.head(self.norm(x)) loss = None if targets is not None: # targets with value -100 are automatically ignored in cross_entropy loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100) return logits, loss