ViuRec / code /model.py
ViuAI's picture
Upload 2 files
7c25c00 verified
Raw History Blame
6.05 kB
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from dataclasses import dataclass
@dataclass
class ModelConfig:
"""Configuration class for ViuResonance100M."""
vocab_size: int = 64000
d_model: int = 1024
n_layers: int = 12
n_heads: int = 16
dropout: float = 0.1
chunk_size: int = 512
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
# Enforce float32 accumulation for stability, then cast back to original dtype
norm_x = x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.weight.float()
return norm_x.to(x.dtype)
class ResonanceLayerKaggle(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
if config.d_model % config.n_heads != 0:
raise ValueError(f"d_model ({config.d_model}) must be divisible by n_heads ({config.n_heads})")
self.n_heads = config.n_heads
self.head_dim = config.d_model // config.n_heads
self.chunk_size = config.chunk_size
self.to_amp = nn.Linear(config.d_model, config.d_model, bias=False)
self.to_freq = nn.Linear(config.d_model, config.n_heads, bias=False)
self.to_phase = nn.Linear(config.d_model, config.n_heads, bias=False)
self.to_v = nn.Linear(config.d_model, config.d_model, bias=False)
self.to_out = nn.Linear(config.d_model, config.d_model, bias=False)
self.norm = RMSNorm(config.d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
B, T, C = x.shape
h = self.norm(x)
A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim)
V = self.to_v(h).view(B, T, self.n_heads, self.head_dim)
freq = torch.tanh(self.to_freq(h)) * 2.0 # B,T,H
phase = torch.tanh(self.to_phase(h)) * math.pi
# Use torch.long for absolute positions to avoid float precision issues during subtraction
pos = torch.arange(T, device=x.device, dtype=torch.long)
out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype)
for i in range(0, T, self.chunk_size):
end = min(i+self.chunk_size, T)
phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1
phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T
# Robust distance calculation in torch.long to prevent overflow/dtype issues
row_idx = torch.arange(i, end, device=x.device, dtype=torch.long)[:, None]
col_idx = pos[None, :]
dist_long = (row_idx - col_idx).clamp(min=0)
dist = dist_long.to(freq.dtype).view(1, 1, end-i, T)
freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1)
angle = (phase_i - phase_j) + freq_i * dist * 0.05
# Pure Cosine similarity. We apply the sqrt(d) scaling during the gating phase below
score = torch.cos(angle)
# Boolean causal mask for precision and stability
causal_bool = (row_idx >= col_idx).view(1,1,end-i,T)
# Mask out future tokens
score = score.masked_fill(~causal_bool, 0.0)
score = score / math.sqrt(self.head_dim)
amp = A.norm(dim=-1).permute(0,2,1) # B,H,T
amp_i = amp[:,:,i:end].unsqueeze(-1)
amp_j = amp.unsqueeze(-2)
score = score * torch.sigmoid(amp_i * amp_j)
V_t = V.permute(0,2,1,3).contiguous() # B,H,T,D
Bh = B * self.n_heads
assert C == self.n_heads * self.head_dim, f"C ({C}) != n_heads*head_dim ({self.n_heads*self.head_dim})"
s = score.reshape(Bh, end-i, T)
v = V_t.reshape(Bh, T, self.head_dim)
o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C)
out[:, i:end] = o
return self.to_out(out)
class ViuResonance100M(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
self.config = config
self.emb = nn.Embedding(config.vocab_size, config.d_model)
self.layers = nn.ModuleList([ResonanceLayerKaggle(config) for _ in range(config.n_layers)])
self.norm = RMSNorm(config.d_model)
self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
self.resid_dropout = nn.Dropout(config.dropout)
# Proper initialization sequence to preserve weight tying!
# First randomly initialize everything...
self.apply(self._init_weights)
# THEN explicitly tie the weights
self.head.weight = self.emb.weight
def _init_weights(self, module: nn.Module):
if isinstance(module, nn.Linear):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
elif isinstance(module, nn.Embedding):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
elif isinstance(module, RMSNorm):
torch.nn.init.ones_(module.weight)
def forward(self, idx: torch.Tensor, targets: torch.Tensor = None, use_checkpoint: bool = True) -> tuple[torch.Tensor, torch.Tensor | None]:
x = self.emb(idx)
for layer in self.layers:
if self.training and use_checkpoint:
x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False))
else:
x = x + self.resid_dropout(layer(x))
logits = self.head(self.norm(x))
loss = None
if targets is not None:
# targets with value -100 are automatically ignored in cross_entropy
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100)
return logits, loss