File size: 6,054 Bytes
7d2a7b5 a29d041 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 7c25c00 7d2a7b5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from dataclasses import dataclass
@dataclass
class ModelConfig:
"""Configuration class for ViuResonance100M."""
vocab_size: int = 64000
d_model: int = 1024
n_layers: int = 12
n_heads: int = 16
dropout: float = 0.1
chunk_size: int = 512
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
# Enforce float32 accumulation for stability, then cast back to original dtype
norm_x = x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.weight.float()
return norm_x.to(x.dtype)
class ResonanceLayerKaggle(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
if config.d_model % config.n_heads != 0:
raise ValueError(f"d_model ({config.d_model}) must be divisible by n_heads ({config.n_heads})")
self.n_heads = config.n_heads
self.head_dim = config.d_model // config.n_heads
self.chunk_size = config.chunk_size
self.to_amp = nn.Linear(config.d_model, config.d_model, bias=False)
self.to_freq = nn.Linear(config.d_model, config.n_heads, bias=False)
self.to_phase = nn.Linear(config.d_model, config.n_heads, bias=False)
self.to_v = nn.Linear(config.d_model, config.d_model, bias=False)
self.to_out = nn.Linear(config.d_model, config.d_model, bias=False)
self.norm = RMSNorm(config.d_model)
def forward(self, x: torch.Tensor) -> torch.Tensor:
B, T, C = x.shape
h = self.norm(x)
A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim)
V = self.to_v(h).view(B, T, self.n_heads, self.head_dim)
freq = torch.tanh(self.to_freq(h)) * 2.0 # B,T,H
phase = torch.tanh(self.to_phase(h)) * math.pi
# Use torch.long for absolute positions to avoid float precision issues during subtraction
pos = torch.arange(T, device=x.device, dtype=torch.long)
out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype)
for i in range(0, T, self.chunk_size):
end = min(i+self.chunk_size, T)
phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1
phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T
# Robust distance calculation in torch.long to prevent overflow/dtype issues
row_idx = torch.arange(i, end, device=x.device, dtype=torch.long)[:, None]
col_idx = pos[None, :]
dist_long = (row_idx - col_idx).clamp(min=0)
dist = dist_long.to(freq.dtype).view(1, 1, end-i, T)
freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1)
angle = (phase_i - phase_j) + freq_i * dist * 0.05
# Pure Cosine similarity. We apply the sqrt(d) scaling during the gating phase below
score = torch.cos(angle)
# Boolean causal mask for precision and stability
causal_bool = (row_idx >= col_idx).view(1,1,end-i,T)
# Mask out future tokens
score = score.masked_fill(~causal_bool, 0.0)
score = score / math.sqrt(self.head_dim)
amp = A.norm(dim=-1).permute(0,2,1) # B,H,T
amp_i = amp[:,:,i:end].unsqueeze(-1)
amp_j = amp.unsqueeze(-2)
score = score * torch.sigmoid(amp_i * amp_j)
V_t = V.permute(0,2,1,3).contiguous() # B,H,T,D
Bh = B * self.n_heads
assert C == self.n_heads * self.head_dim, f"C ({C}) != n_heads*head_dim ({self.n_heads*self.head_dim})"
s = score.reshape(Bh, end-i, T)
v = V_t.reshape(Bh, T, self.head_dim)
o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C)
out[:, i:end] = o
return self.to_out(out)
class ViuResonance100M(nn.Module):
def __init__(self, config: ModelConfig):
super().__init__()
self.config = config
self.emb = nn.Embedding(config.vocab_size, config.d_model)
self.layers = nn.ModuleList([ResonanceLayerKaggle(config) for _ in range(config.n_layers)])
self.norm = RMSNorm(config.d_model)
self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
self.resid_dropout = nn.Dropout(config.dropout)
# Proper initialization sequence to preserve weight tying!
# First randomly initialize everything...
self.apply(self._init_weights)
# THEN explicitly tie the weights
self.head.weight = self.emb.weight
def _init_weights(self, module: nn.Module):
if isinstance(module, nn.Linear):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
elif isinstance(module, nn.Embedding):
torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
elif isinstance(module, RMSNorm):
torch.nn.init.ones_(module.weight)
def forward(self, idx: torch.Tensor, targets: torch.Tensor = None, use_checkpoint: bool = True) -> tuple[torch.Tensor, torch.Tensor | None]:
x = self.emb(idx)
for layer in self.layers:
if self.training and use_checkpoint:
x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False))
else:
x = x + self.resid_dropout(layer(x))
logits = self.head(self.norm(x))
loss = None
if targets is not None:
# targets with value -100 are automatically ignored in cross_entropy
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100)
return logits, loss
|