File size: 6,054 Bytes
7d2a7b5
 
 
 
a29d041
7c25c00
 
 
 
 
 
 
 
 
 
 
7d2a7b5
 
7c25c00
7d2a7b5
7c25c00
7d2a7b5
 
7c25c00
 
 
 
7d2a7b5
 
7c25c00
7d2a7b5
7c25c00
 
 
 
 
 
 
 
 
 
 
 
 
7d2a7b5
7c25c00
7d2a7b5
 
 
 
 
 
 
7c25c00
 
7d2a7b5
 
7c25c00
 
7d2a7b5
 
 
7c25c00
 
 
 
 
7d2a7b5
 
 
 
7c25c00
 
 
 
 
 
 
 
 
7d2a7b5
 
 
 
 
 
7c25c00
7d2a7b5
7c25c00
 
7d2a7b5
 
 
 
 
 
 
 
7c25c00
7d2a7b5
7c25c00
 
 
 
 
 
 
 
 
7d2a7b5
7c25c00
 
7d2a7b5
7c25c00
7d2a7b5
 
 
 
7c25c00
 
7d2a7b5
7c25c00
7d2a7b5
 
7c25c00
 
 
 
 
7d2a7b5
 
 
7c25c00
7d2a7b5
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from dataclasses import dataclass

@dataclass
class ModelConfig:
    """Configuration class for ViuResonance100M."""
    vocab_size: int = 64000
    d_model: int = 1024
    n_layers: int = 12
    n_heads: int = 16
    dropout: float = 0.1
    chunk_size: int = 512

class RMSNorm(nn.Module):
    def __init__(self, dim: int, eps: float = 1e-6):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(dim))
        self.eps = eps
        
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        # Enforce float32 accumulation for stability, then cast back to original dtype
        norm_x = x.float() * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + self.eps) * self.weight.float()
        return norm_x.to(x.dtype)

class ResonanceLayerKaggle(nn.Module):
    def __init__(self, config: ModelConfig):
        super().__init__()
        if config.d_model % config.n_heads != 0:
            raise ValueError(f"d_model ({config.d_model}) must be divisible by n_heads ({config.n_heads})")
            
        self.n_heads = config.n_heads
        self.head_dim = config.d_model // config.n_heads
        self.chunk_size = config.chunk_size
        
        self.to_amp = nn.Linear(config.d_model, config.d_model, bias=False)
        self.to_freq = nn.Linear(config.d_model, config.n_heads, bias=False)
        self.to_phase = nn.Linear(config.d_model, config.n_heads, bias=False)
        self.to_v = nn.Linear(config.d_model, config.d_model, bias=False)
        self.to_out = nn.Linear(config.d_model, config.d_model, bias=False)
        self.norm = RMSNorm(config.d_model)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B, T, C = x.shape
        h = self.norm(x)
        A = self.to_amp(h).view(B, T, self.n_heads, self.head_dim)
        V = self.to_v(h).view(B, T, self.n_heads, self.head_dim)
        freq = torch.tanh(self.to_freq(h)) * 2.0  # B,T,H
        phase = torch.tanh(self.to_phase(h)) * math.pi

        # Use torch.long for absolute positions to avoid float precision issues during subtraction
        pos = torch.arange(T, device=x.device, dtype=torch.long)
        out = torch.zeros(B, T, C, device=x.device, dtype=x.dtype)
        
        for i in range(0, T, self.chunk_size):
            end = min(i+self.chunk_size, T)
            phase_i = phase[:, i:end, :].permute(0,2,1).unsqueeze(-1) # B,H,chunk,1
            phase_j = phase.permute(0,2,1).unsqueeze(-2) # B,H,1,T
            
            # Robust distance calculation in torch.long to prevent overflow/dtype issues
            row_idx = torch.arange(i, end, device=x.device, dtype=torch.long)[:, None]
            col_idx = pos[None, :]
            dist_long = (row_idx - col_idx).clamp(min=0)
            dist = dist_long.to(freq.dtype).view(1, 1, end-i, T)
            
            freq_i = freq[:, i:end, :].permute(0,2,1).unsqueeze(-1)
            angle = (phase_i - phase_j) + freq_i * dist * 0.05
            
            # Pure Cosine similarity. We apply the sqrt(d) scaling during the gating phase below
            score = torch.cos(angle)
            
            # Boolean causal mask for precision and stability
            causal_bool = (row_idx >= col_idx).view(1,1,end-i,T)
            
            # Mask out future tokens
            score = score.masked_fill(~causal_bool, 0.0)
            score = score / math.sqrt(self.head_dim)
            
            amp = A.norm(dim=-1).permute(0,2,1) # B,H,T
            amp_i = amp[:,:,i:end].unsqueeze(-1)
            amp_j = amp.unsqueeze(-2)
            score = score * torch.sigmoid(amp_i * amp_j)
            
            V_t = V.permute(0,2,1,3).contiguous() # B,H,T,D
            Bh = B * self.n_heads
            assert C == self.n_heads * self.head_dim, f"C ({C}) != n_heads*head_dim ({self.n_heads*self.head_dim})"
            
            s = score.reshape(Bh, end-i, T)
            v = V_t.reshape(Bh, T, self.head_dim)
            o = torch.bmm(s, v).view(B, self.n_heads, end-i, self.head_dim).permute(0,2,1,3).reshape(B, end-i, C)
            out[:, i:end] = o

        return self.to_out(out)

class ViuResonance100M(nn.Module):
    def __init__(self, config: ModelConfig):
        super().__init__()
        self.config = config
        self.emb = nn.Embedding(config.vocab_size, config.d_model)
        self.layers = nn.ModuleList([ResonanceLayerKaggle(config) for _ in range(config.n_layers)])
        self.norm = RMSNorm(config.d_model)
        self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
        self.resid_dropout = nn.Dropout(config.dropout)
        
        # Proper initialization sequence to preserve weight tying!
        # First randomly initialize everything...
        self.apply(self._init_weights)
        # THEN explicitly tie the weights
        self.head.weight = self.emb.weight

    def _init_weights(self, module: nn.Module):
        if isinstance(module, nn.Linear):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
        elif isinstance(module, nn.Embedding):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
        elif isinstance(module, RMSNorm):
            torch.nn.init.ones_(module.weight)

    def forward(self, idx: torch.Tensor, targets: torch.Tensor = None, use_checkpoint: bool = True) -> tuple[torch.Tensor, torch.Tensor | None]:
        x = self.emb(idx)
        for layer in self.layers:
            if self.training and use_checkpoint:
                x = x + self.resid_dropout(checkpoint(layer, x, use_reentrant=False))
            else:
                x = x + self.resid_dropout(layer(x))
                
        logits = self.head(self.norm(x))
        loss = None
        if targets is not None:
            # targets with value -100 are automatically ignored in cross_entropy
            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-100)
        return logits, loss