File size: 8,678 Bytes
29d2351
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
import math
import torch
import torch.nn as nn
import torch.nn.functional as F

class RMSNorm(nn.Module):
    def __init__(self, dim: int, eps: float = 1e-6):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(dim))

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        variance = x.pow(2).mean(-1, keepdim=True)
        return x * torch.rsqrt(variance + self.eps) * self.weight

def precompute_rope_freqs(head_dim: int, seq_len: int, theta: float = 10000.0, device="cpu"):
    freqs = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device).float() / head_dim))
    t = torch.arange(seq_len, device=device).float()
    freqs = torch.outer(t, freqs)
    return torch.cos(freqs), torch.sin(freqs)

def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
    # x: (B, num_heads, seq_len, head_dim)
    T = x.shape[2]
    cos = cos[:T].unsqueeze(0).unsqueeze(1)
    sin = sin[:T].unsqueeze(0).unsqueeze(1)

    x_real = x[..., 0::2]
    x_imag = x[..., 1::2]
    out_real = x_real * cos - x_imag * sin
    out_imag = x_real * sin + x_imag * cos
    return torch.stack([out_real, out_imag], dim=-1).flatten(-2)

class QKVEGQAAttention(nn.Module):
    """
    Attention z Value-State Gating (QKVE) i Grouped-Query Attention (GQA):
    V_mod = V * (1 + tanh(E))
    num_heads = 12 (Query), num_kv_heads = 4 (Key, Value, Gate E)
    """
    def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4):
        super().__init__()
        self.d_model = d_model
        self.num_heads = num_heads
        self.num_kv_heads = num_kv_heads
        self.head_dim = d_model // num_heads

        self.q_proj = nn.Linear(d_model, d_model, bias=False)
        self.k_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False)
        self.v_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False)
        self.e_proj = nn.Linear(d_model, num_kv_heads * self.head_dim, bias=False) # Bramka wartości dla KV
        self.out_proj = nn.Linear(d_model, d_model, bias=False)

    def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
        B, T, C = x.shape
        q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
        k = self.k_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
        v = self.v_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
        e = self.e_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)

        q = apply_rope(q, cos, sin)
        k = apply_rope(k, cos, sin)

        # Modulacja wartości przez bramkę E:
        v_mod = v * (1.0 + torch.tanh(e))

        # GQA: powielenie KV dla odpowiadających im głów Query
        if self.num_kv_heads != self.num_heads:
            repeats = self.num_heads // self.num_kv_heads
            k = k.repeat_interleave(repeats, dim=1)
            v_mod = v_mod.repeat_interleave(repeats, dim=1)

        # Wydajna atencja przyczynowa (PyTorch SDPA / FlashAttention)
        out = F.scaled_dot_product_attention(q, k, v_mod, is_causal=True)
        out = out.transpose(1, 2).contiguous().view(B, T, C)
        return self.out_proj(out)

class SwiGLUFFN(nn.Module):
    def __init__(self, d_model: int, ffn_hidden: int):
        super().__init__()
        self.w_gate = nn.Linear(d_model, ffn_hidden, bias=False)
        self.w_up = nn.Linear(d_model, ffn_hidden, bias=False)
        self.w_down = nn.Linear(ffn_hidden, d_model, bias=False)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.w_down(F.silu(self.w_gate(x)) * self.w_up(x))

class QKVE30Block(nn.Module):
    """
    Pojedynczy blok QKVE z GQA i 2 warstwami SwiGLU:
    1 moduł uwagi QKVE-GQA + 2 warstwy SwiGLU FFN
    """
    def __init__(self, d_model: int = 384, num_heads: int = 12, num_kv_heads: int = 4, base_ffn: int = 512):
        super().__init__()
        self.attn_norm = RMSNorm(d_model)
        self.attn = QKVEGQAAttention(d_model, num_heads, num_kv_heads)

        # 2 warstwy SwiGLU z normami RMSNorm
        self.ffn_norms = nn.ModuleList([RMSNorm(d_model) for _ in range(2)])
        self.ffn_layers = nn.ModuleList([
            SwiGLUFFN(d_model, base_ffn),
            SwiGLUFFN(d_model, base_ffn)
        ])

    def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
        # Atencja z residual
        x = x + self.attn(self.attn_norm(x), cos, sin)

        # 2 warstwy SwiGLU z residual
        for norm, ffn in zip(self.ffn_norms, self.ffn_layers):
            x = x + ffn(norm(x))

        return x

class QKVE30Transformer(nn.Module):
    """
    Model QKVE-61M-30block:
    30 bloków transformera z Grouped-Query Attention (GQA) i Deep SwiGLU FFN (60 warstw FFN)
    Łącznie 61.03M parametrów.
    """
    def __init__(
        self,
        vocab_size: int = 32128,
        d_model: int = 384,
        num_layers: int = 30,
        num_heads: int = 12,
        num_kv_heads: int = 4,
        base_ffn: int = 512,
        max_seq_len: int = 1024
    ):
        super().__init__()
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.max_seq_len = max_seq_len

        self.tok_embeddings = nn.Embedding(vocab_size, d_model)
        self.blocks = nn.ModuleList([
            QKVE30Block(d_model, num_heads, num_kv_heads, base_ffn) for _ in range(num_layers)
        ])
        self.norm = RMSNorm(d_model)

        # Wiązanie wag: lm_head współdzieli wagi z embeddingami
        self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
        self.lm_head.weight = self.tok_embeddings.weight

        # Precompute RoPE frequencies
        cos, sin = precompute_rope_freqs(d_model // num_heads, max_seq_len)
        self.register_buffer("rope_cos", cos)
        self.register_buffer("rope_sin", sin)

        # Inicjalizacja wag
        self.apply(self._init_weights)

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
        elif isinstance(module, nn.Embedding):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)

    def forward(self, input_ids: torch.Tensor, targets: torch.Tensor = None):
        B, T = input_ids.shape
        x = self.tok_embeddings(input_ids)
        cos = self.rope_cos[:T]
        sin = self.rope_sin[:T]

        for block in self.blocks:
            x = block(x, cos, sin)

        x = self.norm(x)
        logits = self.lm_head(x)

        loss = None
        if targets is not None:
            loss = F.cross_entropy(logits.view(-1, self.vocab_size), targets.view(-1))

        return logits, loss

    @torch.no_grad()
    def generate(
        self,
        input_ids: torch.Tensor,
        max_new_tokens: int = 100,
        temperature: float = 0.8,
        top_p: float = 0.9,
        eos_token_id: int = 2
    ):
        self.eval()
        for _ in range(max_new_tokens):
            idx_cond = input_ids if input_ids.size(1) <= self.max_seq_len else input_ids[:, -self.max_seq_len:]
            logits, _ = self(idx_cond)
            logits = logits[:, -1, :] / max(temperature, 1e-5)

            if top_p < 1.0:
                sorted_logits, sorted_indices = torch.sort(logits, descending=True)
                cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
                sorted_indices_to_remove = cumulative_probs > top_p
                sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
                sorted_indices_to_remove[..., 0] = 0
                indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
                logits[indices_to_remove] = -float('Inf')

            probs = F.softmax(logits, dim=-1)
            next_token = torch.multinomial(probs, num_samples=1)
            input_ids = torch.cat((input_ids, next_token), dim=1)

            if eos_token_id is not None and (next_token == eos_token_id).all():
                break

        return input_ids

def count_parameters(model):
    total = sum(p.numel() for p in model.parameters())
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    return total, trainable

if __name__ == "__main__":
    model = QKVE30Transformer()
    total, trainable = count_parameters(model)
    print(f"✨ Model QKVE-61M-30block zainicjalizowany pomyślnie!")
    print(f"   Liczba parametrów (total):     {total:,} ({total/1e6:.2f}M)")
    print(f"   Liczba parametrów (trainable): {trainable:,} ({trainable/1e6:.2f}M)")