File size: 7,715 Bytes
22eb156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
"""BananaMind 2 style decoder with optional LFT or ternary fake quantization.

The model uses pre-RMSNorm, RoPE, grouped-query attention, SwiGLU and tied
embeddings. BananaMind 2 mode also applies QK norm. LFT changes only routing.
Ternary mode uses floating-point master weights and fake-quantized projections.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
from transformers import PreTrainedModel
from transformers.generation import GenerationMixin
from transformers.modeling_outputs import CausalLMOutputWithPast

try:
    from .configuration_bananaall import BananaAllConfig
except ImportError:
    from configuration_bananaall import BananaAllConfig


class RMSNorm(nn.Module):
    def __init__(self, size, eps=1e-6):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(size))
        self.eps = eps

    def forward(self, x):
        y = x.float()
        return (y * torch.rsqrt(y.square().mean(-1, keepdim=True) + self.eps) * self.weight.float()).to(x.dtype)


class TernaryLinear(nn.Linear):
    """W1.58A8 fake quantization with straight-through gradients.

    The master weights remain floating point for optimization and checkpoints.
    This layer does not pack ternary weights or use a low-bit inference kernel.
    """

    def forward(self, x):
        weights = self.weight.float()
        weight_scale = weights.detach().abs().mean().clamp_min(1e-6)
        quantized_weights = (weights / weight_scale).round().clamp(-1, 1) * weight_scale
        fake_weights = self.weight + (quantized_weights.to(self.weight.dtype) - self.weight).detach()

        activations = x.float()
        activation_scale = activations.detach().abs().amax(dim=-1, keepdim=True).clamp_min(1e-6) / 127
        quantized_activations = (activations / activation_scale).round().clamp(-127, 127) * activation_scale
        fake_activations = x + (quantized_activations.to(x.dtype) - x).detach()
        return F.linear(fake_activations, fake_weights, self.bias)


def apply_rope(x, theta, position_ids):
    dim = x.shape[-1]
    inv = 1.0 / (theta ** (torch.arange(0, dim, 2, device=x.device, dtype=torch.float32) / dim))
    angles = position_ids.float().unsqueeze(-1) * inv
    cos = angles.cos().unsqueeze(1).to(x.dtype)
    sin = angles.sin().unsqueeze(1).to(x.dtype)
    even, odd = x[..., ::2], x[..., 1::2]
    return torch.stack((even * cos - odd * sin, even * sin + odd * cos), dim=-1).flatten(-2)


class Attention(nn.Module):
    def __init__(self, config):
        super().__init__()
        h, d, kv = config.num_attention_heads, config.head_dim, config.num_key_value_heads
        self.h, self.d, self.kv, self.theta = h, d, kv, config.rope_theta
        linear = TernaryLinear if config.ternary else nn.Linear
        self.q_proj = linear(config.hidden_size, h * d, bias=False)
        self.k_proj = linear(config.hidden_size, kv * d, bias=False)
        self.v_proj = linear(config.hidden_size, kv * d, bias=False)
        self.o_proj = linear(h * d, config.hidden_size, bias=False)
        self.q_norm = RMSNorm(d, config.rms_norm_eps) if config.architecture_style == "bananamind2" else nn.Identity()
        self.k_norm = RMSNorm(d, config.rms_norm_eps) if config.architecture_style == "bananamind2" else nn.Identity()

    def forward(self, x, attention_mask=None):
        b, t, _ = x.shape
        q = self.q_norm(self.q_proj(x).view(b, t, self.h, self.d).transpose(1, 2))
        k = self.k_norm(self.k_proj(x).view(b, t, self.kv, self.d).transpose(1, 2))
        v = self.v_proj(x).view(b, t, self.kv, self.d).transpose(1, 2)
        positions = torch.arange(t, device=x.device).unsqueeze(0)
        q, k = apply_rope(q, self.theta, positions), apply_rope(k, self.theta, positions)
        k = k.repeat_interleave(self.h // self.kv, dim=1)
        v = v.repeat_interleave(self.h // self.kv, dim=1)
        if attention_mask is None:
            y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        else:
            causal = torch.ones(t, t, device=x.device, dtype=torch.bool).tril()
            mask = causal[None, None] & attention_mask[:, None, None, :].bool()
            y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask)
        return self.o_proj(y.transpose(1, 2).contiguous().view(b, t, self.h * self.d))


class Block(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.norm1 = RMSNorm(config.hidden_size, config.rms_norm_eps)
        self.attn = Attention(config)
        self.norm2 = RMSNorm(config.hidden_size, config.rms_norm_eps)
        linear = TernaryLinear if config.ternary else nn.Linear
        self.gate_proj = linear(config.hidden_size, config.intermediate_size, bias=False)
        self.up_proj = linear(config.hidden_size, config.intermediate_size, bias=False)
        self.down_proj = linear(config.intermediate_size, config.hidden_size, bias=False)

    def forward(self, x, attention_mask=None):
        x = x + self.attn(self.norm1(x), attention_mask)
        z = self.norm2(x)
        return x + self.down_proj(F.silu(self.gate_proj(z)) * self.up_proj(z))


class BananaAllForCausalLM(PreTrainedModel, GenerationMixin):
    config_class = BananaAllConfig
    base_model_prefix = "model"
    _supports_sdpa = True
    # forward() returns a mean loss per microbatch and ignores **kwargs.
    # Tell Trainer to divide it by the gradient-accumulation count.
    accepts_loss_kwargs = False

    def __init__(self, config):
        super().__init__(config)
        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
        self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)])
        self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
        self.lm_head = (TernaryLinear if config.ternary else nn.Linear)(config.hidden_size, config.vocab_size, bias=False)
        self.post_init()
        self.tie_weights()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, value):
        self.lm_head = value

    def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):
        x = self.embed_tokens(input_ids)
        if self.config.lft and len(self.layers) > 1:
            x = self.layers[0](x, attention_mask)
            for i in range(1, len(self.layers)):
                x = self.layers[i](x, attention_mask)
                if i < len(self.layers) - 1:
                    x = self.layers[i - 1](x, attention_mask)
                    x = self.layers[i](x, attention_mask)
        else:
            for layer in self.layers:
                x = layer(x, attention_mask)
        logits = self.lm_head(self.norm(x))
        loss = None
        if labels is not None:
            shifted_logits = logits[:, :-1, :].contiguous().float()
            shifted_labels = labels[:, 1:].contiguous()
            loss = F.cross_entropy(shifted_logits.view(-1, shifted_logits.size(-1)), shifted_labels.view(-1), ignore_index=-100)
        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=None)

    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
        return {"input_ids": input_ids, "attention_mask": attention_mask}


def register():
    from transformers import AutoConfig, AutoModelForCausalLM
    try:
        AutoConfig.register("bananaall", BananaAllConfig)
    except ValueError:
        pass
    try:
        AutoModelForCausalLM.register(BananaAllConfig, BananaAllForCausalLM)
    except ValueError:
        pass