File size: 5,063 Bytes
c394b65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
"""TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input.

  1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm
  2. token stream: int4 token embeddings -> projection -> learned positions ->
     small transformer -> attention pool

The streams are concatenated and classified into Emo's 812 emoji labels.
`set_qat(True)` fake-quantises every weight matrix exactly the way `export.py`
stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated
is what ships.
"""

from __future__ import annotations

from dataclasses import asdict, dataclass

import torch
import torch.nn as nn
import torch.nn.functional as F

from .quant import fake_quant


@dataclass
class TinyEmoConfig:
    vocab: int = 2048
    n_buckets: int = 16384  # power of two, <= 32768
    ng_dim: int = 16
    tok_dim: int = 48
    d_model: int = 96
    n_layers: int = 2
    n_heads: int = 2
    ffn: int = 192
    head_hidden: int = 112
    n_labels: int = 812
    max_tokens: int = 16
    max_features: int = 96
    dropout: float = 0.0

    def to_dict(self) -> dict:
        return asdict(self)


class QLinear(nn.Linear):
    qat = False

    def forward(self, x):
        w = fake_quant(self.weight, "int8_row") if self.qat else self.weight
        return F.linear(x, w, self.bias)


class QEmbedding(nn.Embedding):
    qat = False
    mode = "int4_row"

    def forward(self, ids):
        w = fake_quant(self.weight, self.mode) if self.qat else self.weight
        return F.embedding(ids, w)


class TinyEmo(nn.Module):
    def __init__(self, cfg: TinyEmoConfig):
        super().__init__()
        assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two"
        self.cfg = cfg
        self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim)
        self.tok_emb.mode = "int4_row"
        self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False)
        self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02)
        self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim)
        self.ng_emb.mode = "int4_tensor"
        nn.init.normal_(self.ng_emb.weight, std=0.1)
        self.ng_norm = nn.LayerNorm(cfg.ng_dim)
        self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)])
        self.final_norm = nn.LayerNorm(cfg.d_model)
        self.pool_proj = QLinear(cfg.d_model, cfg.d_model)
        self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02)
        self.pool_norm = nn.LayerNorm(cfg.d_model)
        self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden)
        self.head2 = QLinear(cfg.head_hidden, cfg.n_labels)
        self.drop = nn.Dropout(cfg.dropout)

    def set_qat(self, on: bool) -> None:
        for m in self.modules():
            if isinstance(m, (QLinear, QEmbedding)):
                m.qat = on

    def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count):
        cfg = self.cfg
        # ---- n-gram stream ----------------------------------------------
        e = self.ng_emb(ng_bucket % cfg.n_buckets)  # B,F,H,ng
        e = e * ng_sign.unsqueeze(-1).to(e.dtype)
        ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype)
        ng = self.ng_norm(ng)

        # ---- token stream -----------------------------------------------
        t = tok_ids.shape[1]
        x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t]
        key_mask = tok_mask.unsqueeze(1).unsqueeze(1)  # B,1,1,T
        for blk in self.blocks:
            x = blk(x, key_mask)
        x = self.final_norm(x)
        score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5)
        score = score.masked_fill(~tok_mask, -1e9)
        a = torch.softmax(score, dim=1)
        pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1))

        z = torch.cat([ng, pooled], dim=-1)
        z = self.drop(F.gelu(self.head1(z)))
        return self.head2(z)


class _Block(nn.Module):
    def __init__(self, cfg: TinyEmoConfig):
        super().__init__()
        d = cfg.d_model
        self.n_heads = cfg.n_heads
        self.norm1 = nn.LayerNorm(d)
        self.qkv = QLinear(d, 3 * d)
        self.proj = QLinear(d, d)
        self.norm2 = nn.LayerNorm(d)
        self.ff1 = QLinear(d, cfg.ffn)
        self.ff2 = QLinear(cfg.ffn, d)
        self.drop = nn.Dropout(cfg.dropout)

    def forward(self, x, key_mask):
        b, t, d = x.shape
        h = self.norm1(x)
        qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv.unbind(0)
        score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5)
        score = score.masked_fill(~key_mask, -1e9)
        a = self.drop(torch.softmax(score, dim=-1))
        o = (a @ v).transpose(1, 2).reshape(b, t, d)
        x = x + self.drop(self.proj(o))
        h = self.norm2(x)
        return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h)))))


def count_params(model: nn.Module) -> int:
    return sum(p.numel() for p in model.parameters())