File size: 5,063 Bytes
c394b65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input.
1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm
2. token stream: int4 token embeddings -> projection -> learned positions ->
small transformer -> attention pool
The streams are concatenated and classified into Emo's 812 emoji labels.
`set_qat(True)` fake-quantises every weight matrix exactly the way `export.py`
stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated
is what ships.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
from .quant import fake_quant
@dataclass
class TinyEmoConfig:
vocab: int = 2048
n_buckets: int = 16384 # power of two, <= 32768
ng_dim: int = 16
tok_dim: int = 48
d_model: int = 96
n_layers: int = 2
n_heads: int = 2
ffn: int = 192
head_hidden: int = 112
n_labels: int = 812
max_tokens: int = 16
max_features: int = 96
dropout: float = 0.0
def to_dict(self) -> dict:
return asdict(self)
class QLinear(nn.Linear):
qat = False
def forward(self, x):
w = fake_quant(self.weight, "int8_row") if self.qat else self.weight
return F.linear(x, w, self.bias)
class QEmbedding(nn.Embedding):
qat = False
mode = "int4_row"
def forward(self, ids):
w = fake_quant(self.weight, self.mode) if self.qat else self.weight
return F.embedding(ids, w)
class TinyEmo(nn.Module):
def __init__(self, cfg: TinyEmoConfig):
super().__init__()
assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two"
self.cfg = cfg
self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim)
self.tok_emb.mode = "int4_row"
self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False)
self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02)
self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim)
self.ng_emb.mode = "int4_tensor"
nn.init.normal_(self.ng_emb.weight, std=0.1)
self.ng_norm = nn.LayerNorm(cfg.ng_dim)
self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)])
self.final_norm = nn.LayerNorm(cfg.d_model)
self.pool_proj = QLinear(cfg.d_model, cfg.d_model)
self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02)
self.pool_norm = nn.LayerNorm(cfg.d_model)
self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden)
self.head2 = QLinear(cfg.head_hidden, cfg.n_labels)
self.drop = nn.Dropout(cfg.dropout)
def set_qat(self, on: bool) -> None:
for m in self.modules():
if isinstance(m, (QLinear, QEmbedding)):
m.qat = on
def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count):
cfg = self.cfg
# ---- n-gram stream ----------------------------------------------
e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng
e = e * ng_sign.unsqueeze(-1).to(e.dtype)
ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype)
ng = self.ng_norm(ng)
# ---- token stream -----------------------------------------------
t = tok_ids.shape[1]
x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t]
key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T
for blk in self.blocks:
x = blk(x, key_mask)
x = self.final_norm(x)
score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5)
score = score.masked_fill(~tok_mask, -1e9)
a = torch.softmax(score, dim=1)
pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1))
z = torch.cat([ng, pooled], dim=-1)
z = self.drop(F.gelu(self.head1(z)))
return self.head2(z)
class _Block(nn.Module):
def __init__(self, cfg: TinyEmoConfig):
super().__init__()
d = cfg.d_model
self.n_heads = cfg.n_heads
self.norm1 = nn.LayerNorm(d)
self.qkv = QLinear(d, 3 * d)
self.proj = QLinear(d, d)
self.norm2 = nn.LayerNorm(d)
self.ff1 = QLinear(d, cfg.ffn)
self.ff2 = QLinear(cfg.ffn, d)
self.drop = nn.Dropout(cfg.dropout)
def forward(self, x, key_mask):
b, t, d = x.shape
h = self.norm1(x)
qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5)
score = score.masked_fill(~key_mask, -1e9)
a = self.drop(torch.softmax(score, dim=-1))
o = (a @ v).transpose(1, 2).reshape(b, t, d)
x = x + self.drop(self.proj(o))
h = self.norm2(x)
return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h)))))
def count_params(model: nn.Module) -> int:
return sum(p.numel() for p in model.parameters())
|