tinyemo / model.py
anudit's picture
Release tinyemo model and inference code
c394b65 verified
Raw History Blame Contribute Delete
5.06 kB
"""TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input.
1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm
2. token stream: int4 token embeddings -> projection -> learned positions ->
small transformer -> attention pool
The streams are concatenated and classified into Emo's 812 emoji labels.
`set_qat(True)` fake-quantises every weight matrix exactly the way `export.py`
stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated
is what ships.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
from .quant import fake_quant
@dataclass
class TinyEmoConfig:
vocab: int = 2048
n_buckets: int = 16384 # power of two, <= 32768
ng_dim: int = 16
tok_dim: int = 48
d_model: int = 96
n_layers: int = 2
n_heads: int = 2
ffn: int = 192
head_hidden: int = 112
n_labels: int = 812
max_tokens: int = 16
max_features: int = 96
dropout: float = 0.0
def to_dict(self) -> dict:
return asdict(self)
class QLinear(nn.Linear):
qat = False
def forward(self, x):
w = fake_quant(self.weight, "int8_row") if self.qat else self.weight
return F.linear(x, w, self.bias)
class QEmbedding(nn.Embedding):
qat = False
mode = "int4_row"
def forward(self, ids):
w = fake_quant(self.weight, self.mode) if self.qat else self.weight
return F.embedding(ids, w)
class TinyEmo(nn.Module):
def __init__(self, cfg: TinyEmoConfig):
super().__init__()
assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two"
self.cfg = cfg
self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim)
self.tok_emb.mode = "int4_row"
self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False)
self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02)
self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim)
self.ng_emb.mode = "int4_tensor"
nn.init.normal_(self.ng_emb.weight, std=0.1)
self.ng_norm = nn.LayerNorm(cfg.ng_dim)
self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)])
self.final_norm = nn.LayerNorm(cfg.d_model)
self.pool_proj = QLinear(cfg.d_model, cfg.d_model)
self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02)
self.pool_norm = nn.LayerNorm(cfg.d_model)
self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden)
self.head2 = QLinear(cfg.head_hidden, cfg.n_labels)
self.drop = nn.Dropout(cfg.dropout)
def set_qat(self, on: bool) -> None:
for m in self.modules():
if isinstance(m, (QLinear, QEmbedding)):
m.qat = on
def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count):
cfg = self.cfg
# ---- n-gram stream ----------------------------------------------
e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng
e = e * ng_sign.unsqueeze(-1).to(e.dtype)
ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype)
ng = self.ng_norm(ng)
# ---- token stream -----------------------------------------------
t = tok_ids.shape[1]
x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t]
key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T
for blk in self.blocks:
x = blk(x, key_mask)
x = self.final_norm(x)
score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5)
score = score.masked_fill(~tok_mask, -1e9)
a = torch.softmax(score, dim=1)
pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1))
z = torch.cat([ng, pooled], dim=-1)
z = self.drop(F.gelu(self.head1(z)))
return self.head2(z)
class _Block(nn.Module):
def __init__(self, cfg: TinyEmoConfig):
super().__init__()
d = cfg.d_model
self.n_heads = cfg.n_heads
self.norm1 = nn.LayerNorm(d)
self.qkv = QLinear(d, 3 * d)
self.proj = QLinear(d, d)
self.norm2 = nn.LayerNorm(d)
self.ff1 = QLinear(d, cfg.ffn)
self.ff2 = QLinear(cfg.ffn, d)
self.drop = nn.Dropout(cfg.dropout)
def forward(self, x, key_mask):
b, t, d = x.shape
h = self.norm1(x)
qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5)
score = score.masked_fill(~key_mask, -1e9)
a = self.drop(torch.softmax(score, dim=-1))
o = (a @ v).transpose(1, 2).reshape(b, t, d)
x = x + self.drop(self.proj(o))
h = self.norm2(x)
return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h)))))
def count_params(model: nn.Module) -> int:
return sum(p.numel() for p in model.parameters())