"""TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input. 1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm 2. token stream: int4 token embeddings -> projection -> learned positions -> small transformer -> attention pool The streams are concatenated and classified into Emo's 812 emoji labels. `set_qat(True)` fake-quantises every weight matrix exactly the way `export.py` stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated is what ships. """ from __future__ import annotations from dataclasses import asdict, dataclass import torch import torch.nn as nn import torch.nn.functional as F from .quant import fake_quant @dataclass class TinyEmoConfig: vocab: int = 2048 n_buckets: int = 16384 # power of two, <= 32768 ng_dim: int = 16 tok_dim: int = 48 d_model: int = 96 n_layers: int = 2 n_heads: int = 2 ffn: int = 192 head_hidden: int = 112 n_labels: int = 812 max_tokens: int = 16 max_features: int = 96 dropout: float = 0.0 def to_dict(self) -> dict: return asdict(self) class QLinear(nn.Linear): qat = False def forward(self, x): w = fake_quant(self.weight, "int8_row") if self.qat else self.weight return F.linear(x, w, self.bias) class QEmbedding(nn.Embedding): qat = False mode = "int4_row" def forward(self, ids): w = fake_quant(self.weight, self.mode) if self.qat else self.weight return F.embedding(ids, w) class TinyEmo(nn.Module): def __init__(self, cfg: TinyEmoConfig): super().__init__() assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two" self.cfg = cfg self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim) self.tok_emb.mode = "int4_row" self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False) self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02) self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim) self.ng_emb.mode = "int4_tensor" nn.init.normal_(self.ng_emb.weight, std=0.1) self.ng_norm = nn.LayerNorm(cfg.ng_dim) self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)]) self.final_norm = nn.LayerNorm(cfg.d_model) self.pool_proj = QLinear(cfg.d_model, cfg.d_model) self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02) self.pool_norm = nn.LayerNorm(cfg.d_model) self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden) self.head2 = QLinear(cfg.head_hidden, cfg.n_labels) self.drop = nn.Dropout(cfg.dropout) def set_qat(self, on: bool) -> None: for m in self.modules(): if isinstance(m, (QLinear, QEmbedding)): m.qat = on def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count): cfg = self.cfg # ---- n-gram stream ---------------------------------------------- e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng e = e * ng_sign.unsqueeze(-1).to(e.dtype) ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype) ng = self.ng_norm(ng) # ---- token stream ----------------------------------------------- t = tok_ids.shape[1] x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t] key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T for blk in self.blocks: x = blk(x, key_mask) x = self.final_norm(x) score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5) score = score.masked_fill(~tok_mask, -1e9) a = torch.softmax(score, dim=1) pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1)) z = torch.cat([ng, pooled], dim=-1) z = self.drop(F.gelu(self.head1(z))) return self.head2(z) class _Block(nn.Module): def __init__(self, cfg: TinyEmoConfig): super().__init__() d = cfg.d_model self.n_heads = cfg.n_heads self.norm1 = nn.LayerNorm(d) self.qkv = QLinear(d, 3 * d) self.proj = QLinear(d, d) self.norm2 = nn.LayerNorm(d) self.ff1 = QLinear(d, cfg.ffn) self.ff2 = QLinear(cfg.ffn, d) self.drop = nn.Dropout(cfg.dropout) def forward(self, x, key_mask): b, t, d = x.shape h = self.norm1(x) qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv.unbind(0) score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5) score = score.masked_fill(~key_mask, -1e9) a = self.drop(torch.softmax(score, dim=-1)) o = (a @ v).transpose(1, 2).reshape(b, t, d) x = x + self.drop(self.proj(o)) h = self.norm2(x) return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h))))) def count_params(model: nn.Module) -> int: return sum(p.numel() for p in model.parameters())