Download model.py from anudit/tinyemo: direct link, hf CLI and curl.
- Browser
- Download file 5.06 kB
-
https://huggingface.co/anudit/tinyemo/resolve/main/model.py
- Command line
-
hf download hf://anudit/tinyemo/model.py
-
curl -L -o model.py https://huggingface.co/anudit/tinyemo/resolve/main/model.py
5.06 kB
| """TinyEmo student (v2): two streams, quantisation-aware, <=4-word English input. | |
| 1. hashed n-gram stream: power-of-two bucket table (int4) x 2 hashes, signed mean -> LayerNorm | |
| 2. token stream: int4 token embeddings -> projection -> learned positions -> | |
| small transformer -> attention pool | |
| The streams are concatenated and classified into Emo's 812 emoji labels. | |
| `set_qat(True)` fake-quantises every weight matrix exactly the way `export.py` | |
| stores it (int8 per-row linears, int4 embeddings), so what is trained/evaluated | |
| is what ships. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import asdict, dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .quant import fake_quant | |
| class TinyEmoConfig: | |
| vocab: int = 2048 | |
| n_buckets: int = 16384 # power of two, <= 32768 | |
| ng_dim: int = 16 | |
| tok_dim: int = 48 | |
| d_model: int = 96 | |
| n_layers: int = 2 | |
| n_heads: int = 2 | |
| ffn: int = 192 | |
| head_hidden: int = 112 | |
| n_labels: int = 812 | |
| max_tokens: int = 16 | |
| max_features: int = 96 | |
| dropout: float = 0.0 | |
| def to_dict(self) -> dict: | |
| return asdict(self) | |
| class QLinear(nn.Linear): | |
| qat = False | |
| def forward(self, x): | |
| w = fake_quant(self.weight, "int8_row") if self.qat else self.weight | |
| return F.linear(x, w, self.bias) | |
| class QEmbedding(nn.Embedding): | |
| qat = False | |
| mode = "int4_row" | |
| def forward(self, ids): | |
| w = fake_quant(self.weight, self.mode) if self.qat else self.weight | |
| return F.embedding(ids, w) | |
| class TinyEmo(nn.Module): | |
| def __init__(self, cfg: TinyEmoConfig): | |
| super().__init__() | |
| assert cfg.n_buckets & (cfg.n_buckets - 1) == 0, "n_buckets must be a power of two" | |
| self.cfg = cfg | |
| self.tok_emb = QEmbedding(cfg.vocab, cfg.tok_dim) | |
| self.tok_emb.mode = "int4_row" | |
| self.tok_proj = QLinear(cfg.tok_dim, cfg.d_model, bias=False) | |
| self.pos_emb = nn.Parameter(torch.randn(cfg.max_tokens, cfg.d_model) * 0.02) | |
| self.ng_emb = QEmbedding(cfg.n_buckets, cfg.ng_dim) | |
| self.ng_emb.mode = "int4_tensor" | |
| nn.init.normal_(self.ng_emb.weight, std=0.1) | |
| self.ng_norm = nn.LayerNorm(cfg.ng_dim) | |
| self.blocks = nn.ModuleList([_Block(cfg) for _ in range(cfg.n_layers)]) | |
| self.final_norm = nn.LayerNorm(cfg.d_model) | |
| self.pool_proj = QLinear(cfg.d_model, cfg.d_model) | |
| self.pool_q = nn.Parameter(torch.randn(cfg.d_model) * 0.02) | |
| self.pool_norm = nn.LayerNorm(cfg.d_model) | |
| self.head1 = QLinear(cfg.ng_dim + cfg.d_model, cfg.head_hidden) | |
| self.head2 = QLinear(cfg.head_hidden, cfg.n_labels) | |
| self.drop = nn.Dropout(cfg.dropout) | |
| def set_qat(self, on: bool) -> None: | |
| for m in self.modules(): | |
| if isinstance(m, (QLinear, QEmbedding)): | |
| m.qat = on | |
| def forward(self, tok_ids, tok_mask, ng_bucket, ng_sign, ng_count): | |
| cfg = self.cfg | |
| # ---- n-gram stream ---------------------------------------------- | |
| e = self.ng_emb(ng_bucket % cfg.n_buckets) # B,F,H,ng | |
| e = e * ng_sign.unsqueeze(-1).to(e.dtype) | |
| ng = e.sum(dim=(1, 2)) / ng_count.clamp(min=1).unsqueeze(-1).to(e.dtype) | |
| ng = self.ng_norm(ng) | |
| # ---- token stream ----------------------------------------------- | |
| t = tok_ids.shape[1] | |
| x = self.tok_proj(self.tok_emb(tok_ids)) + self.pos_emb[:t] | |
| key_mask = tok_mask.unsqueeze(1).unsqueeze(1) # B,1,1,T | |
| for blk in self.blocks: | |
| x = blk(x, key_mask) | |
| x = self.final_norm(x) | |
| score = (self.pool_proj(torch.tanh(x)) @ self.pool_q) / (cfg.d_model**0.5) | |
| score = score.masked_fill(~tok_mask, -1e9) | |
| a = torch.softmax(score, dim=1) | |
| pooled = self.pool_norm((a.unsqueeze(-1) * x).sum(dim=1)) | |
| z = torch.cat([ng, pooled], dim=-1) | |
| z = self.drop(F.gelu(self.head1(z))) | |
| return self.head2(z) | |
| class _Block(nn.Module): | |
| def __init__(self, cfg: TinyEmoConfig): | |
| super().__init__() | |
| d = cfg.d_model | |
| self.n_heads = cfg.n_heads | |
| self.norm1 = nn.LayerNorm(d) | |
| self.qkv = QLinear(d, 3 * d) | |
| self.proj = QLinear(d, d) | |
| self.norm2 = nn.LayerNorm(d) | |
| self.ff1 = QLinear(d, cfg.ffn) | |
| self.ff2 = QLinear(cfg.ffn, d) | |
| self.drop = nn.Dropout(cfg.dropout) | |
| def forward(self, x, key_mask): | |
| b, t, d = x.shape | |
| h = self.norm1(x) | |
| qkv = self.qkv(h).reshape(b, t, 3, self.n_heads, d // self.n_heads).permute(2, 0, 3, 1, 4) | |
| q, k, v = qkv.unbind(0) | |
| score = (q @ k.transpose(-1, -2)) / ((d // self.n_heads) ** 0.5) | |
| score = score.masked_fill(~key_mask, -1e9) | |
| a = self.drop(torch.softmax(score, dim=-1)) | |
| o = (a @ v).transpose(1, 2).reshape(b, t, d) | |
| x = x + self.drop(self.proj(o)) | |
| h = self.norm2(x) | |
| return x + self.drop(self.ff2(self.drop(F.gelu(self.ff1(h))))) | |
| def count_params(model: nn.Module) -> int: | |
| return sum(p.numel() for p in model.parameters()) | |