tinyemo / runtime.py
anudit's picture
Release tinyemo model and inference code
c394b65 verified
Raw History Blame Contribute Delete
6.98 kB
"""Dependency-free (NumPy) inference for the exported TinyEmo artifact.
rt = TinyEmoRuntime("tiny_emo/artifacts_v2/tiny_emo.bin")
rt.suggest("Walk the dog!!") # -> [("🐕", 0.31)] (default: 1)
rt.suggest("Walk the dog!!", 3) # -> up to 3 (clamped to 1..3)
Input is normalised first (lower-case, punctuation/emoji stripped, last 4 words).
"""
from __future__ import annotations
import json
import math
import struct
from pathlib import Path
import numpy as np
from .en_features import N_HASHES, featurize
from .normalize import MAX_WORDS, normalize
from .tiny_tokenizer import TinyTokenizer
MAX_OUTPUT = 3
def _gelu(x: np.ndarray) -> np.ndarray:
# exact GELU with an A&S 7.1.26 erf (|err| < 1.5e-7)
z = x / math.sqrt(2.0)
a = np.abs(z)
t = 1.0 / (1.0 + 0.3275911 * a)
poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))))
erf = np.sign(z) * (1.0 - poly * np.exp(-a * a))
return 0.5 * x * (1.0 + erf)
def _softmax(x: np.ndarray, axis: int = -1) -> np.ndarray:
x = x - x.max(axis=axis, keepdims=True)
e = np.exp(x)
return e / e.sum(axis=axis, keepdims=True)
def _ln(x, w, b, eps=1e-5):
m = x.mean(-1, keepdims=True)
v = ((x - m) ** 2).mean(-1, keepdims=True)
return (x - m) / np.sqrt(v + eps) * w + b
def _unpack_nibbles(buf: bytes, n: int) -> np.ndarray:
b = np.frombuffer(buf, np.uint8)
q = np.empty(b.size * 2, np.int8)
q[0::2] = (b & 0x0F).astype(np.int8) - 8
q[1::2] = (b >> 4).astype(np.int8) - 8
return q[:n]
class TinyEmoRuntime:
def __init__(self, weights_path: str, tokenizer_path: str | None = None):
data = Path(weights_path).read_bytes()
if data[:4] != b"TEM2":
raise ValueError("bad model file")
(ml,) = struct.unpack_from("<H", data, 4)
self.meta = json.loads(data[6 : 6 + ml])
off = 6 + ml
(ll,) = struct.unpack_from("<I", data, off)
off += 4
self.labels = data[off : off + ll].decode("utf-8").split("\n")
off += ll
self.w: dict[str, np.ndarray] = {}
while off < len(data):
nl = data[off]
name = data[off + 1 : off + 1 + nl].decode()
off += 1 + nl
nd = data[off]
dims = struct.unpack_from("<%dH" % nd, data, off + 1)
off += 1 + 2 * nd
mode = data[off]
off += 1
n = int(np.prod(dims))
if mode == 0:
arr = np.frombuffer(data, "<f2", n, off).astype(np.float32)
off += 2 * n
else:
ns = dims[0] if mode in (1, 2) else 1
scales = np.frombuffer(data, "<f2", ns, off).astype(np.float32)
off += 2 * ns
if mode == 1:
q = np.frombuffer(data, np.int8, n, off).astype(np.float32)
off += n
else:
nb = (n + 1) // 2
q = _unpack_nibbles(data[off : off + nb], n).astype(np.float32)
off += nb
arr = q.reshape(dims) * (scales[:, None] if mode in (1, 2) else scales[0])
self.w[name] = arr.reshape(dims)
tok = tokenizer_path or str(Path(weights_path).with_name("tiny_tokenizer.bin"))
self.tok = TinyTokenizer.load(tok)
m = self.meta
self.d, self.heads, self.T, self.F = m["d_model"], m["n_heads"], m["max_tokens"], m["max_features"]
# ------------------------------------------------------------------
def encode_batch(self, texts: list[str]):
"""Normalised texts -> (tok_ids[B,T], tok_mask[B,T], ng_raw[B,F,H], ng_sign[B,F,H])."""
b = len(texts)
ids = np.zeros((b, self.T), np.int64)
mask = np.zeros((b, self.T), bool)
raw = np.zeros((b, self.F, N_HASHES), np.int64)
sign = np.zeros((b, self.F, N_HASHES), np.int8)
for i, text in enumerate(texts):
t = self.tok.encode(normalize(text), self.T)
ids[i, : len(t)] = t
mask[i, : len(t)] = True
r, s = featurize(normalize(text), self.F)
raw[i, : len(r)] = r
sign[i, : len(s)] = s
return ids, mask, raw, sign
def _block(self, x, mask, i):
w, p = self.w, f"blocks.{i}."
b, t, d = x.shape
hd = d // self.heads
h = _ln(x, w[p + "norm1.weight"], w[p + "norm1.bias"])
qkv = h @ w[p + "qkv.weight"].T + w[p + "qkv.bias"]
q, k, v = qkv.reshape(b, t, 3, self.heads, hd).transpose(2, 0, 3, 1, 4)
s = q @ k.transpose(0, 1, 3, 2) / math.sqrt(hd)
s = np.where(mask[:, None, None, :], s, -1e9)
o = (_softmax(s) @ v).transpose(0, 2, 1, 3).reshape(b, t, d)
x = x + o @ w[p + "proj.weight"].T + w[p + "proj.bias"]
h = _ln(x, w[p + "norm2.weight"], w[p + "norm2.bias"])
h = _gelu(h @ w[p + "ff1.weight"].T + w[p + "ff1.bias"])
return x + h @ w[p + "ff2.weight"].T + w[p + "ff2.bias"]
def forward(self, ids, mask, raw, sign) -> np.ndarray:
w, m = self.w, self.meta
# token stream
x = w["tok_emb.weight"][ids] @ w["tok_proj.weight"].T + w["pos_emb"][: ids.shape[1]]
for i in range(m["n_layers"]):
x = self._block(x, mask, i)
x = _ln(x, w["final_norm.weight"], w["final_norm.bias"])
proj = np.tanh(x) @ w["pool_proj.weight"].T + w["pool_proj.bias"]
score = np.where(mask, proj @ w["pool_q"] / math.sqrt(self.d), -1e9)
pooled = (_softmax(score, 1)[:, :, None] * x).sum(1)
pooled = _ln(pooled, w["pool_norm.weight"], w["pool_norm.bias"])
# n-gram stream
e = w["ng_emb.weight"][raw % m["n_buckets"]] * sign[..., None] # B,F,H,ng
count = np.maximum((sign[:, :, 0] != 0).sum(1), 1).astype(np.float32)
ng = _ln(e.sum((1, 2)) / count[:, None], w["ng_norm.weight"], w["ng_norm.bias"])
z = _gelu(np.concatenate([ng, pooled], -1) @ w["head1.weight"].T + w["head1.bias"])
return z @ w["head2.weight"].T + w["head2.bias"]
# ------------------------------------------------------------------
def logits_batch(self, texts: list[str], bs: int = 512) -> np.ndarray:
out = []
for i in range(0, len(texts), bs):
out.append(self.forward(*self.encode_batch(texts[i : i + bs])))
return np.concatenate(out) if out else np.zeros((0, self.meta["n_labels"]), np.float32)
def logits(self, text: str) -> np.ndarray:
return self.logits_batch([text])[0]
def suggest(self, text: str, limit: int = 1) -> list[tuple[str, float]]:
"""Up to `limit` emoji suggestions (1-3, default 1); [] for empty input."""
limit = max(1, min(MAX_OUTPUT, int(limit)))
if not normalize(text):
return []
p = _softmax(self.logits(text))
order = np.argsort(-p)[:limit]
return [(self.labels[i], float(p[i])) for i in order]