Download inference/bengu_infer.py from OrkhonAI/bengu-1: direct link, hf CLI and curl.
- Browser
- Download file 6.78 kB
-
https://huggingface.co/OrkhonAI/bengu-1/resolve/main/inference/bengu_infer.py
- Command line
-
hf download hf://OrkhonAI/bengu-1/inference/bengu_infer.py
-
curl -L -o bengu_infer.py https://huggingface.co/OrkhonAI/bengu-1/resolve/main/inference/bengu_infer.py
6.78 kB
| """Bengu-1 inference: e-mail category, protected and needs_reply, on the CPU. | |
| Needs: torch, transformers, safetensors, numpy (and bengu_view.py next to this file for raw mail). | |
| python bengu_infer.py <package_dir> message.eml [more.eml ...] | |
| from bengu_infer import Bengu1 | |
| model = Bengu1("<package_dir>") | |
| model.classify_raw(open("message.eml", "rb").read()) | |
| model.predict(["<an already built input view>"]) | |
| The package holds one or two member encoders (members/<name>/: model.safetensors, encoder_config/, | |
| tokenizer/ as trained, member.json) and ensemble.json (labels, temperatures, thresholds, combination, the | |
| optional category stacker). Each member's probabilities are temperature-scaled, averaged (protected: mean or | |
| max), calibrated again, and the stacker, when present, gives the category. protected is never set for spam | |
| or phishing; needs_reply and protected use the thresholds in ensemble.json. | |
| """ | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| JUNK = ("spam", "phishing_or_scam") | |
| class Head(nn.Module): | |
| def __init__(self, encoder, n_cat, pooling="mean"): | |
| super().__init__() | |
| self.encoder = encoder | |
| hidden = encoder.config.hidden_size | |
| self.pooling = pooling | |
| self.drop = nn.Dropout(0.1) | |
| self.cat = nn.Linear(hidden, n_cat) | |
| self.prot = nn.Linear(hidden, 1) | |
| self.reply = nn.Linear(hidden, 1) | |
| self.register_buffer("temperature", torch.ones(3)) | |
| def forward(self, input_ids, attention_mask): | |
| h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state | |
| if self.pooling == "cls": | |
| pooled = h[:, 0] | |
| else: | |
| m = attention_mask.unsqueeze(-1).to(h.dtype) | |
| pooled = (h * m).sum(1) / m.sum(1).clamp_min(1.0) | |
| pooled = self.drop(pooled) | |
| return self.cat(pooled), self.prot(pooled).squeeze(-1), self.reply(pooled).squeeze(-1) | |
| def encode(tok, texts, max_len, head_tail): | |
| """At most max_len tokens; head_tail keeps the first 3/4 and the last 1/4 of an over-long text.""" | |
| if not head_tail: | |
| return tok(texts, truncation=True, max_length=max_len, padding=True, return_tensors="pt") | |
| head = (max_len * 3) // 4 | |
| rows = [] | |
| for t in texts: | |
| ids = tok(t, truncation=False)["input_ids"] | |
| if len(ids) > max_len: | |
| ids = ids[:head] + ids[len(ids) - (max_len - head):] | |
| rows.append(ids) | |
| width = max(len(r) for r in rows) | |
| pad = tok.pad_token_id if tok.pad_token_id is not None else 0 | |
| return {"input_ids": torch.tensor([r + [pad] * (width - len(r)) for r in rows]), | |
| "attention_mask": torch.tensor([[1] * len(r) + [0] * (width - len(r)) for r in rows])} | |
| def _logit(p): | |
| p = np.clip(p, 1e-6, 1 - 1e-6) | |
| return np.log(p / (1 - p)) | |
| def _sha256(path): | |
| import hashlib | |
| h = hashlib.sha256() | |
| with open(path, "rb") as f: | |
| for block in iter(lambda: f.read(1 << 20), b""): | |
| h.update(block) | |
| return h.hexdigest() | |
| class Bengu1: | |
| def __init__(self, package_dir, device="cpu"): | |
| from safetensors.torch import load_file | |
| from transformers import AutoConfig, AutoModel, AutoTokenizer | |
| root = Path(package_dir) | |
| self.spec = json.loads((root / "ensemble.json").read_text()) | |
| self.labels = self.spec["labels"] | |
| self.members = [] | |
| for m in self.spec["members"]: | |
| d = root / m["dir"] | |
| if _sha256(d / "model.safetensors") != m["model_sha256"]: | |
| raise RuntimeError(f"{m['dir']}/model.safetensors does not match its sha256") | |
| encoder = AutoModel.from_config(AutoConfig.from_pretrained(d / "encoder_config")) | |
| model = Head(encoder, len(self.labels), m["pooling"]).float() | |
| model.load_state_dict(load_file(str(d / "model.safetensors")), strict=True) | |
| model.to(device).eval() | |
| self.members.append((model, AutoTokenizer.from_pretrained(d / "tokenizer"), m)) | |
| def probs(self, texts, batch=32): | |
| parts = [] | |
| for model, tok, m in self.members: | |
| device = next(model.parameters()).device | |
| t = model.temperature.cpu().tolist() | |
| cat, prot, reply = [], [], [] | |
| for i in range(0, len(texts), batch): | |
| enc = encode(tok, texts[i:i + batch], m["max_len"], m["head_tail"]) | |
| c, p, r = model(enc["input_ids"].to(device), enc["attention_mask"].to(device)) | |
| cat.append(torch.softmax(c.float().cpu() / t[0], -1).numpy()) | |
| prot.append(torch.sigmoid(p.float().cpu() / t[1]).numpy()) | |
| reply.append(torch.sigmoid(r.float().cpu() / t[2]).numpy()) | |
| parts.append({"cat": np.concatenate(cat), "prot": np.concatenate(prot), "reply": np.concatenate(reply)}) | |
| avg = {k: np.mean([x[k] for x in parts], axis=0) for k in ("cat", "prot", "reply")} | |
| if self.spec.get("protected_combination", "mean") == "max": | |
| avg["prot"] = np.max([x["prot"] for x in parts], axis=0) | |
| t = self.spec["temperatures"] | |
| logp = np.log(np.clip(avg["cat"], 1e-9, 1.0)) / t[0] | |
| cat = np.exp(logp - logp.max(1, keepdims=True)) | |
| out = {"cat": cat / cat.sum(1, keepdims=True), "prot": 1 / (1 + np.exp(-_logit(avg["prot"]) / t[1])), | |
| "reply": 1 / (1 + np.exp(-_logit(avg["reply"]) / t[2]))} | |
| stk = self.spec.get("stacker") | |
| if stk: | |
| X = np.concatenate([np.log(np.clip(x["cat"], 1e-9, 1.0)) for x in parts], axis=1) | |
| z = X @ np.asarray(stk["W"]).T + np.asarray(stk["b"]) | |
| z = np.exp(z - z.max(1, keepdims=True)) | |
| out["cat"] = z / z.sum(1, keepdims=True) | |
| return out | |
| def predict(self, texts, batch=32): | |
| p = self.probs(texts, batch) | |
| always = set(self.spec.get("protected_categories", ())) | |
| out = [] | |
| for j in range(len(texts)): | |
| cat = self.labels[int(p["cat"][j].argmax())] | |
| out.append({"category": cat, | |
| "protected": (bool(p["prot"][j] >= self.spec["protected_threshold"]) and cat not in JUNK) | |
| or cat in always, | |
| "needs_reply": bool(p["reply"][j] >= self.spec["needs_reply_threshold"]), | |
| "confidence": float(p["cat"][j].max())}) | |
| return out | |
| def classify_raw(self, raw: bytes): | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from bengu_view import view | |
| return self.predict([view(raw)])[0] | |
| if __name__ == "__main__": | |
| model = Bengu1(sys.argv[1]) | |
| for path in sys.argv[2:]: | |
| print(json.dumps({"file": path, **model.classify_raw(Path(path).read_bytes())})) | |