"""Bengu-1 inference: e-mail category, protected and needs_reply, on the CPU. Needs: torch, transformers, safetensors, numpy (and bengu_view.py next to this file for raw mail). python bengu_infer.py message.eml [more.eml ...] from bengu_infer import Bengu1 model = Bengu1("") model.classify_raw(open("message.eml", "rb").read()) model.predict([""]) The package holds one or two member encoders (members//: model.safetensors, encoder_config/, tokenizer/ as trained, member.json) and ensemble.json (labels, temperatures, thresholds, combination, the optional category stacker). Each member's probabilities are temperature-scaled, averaged (protected: mean or max), calibrated again, and the stacker, when present, gives the category. protected is never set for spam or phishing; needs_reply and protected use the thresholds in ensemble.json. """ import json import sys from pathlib import Path import numpy as np import torch from torch import nn JUNK = ("spam", "phishing_or_scam") class Head(nn.Module): def __init__(self, encoder, n_cat, pooling="mean"): super().__init__() self.encoder = encoder hidden = encoder.config.hidden_size self.pooling = pooling self.drop = nn.Dropout(0.1) self.cat = nn.Linear(hidden, n_cat) self.prot = nn.Linear(hidden, 1) self.reply = nn.Linear(hidden, 1) self.register_buffer("temperature", torch.ones(3)) def forward(self, input_ids, attention_mask): h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state if self.pooling == "cls": pooled = h[:, 0] else: m = attention_mask.unsqueeze(-1).to(h.dtype) pooled = (h * m).sum(1) / m.sum(1).clamp_min(1.0) pooled = self.drop(pooled) return self.cat(pooled), self.prot(pooled).squeeze(-1), self.reply(pooled).squeeze(-1) def encode(tok, texts, max_len, head_tail): """At most max_len tokens; head_tail keeps the first 3/4 and the last 1/4 of an over-long text.""" if not head_tail: return tok(texts, truncation=True, max_length=max_len, padding=True, return_tensors="pt") head = (max_len * 3) // 4 rows = [] for t in texts: ids = tok(t, truncation=False)["input_ids"] if len(ids) > max_len: ids = ids[:head] + ids[len(ids) - (max_len - head):] rows.append(ids) width = max(len(r) for r in rows) pad = tok.pad_token_id if tok.pad_token_id is not None else 0 return {"input_ids": torch.tensor([r + [pad] * (width - len(r)) for r in rows]), "attention_mask": torch.tensor([[1] * len(r) + [0] * (width - len(r)) for r in rows])} def _logit(p): p = np.clip(p, 1e-6, 1 - 1e-6) return np.log(p / (1 - p)) def _sha256(path): import hashlib h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) return h.hexdigest() class Bengu1: def __init__(self, package_dir, device="cpu"): from safetensors.torch import load_file from transformers import AutoConfig, AutoModel, AutoTokenizer root = Path(package_dir) self.spec = json.loads((root / "ensemble.json").read_text()) self.labels = self.spec["labels"] self.members = [] for m in self.spec["members"]: d = root / m["dir"] if _sha256(d / "model.safetensors") != m["model_sha256"]: raise RuntimeError(f"{m['dir']}/model.safetensors does not match its sha256") encoder = AutoModel.from_config(AutoConfig.from_pretrained(d / "encoder_config")) model = Head(encoder, len(self.labels), m["pooling"]).float() model.load_state_dict(load_file(str(d / "model.safetensors")), strict=True) model.to(device).eval() self.members.append((model, AutoTokenizer.from_pretrained(d / "tokenizer"), m)) @torch.no_grad() def probs(self, texts, batch=32): parts = [] for model, tok, m in self.members: device = next(model.parameters()).device t = model.temperature.cpu().tolist() cat, prot, reply = [], [], [] for i in range(0, len(texts), batch): enc = encode(tok, texts[i:i + batch], m["max_len"], m["head_tail"]) c, p, r = model(enc["input_ids"].to(device), enc["attention_mask"].to(device)) cat.append(torch.softmax(c.float().cpu() / t[0], -1).numpy()) prot.append(torch.sigmoid(p.float().cpu() / t[1]).numpy()) reply.append(torch.sigmoid(r.float().cpu() / t[2]).numpy()) parts.append({"cat": np.concatenate(cat), "prot": np.concatenate(prot), "reply": np.concatenate(reply)}) avg = {k: np.mean([x[k] for x in parts], axis=0) for k in ("cat", "prot", "reply")} if self.spec.get("protected_combination", "mean") == "max": avg["prot"] = np.max([x["prot"] for x in parts], axis=0) t = self.spec["temperatures"] logp = np.log(np.clip(avg["cat"], 1e-9, 1.0)) / t[0] cat = np.exp(logp - logp.max(1, keepdims=True)) out = {"cat": cat / cat.sum(1, keepdims=True), "prot": 1 / (1 + np.exp(-_logit(avg["prot"]) / t[1])), "reply": 1 / (1 + np.exp(-_logit(avg["reply"]) / t[2]))} stk = self.spec.get("stacker") if stk: X = np.concatenate([np.log(np.clip(x["cat"], 1e-9, 1.0)) for x in parts], axis=1) z = X @ np.asarray(stk["W"]).T + np.asarray(stk["b"]) z = np.exp(z - z.max(1, keepdims=True)) out["cat"] = z / z.sum(1, keepdims=True) return out def predict(self, texts, batch=32): p = self.probs(texts, batch) always = set(self.spec.get("protected_categories", ())) out = [] for j in range(len(texts)): cat = self.labels[int(p["cat"][j].argmax())] out.append({"category": cat, "protected": (bool(p["prot"][j] >= self.spec["protected_threshold"]) and cat not in JUNK) or cat in always, "needs_reply": bool(p["reply"][j] >= self.spec["needs_reply_threshold"]), "confidence": float(p["cat"][j].max())}) return out def classify_raw(self, raw: bytes): sys.path.insert(0, str(Path(__file__).resolve().parent)) from bengu_view import view return self.predict([view(raw)])[0] if __name__ == "__main__": model = Bengu1(sys.argv[1]) for path in sys.argv[2:]: print(json.dumps({"file": path, **model.classify_raw(Path(path).read_bytes())}))