bengu-1 / inference /bengu_infer.py
uzay-eth's picture
Bengu-1 package r4 (bengu repo 1456a0f)
e79f133 verified
Raw History Blame Contribute Delete
6.78 kB
"""Bengu-1 inference: e-mail category, protected and needs_reply, on the CPU.
Needs: torch, transformers, safetensors, numpy (and bengu_view.py next to this file for raw mail).
python bengu_infer.py <package_dir> message.eml [more.eml ...]
from bengu_infer import Bengu1
model = Bengu1("<package_dir>")
model.classify_raw(open("message.eml", "rb").read())
model.predict(["<an already built input view>"])
The package holds one or two member encoders (members/<name>/: model.safetensors, encoder_config/,
tokenizer/ as trained, member.json) and ensemble.json (labels, temperatures, thresholds, combination, the
optional category stacker). Each member's probabilities are temperature-scaled, averaged (protected: mean or
max), calibrated again, and the stacker, when present, gives the category. protected is never set for spam
or phishing; needs_reply and protected use the thresholds in ensemble.json.
"""
import json
import sys
from pathlib import Path
import numpy as np
import torch
from torch import nn
JUNK = ("spam", "phishing_or_scam")
class Head(nn.Module):
def __init__(self, encoder, n_cat, pooling="mean"):
super().__init__()
self.encoder = encoder
hidden = encoder.config.hidden_size
self.pooling = pooling
self.drop = nn.Dropout(0.1)
self.cat = nn.Linear(hidden, n_cat)
self.prot = nn.Linear(hidden, 1)
self.reply = nn.Linear(hidden, 1)
self.register_buffer("temperature", torch.ones(3))
def forward(self, input_ids, attention_mask):
h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
if self.pooling == "cls":
pooled = h[:, 0]
else:
m = attention_mask.unsqueeze(-1).to(h.dtype)
pooled = (h * m).sum(1) / m.sum(1).clamp_min(1.0)
pooled = self.drop(pooled)
return self.cat(pooled), self.prot(pooled).squeeze(-1), self.reply(pooled).squeeze(-1)
def encode(tok, texts, max_len, head_tail):
"""At most max_len tokens; head_tail keeps the first 3/4 and the last 1/4 of an over-long text."""
if not head_tail:
return tok(texts, truncation=True, max_length=max_len, padding=True, return_tensors="pt")
head = (max_len * 3) // 4
rows = []
for t in texts:
ids = tok(t, truncation=False)["input_ids"]
if len(ids) > max_len:
ids = ids[:head] + ids[len(ids) - (max_len - head):]
rows.append(ids)
width = max(len(r) for r in rows)
pad = tok.pad_token_id if tok.pad_token_id is not None else 0
return {"input_ids": torch.tensor([r + [pad] * (width - len(r)) for r in rows]),
"attention_mask": torch.tensor([[1] * len(r) + [0] * (width - len(r)) for r in rows])}
def _logit(p):
p = np.clip(p, 1e-6, 1 - 1e-6)
return np.log(p / (1 - p))
def _sha256(path):
import hashlib
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
class Bengu1:
def __init__(self, package_dir, device="cpu"):
from safetensors.torch import load_file
from transformers import AutoConfig, AutoModel, AutoTokenizer
root = Path(package_dir)
self.spec = json.loads((root / "ensemble.json").read_text())
self.labels = self.spec["labels"]
self.members = []
for m in self.spec["members"]:
d = root / m["dir"]
if _sha256(d / "model.safetensors") != m["model_sha256"]:
raise RuntimeError(f"{m['dir']}/model.safetensors does not match its sha256")
encoder = AutoModel.from_config(AutoConfig.from_pretrained(d / "encoder_config"))
model = Head(encoder, len(self.labels), m["pooling"]).float()
model.load_state_dict(load_file(str(d / "model.safetensors")), strict=True)
model.to(device).eval()
self.members.append((model, AutoTokenizer.from_pretrained(d / "tokenizer"), m))
@torch.no_grad()
def probs(self, texts, batch=32):
parts = []
for model, tok, m in self.members:
device = next(model.parameters()).device
t = model.temperature.cpu().tolist()
cat, prot, reply = [], [], []
for i in range(0, len(texts), batch):
enc = encode(tok, texts[i:i + batch], m["max_len"], m["head_tail"])
c, p, r = model(enc["input_ids"].to(device), enc["attention_mask"].to(device))
cat.append(torch.softmax(c.float().cpu() / t[0], -1).numpy())
prot.append(torch.sigmoid(p.float().cpu() / t[1]).numpy())
reply.append(torch.sigmoid(r.float().cpu() / t[2]).numpy())
parts.append({"cat": np.concatenate(cat), "prot": np.concatenate(prot), "reply": np.concatenate(reply)})
avg = {k: np.mean([x[k] for x in parts], axis=0) for k in ("cat", "prot", "reply")}
if self.spec.get("protected_combination", "mean") == "max":
avg["prot"] = np.max([x["prot"] for x in parts], axis=0)
t = self.spec["temperatures"]
logp = np.log(np.clip(avg["cat"], 1e-9, 1.0)) / t[0]
cat = np.exp(logp - logp.max(1, keepdims=True))
out = {"cat": cat / cat.sum(1, keepdims=True), "prot": 1 / (1 + np.exp(-_logit(avg["prot"]) / t[1])),
"reply": 1 / (1 + np.exp(-_logit(avg["reply"]) / t[2]))}
stk = self.spec.get("stacker")
if stk:
X = np.concatenate([np.log(np.clip(x["cat"], 1e-9, 1.0)) for x in parts], axis=1)
z = X @ np.asarray(stk["W"]).T + np.asarray(stk["b"])
z = np.exp(z - z.max(1, keepdims=True))
out["cat"] = z / z.sum(1, keepdims=True)
return out
def predict(self, texts, batch=32):
p = self.probs(texts, batch)
always = set(self.spec.get("protected_categories", ()))
out = []
for j in range(len(texts)):
cat = self.labels[int(p["cat"][j].argmax())]
out.append({"category": cat,
"protected": (bool(p["prot"][j] >= self.spec["protected_threshold"]) and cat not in JUNK)
or cat in always,
"needs_reply": bool(p["reply"][j] >= self.spec["needs_reply_threshold"]),
"confidence": float(p["cat"][j].max())})
return out
def classify_raw(self, raw: bytes):
sys.path.insert(0, str(Path(__file__).resolve().parent))
from bengu_view import view
return self.predict([view(raw)])[0]
if __name__ == "__main__":
model = Bengu1(sys.argv[1])
for path in sys.argv[2:]:
print(json.dumps({"file": path, **model.classify_raw(Path(path).read_bytes())}))