# Inference for the multi-corpus hate speech ensemble. # # from predict import HateEnsemble # model = HateEnsemble("path/to/this/folder") # model.predict_proba(["some text"]) # probability of "abusive or hateful" # model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither import json, os, re, unicodedata import numpy as np, pandas as pd, joblib from scipy.sparse import diags, hstack from scipy.special import expit RE_URL = re.compile('http\\S+|www\\.\\S+') RE_MENT = re.compile('@\\w+') RE_HTML = re.compile('<[^>]+>') RE_HASH = re.compile('#(\\w+)') RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]') RE_REP = re.compile('(.)\\1{2,}') RE_WS = re.compile('\\s+') TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')] def normalise(t, translit=True): t = unicodedata.normalize("NFKC", str(t)).lower() t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t) t = RE_HASH.sub(r" \1 ", t) t = RE_KEEP.sub(" ", t) t = RE_REP.sub(r"\1\1", t) if translit: for pat, rep in TRANSLIT: t = pat.sub(rep, t) return RE_WS.sub(" ", t).strip() _RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+") def light_clean(t): t = unicodedata.normalize("NFKC", str(t)) t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t) return re.sub(r"\s+", " ", t).strip() class HateEnsemble: def __init__(self, path): self.path = path self.cfg = json.load(open(os.path.join(path, "config.json"))) self.threshold = self.cfg["threshold"] self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {} for f in os.listdir(path): if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f)) if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f)) for l in self.cfg["learners"]: if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"])) lex = pd.read_csv(os.path.join(path, "lexicon.csv")) self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int))) self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None def _lexicon(self, ct): out = np.zeros((len(ct), 4), dtype=np.float32) for i, t in enumerate(ct): toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev] out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1)) return out def _view(self, vn, ct, cache): if vn not in cache: cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr() if vn == "w12+c25" else self.vec[vn].transform(ct)) return cache[vn] def _transformer(self, folder, texts): import torch from transformers import AutoTokenizer, AutoModelForSequenceClassification if folder not in self._tf: p = os.path.join(self.path, folder) self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval()) tok, model = self._tf[folder]; out = [] with torch.no_grad(): for i in range(0, len(texts), 32): enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt") out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy()) return np.concatenate(out) def predict_proba(self, texts): texts = [str(t) for t in texts] ct = [normalise(t) for t in texts] cache, total, dense = {}, np.zeros(len(texts)), None for l in self.cfg["learners"]: k = l["kind"] if k == "tf": p = self._transformer(l["folder"], [light_clean(t) for t in texts]) elif k == "lex": p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1] elif k in ("lgbm", "mlp"): if dense is None: X = self._view("w12+c25", ct, cache) dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32) Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32) p = self.models[l["name"]].predict_proba(Xd)[:, 1] else: X = self._view(l["view"], ct, cache) if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]]) m = self.models[l["name"]] p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1] total += l["weight"] * p return total def predict(self, texts): return (self.predict_proba(texts) > self.threshold).astype(int)