File size: 5,360 Bytes
54df73d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | # Inference for the multi-corpus hate speech ensemble.
#
# from predict import HateEnsemble
# model = HateEnsemble("path/to/this/folder")
# model.predict_proba(["some text"]) # probability of "abusive or hateful"
# model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither
import json, os, re, unicodedata
import numpy as np, pandas as pd, joblib
from scipy.sparse import diags, hstack
from scipy.special import expit
RE_URL = re.compile('http\\S+|www\\.\\S+')
RE_MENT = re.compile('@\\w+')
RE_HTML = re.compile('<[^>]+>')
RE_HASH = re.compile('#(\\w+)')
RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
RE_REP = re.compile('(.)\\1{2,}')
RE_WS = re.compile('\\s+')
TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
def normalise(t, translit=True):
t = unicodedata.normalize("NFKC", str(t)).lower()
t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
t = RE_HASH.sub(r" \1 ", t)
t = RE_KEEP.sub(" ", t)
t = RE_REP.sub(r"\1\1", t)
if translit:
for pat, rep in TRANSLIT:
t = pat.sub(rep, t)
return RE_WS.sub(" ", t).strip()
_RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
def light_clean(t):
t = unicodedata.normalize("NFKC", str(t))
t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
return re.sub(r"\s+", " ", t).strip()
class HateEnsemble:
def __init__(self, path):
self.path = path
self.cfg = json.load(open(os.path.join(path, "config.json")))
self.threshold = self.cfg["threshold"]
self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
for f in os.listdir(path):
if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
for l in self.cfg["learners"]:
if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None
def _lexicon(self, ct):
out = np.zeros((len(ct), 4), dtype=np.float32)
for i, t in enumerate(ct):
toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
return out
def _view(self, vn, ct, cache):
if vn not in cache:
cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
if vn == "w12+c25" else self.vec[vn].transform(ct))
return cache[vn]
def _transformer(self, folder, texts):
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
if folder not in self._tf:
p = os.path.join(self.path, folder)
self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
tok, model = self._tf[folder]; out = []
with torch.no_grad():
for i in range(0, len(texts), 32):
enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
return np.concatenate(out)
def predict_proba(self, texts):
texts = [str(t) for t in texts]
ct = [normalise(t) for t in texts]
cache, total, dense = {}, np.zeros(len(texts)), None
for l in self.cfg["learners"]:
k = l["kind"]
if k == "tf":
p = self._transformer(l["folder"], [light_clean(t) for t in texts])
elif k == "lex":
p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
elif k in ("lgbm", "mlp"):
if dense is None:
X = self._view("w12+c25", ct, cache)
dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
p = self.models[l["name"]].predict_proba(Xd)[:, 1]
else:
X = self._view(l["view"], ct, cache)
if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
m = self.models[l["name"]]
p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
total += l["weight"] * p
return total
def predict(self, texts):
return (self.predict_proba(texts) > self.threshold).astype(int)
|