Download predict.py from Akash212/multicorpus-hate-speech-ensemble: direct link, hf CLI and curl.
- Browser
- Download file 5.36 kB
-
https://huggingface.co/Akash212/multicorpus-hate-speech-ensemble/resolve/main/predict.py
- Command line
-
hf download hf://Akash212/multicorpus-hate-speech-ensemble/predict.py
-
curl -L -o predict.py https://huggingface.co/Akash212/multicorpus-hate-speech-ensemble/resolve/main/predict.py
5.36 kB
| # Inference for the multi-corpus hate speech ensemble. | |
| # | |
| # from predict import HateEnsemble | |
| # model = HateEnsemble("path/to/this/folder") | |
| # model.predict_proba(["some text"]) # probability of "abusive or hateful" | |
| # model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither | |
| import json, os, re, unicodedata | |
| import numpy as np, pandas as pd, joblib | |
| from scipy.sparse import diags, hstack | |
| from scipy.special import expit | |
| RE_URL = re.compile('http\\S+|www\\.\\S+') | |
| RE_MENT = re.compile('@\\w+') | |
| RE_HTML = re.compile('<[^>]+>') | |
| RE_HASH = re.compile('#(\\w+)') | |
| RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]') | |
| RE_REP = re.compile('(.)\\1{2,}') | |
| RE_WS = re.compile('\\s+') | |
| TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')] | |
| def normalise(t, translit=True): | |
| t = unicodedata.normalize("NFKC", str(t)).lower() | |
| t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t) | |
| t = RE_HASH.sub(r" \1 ", t) | |
| t = RE_KEEP.sub(" ", t) | |
| t = RE_REP.sub(r"\1\1", t) | |
| if translit: | |
| for pat, rep in TRANSLIT: | |
| t = pat.sub(rep, t) | |
| return RE_WS.sub(" ", t).strip() | |
| _RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+") | |
| def light_clean(t): | |
| t = unicodedata.normalize("NFKC", str(t)) | |
| t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t) | |
| return re.sub(r"\s+", " ", t).strip() | |
| class HateEnsemble: | |
| def __init__(self, path): | |
| self.path = path | |
| self.cfg = json.load(open(os.path.join(path, "config.json"))) | |
| self.threshold = self.cfg["threshold"] | |
| self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {} | |
| for f in os.listdir(path): | |
| if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f)) | |
| if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f)) | |
| for l in self.cfg["learners"]: | |
| if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"])) | |
| lex = pd.read_csv(os.path.join(path, "lexicon.csv")) | |
| self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int))) | |
| self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None | |
| self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None | |
| def _lexicon(self, ct): | |
| out = np.zeros((len(ct), 4), dtype=np.float32) | |
| for i, t in enumerate(ct): | |
| toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev] | |
| out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1)) | |
| return out | |
| def _view(self, vn, ct, cache): | |
| if vn not in cache: | |
| cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr() | |
| if vn == "w12+c25" else self.vec[vn].transform(ct)) | |
| return cache[vn] | |
| def _transformer(self, folder, texts): | |
| import torch | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
| if folder not in self._tf: | |
| p = os.path.join(self.path, folder) | |
| self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval()) | |
| tok, model = self._tf[folder]; out = [] | |
| with torch.no_grad(): | |
| for i in range(0, len(texts), 32): | |
| enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt") | |
| out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy()) | |
| return np.concatenate(out) | |
| def predict_proba(self, texts): | |
| texts = [str(t) for t in texts] | |
| ct = [normalise(t) for t in texts] | |
| cache, total, dense = {}, np.zeros(len(texts)), None | |
| for l in self.cfg["learners"]: | |
| k = l["kind"] | |
| if k == "tf": | |
| p = self._transformer(l["folder"], [light_clean(t) for t in texts]) | |
| elif k == "lex": | |
| p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1] | |
| elif k in ("lgbm", "mlp"): | |
| if dense is None: | |
| X = self._view("w12+c25", ct, cache) | |
| dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32) | |
| Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32) | |
| p = self.models[l["name"]].predict_proba(Xd)[:, 1] | |
| else: | |
| X = self._view(l["view"], ct, cache) | |
| if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]]) | |
| m = self.models[l["name"]] | |
| p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1] | |
| total += l["weight"] * p | |
| return total | |
| def predict(self, texts): | |
| return (self.predict_proba(texts) > self.threshold).astype(int) | |