File size: 5,360 Bytes
54df73d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# Inference for the multi-corpus hate speech ensemble.
#
#     from predict import HateEnsemble
#     model = HateEnsemble("path/to/this/folder")
#     model.predict_proba(["some text"])   # probability of "abusive or hateful"
#     model.predict(["some text"])         # 1 = abusive or hateful, 0 = neither
import json, os, re, unicodedata
import numpy as np, pandas as pd, joblib
from scipy.sparse import diags, hstack
from scipy.special import expit

RE_URL = re.compile('http\\S+|www\\.\\S+')
RE_MENT = re.compile('@\\w+')
RE_HTML = re.compile('<[^>]+>')
RE_HASH = re.compile('#(\\w+)')
RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
RE_REP = re.compile('(.)\\1{2,}')
RE_WS = re.compile('\\s+')
TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
def normalise(t, translit=True):
    t = unicodedata.normalize("NFKC", str(t)).lower()
    t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
    t = RE_HASH.sub(r" \1 ", t)
    t = RE_KEEP.sub(" ", t)
    t = RE_REP.sub(r"\1\1", t)
    if translit:
        for pat, rep in TRANSLIT:
            t = pat.sub(rep, t)
    return RE_WS.sub(" ", t).strip()


_RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
def light_clean(t):
    t = unicodedata.normalize("NFKC", str(t))
    t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
    return re.sub(r"\s+", " ", t).strip()

class HateEnsemble:
    def __init__(self, path):
        self.path = path
        self.cfg = json.load(open(os.path.join(path, "config.json")))
        self.threshold = self.cfg["threshold"]
        self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
        for f in os.listdir(path):
            if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
            if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
        for l in self.cfg["learners"]:
            if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
        lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
        self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
        self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
        self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None

    def _lexicon(self, ct):
        out = np.zeros((len(ct), 4), dtype=np.float32)
        for i, t in enumerate(ct):
            toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
            out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
        return out

    def _view(self, vn, ct, cache):
        if vn not in cache:
            cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
                         if vn == "w12+c25" else self.vec[vn].transform(ct))
        return cache[vn]

    def _transformer(self, folder, texts):
        import torch
        from transformers import AutoTokenizer, AutoModelForSequenceClassification
        if folder not in self._tf:
            p = os.path.join(self.path, folder)
            self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
        tok, model = self._tf[folder]; out = []
        with torch.no_grad():
            for i in range(0, len(texts), 32):
                enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
                out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
        return np.concatenate(out)

    def predict_proba(self, texts):
        texts = [str(t) for t in texts]
        ct = [normalise(t) for t in texts]
        cache, total, dense = {}, np.zeros(len(texts)), None
        for l in self.cfg["learners"]:
            k = l["kind"]
            if k == "tf":
                p = self._transformer(l["folder"], [light_clean(t) for t in texts])
            elif k == "lex":
                p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
            elif k in ("lgbm", "mlp"):
                if dense is None:
                    X = self._view("w12+c25", ct, cache)
                    dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
                Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
                p = self.models[l["name"]].predict_proba(Xd)[:, 1]
            else:
                X = self._view(l["view"], ct, cache)
                if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
                m = self.models[l["name"]]
                p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
            total += l["weight"] * p
        return total

    def predict(self, texts):
        return (self.predict_proba(texts) > self.threshold).astype(int)