Akash212 commited on
Commit
54df73d
·
verified ·
1 Parent(s): 77e2691
README.md CHANGED
@@ -11,23 +11,20 @@ The ensemble weights and threshold were chosen with HABC on validation data
11
  (minimax objective).
12
 
13
  ## Test results (5,874 texts, never used for any choice)
14
- accuracy 0.7755 | macro-F1 0.7703 | worst-corpus macro-F1 0.5671 | ROC-AUC 0.8531
15
 
16
  | corpus | n | accuracy | macro-F1 |
17
  |---|---|---|---|
18
- | davidson2017 | 896 | 0.9241 | 0.8694 |
19
- | dynabench2021 | 900 | 0.7322 | 0.7321 |
20
- | ethos2022 | 150 | 0.7800 | 0.7702 |
21
- | hateval2019_en | 150 | 0.7800 | 0.7764 |
22
- | hateval2019_es | 75 | 0.7867 | 0.7720 |
23
- | hatexplain2021 | 899 | 0.7531 | 0.7398 |
24
- | local_english | 900 | 0.7411 | 0.7363 |
25
- | local_hindi | 899 | 0.6385 | 0.6049 |
26
- | local_hinglish | 684 | 0.6886 | 0.5702 |
27
- | mlma2019_ar | 492 | 0.7642 | 0.7117 |
28
- | mlma2019_en | 819 | 0.8620 | 0.5671 |
29
- | mlma2019_fr | 593 | 0.7909 | 0.6611 |
30
- | romanurdu2020 | 900 | 0.8456 | 0.8455 |
31
 
32
  ## Use
33
  ```python
 
11
  (minimax objective).
12
 
13
  ## Test results (5,874 texts, never used for any choice)
14
+ accuracy 0.7673 | macro-F1 0.7386 | worst-corpus macro-F1 0.5312 | ROC-AUC 0.8264
15
 
16
  | corpus | n | accuracy | macro-F1 |
17
  |---|---|---|---|
18
+ | davidson2017 | 896 | 0.9118 | 0.8486 |
19
+ | dynabench2021 | 900 | 0.6044 | 0.6042 |
20
+ | ethos2022 | 150 | 0.6667 | 0.6563 |
21
+ | hateval2019_en | 150 | 0.6867 | 0.6791 |
22
+ | hateval2019_es | 75 | 0.6267 | 0.5879 |
23
+ | hatexplain2021 | 899 | 0.6841 | 0.6714 |
24
+ | mlma2019_ar | 492 | 0.7764 | 0.7180 |
25
+ | mlma2019_en | 819 | 0.8449 | 0.5312 |
26
+ | mlma2019_fr | 593 | 0.7707 | 0.6035 |
27
+ | romanurdu2020 | 900 | 0.8333 | 0.8333 |
 
 
 
28
 
29
  ## Use
30
  ```python
config.json CHANGED
@@ -1,59 +1,51 @@
1
- {
2
- "threshold": 0.5472934029255663,
3
- "learners": [
4
- {
5
- "name": "HPT_LIN_NB-SVM_c36",
6
- "file": "learner_00.joblib",
7
- "weight": 0.3166997041296016,
8
- "kind": "nbsvm",
9
- "view": "c36"
10
- },
11
- {
12
- "name": "LR_c25_C0.5",
13
- "file": "learner_01.joblib",
14
- "weight": 0.008904331670713563,
15
- "kind": "lr",
16
- "view": "c25"
17
- },
18
- {
19
- "name": "NBSVM_c25",
20
- "file": "learner_02.joblib",
21
- "weight": 0.24215469271926385,
22
- "kind": "nbsvm",
23
- "view": "c25"
24
- },
25
- {
26
- "name": "NN_MLP",
27
- "file": "learner_03.joblib",
28
- "weight": 0.11410717530325441,
29
- "kind": "mlp"
30
- },
31
- {
32
- "name": "TF_XLM_ROBERTA_BASE",
33
- "kind": "tf",
34
- "folder": "TF_XLM_ROBERTA_BASE",
35
- "weight": 0.31813409617716676
36
- }
37
- ],
38
- "transformers": {
39
- "TF_XLM_ROBERTA_BASE": "xlm-roberta-base"
40
- },
41
- "tf_max_len": 128,
42
- "label_names": {
43
- "0": "neither",
44
- "1": "abusive or hateful"
45
- },
46
- "test_metrics": {
47
- "accuracy": 0.7755,
48
- "macro_f1": 0.7703,
49
- "worst_corpus_macro_f1": 0.5671,
50
- "roc_auc": 0.8531,
51
- "pr_auc": 0.8893
52
- },
53
- "versions": {
54
- "python": "3.13.16",
55
- "numpy": "2.1.3",
56
- "sklearn": "1.6.1",
57
- "lightgbm": "4.6.0"
58
- }
59
  }
 
1
+ {
2
+ "threshold": 0.5293249358841979,
3
+ "learners": [
4
+ {
5
+ "name": "LR_w12_C4.0",
6
+ "file": "learner_00.joblib",
7
+ "weight": 0.5506429024746459,
8
+ "kind": "lr",
9
+ "view": "w12"
10
+ },
11
+ {
12
+ "name": "NBLR_c25_C4.0",
13
+ "file": "learner_01.joblib",
14
+ "weight": 0.010877087742393286,
15
+ "kind": "nblr",
16
+ "view": "c25"
17
+ },
18
+ {
19
+ "name": "NBLR_w12_C4.0",
20
+ "file": "learner_02.joblib",
21
+ "weight": 0.04974746258049213,
22
+ "kind": "nblr",
23
+ "view": "w12"
24
+ },
25
+ {
26
+ "name": "TREE_LGBM",
27
+ "file": "learner_03.joblib",
28
+ "weight": 0.3887325472024688,
29
+ "kind": "lgbm"
30
+ }
31
+ ],
32
+ "transformers": {},
33
+ "tf_max_len": 128,
34
+ "label_names": {
35
+ "0": "neither",
36
+ "1": "abusive or hateful"
37
+ },
38
+ "test_metrics": {
39
+ "accuracy": 0.7673,
40
+ "macro_f1": 0.7386,
41
+ "worst_corpus_macro_f1": 0.5312,
42
+ "roc_auc": 0.8264,
43
+ "pr_auc": 0.8984
44
+ },
45
+ "versions": {
46
+ "python": "3.12.6",
47
+ "numpy": "2.2.4",
48
+ "sklearn": "1.8.0",
49
+ "lightgbm": "4.6.0"
50
+ }
 
 
 
 
 
 
 
 
51
  }
dense_scaler.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:473eaad6a496da9a7244e01aaf5ff772e01d1e3d73b210c4f1dff896c26a2e21
3
- size 6855
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef1fa1d89402e32a3d73936e055184f086ce74ff8b3e98642c74633eb31f19af
3
+ size 6823
learner_00.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:34145099d031b499595fa4e39897b5ac3f5f8d01e2c831da6f220929c543d73f
3
- size 902620
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2672fa85b28c12172fae881d4d7bf2ebec94add6b7eab6885dde31edbec43aee
3
+ size 255499
learner_01.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4118d3b1ddfb1a8763ef57357ef5d897dc6e831f172860c06f190b7f3dfb81d6
3
- size 798891
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7af998fb8020b9c3b8cf3e461e52b0f1f23eb45b8ae0e9bf1659ef625e459ca2
3
+ size 531611
learner_02.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2873ac7150fb678a098c54b3c81ce7df89a8cbf9acbe25c04290224aee78a891
3
- size 798747
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd2a730aa94fe52f28a06e5f12a3f27e7870c7484fe4b3a98f4efa027524af7f
3
+ size 255499
learner_03.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9b202bf5247a9047ffba7f63e08cee16bff7666e1cc67cc3275917871a1bf061
3
- size 2503185
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5350829a54845581c0fc00395dd4db966d0efbd71e7e73c9222e9310e54252a5
3
+ size 2199780
lexicon.csv CHANGED
@@ -1,207 +1,207 @@
1
- word,severity
2
- badir,1
3
- badirchand,1
4
- bakland,1
5
- bhadva,2
6
- bhootnika,3
7
- chinaal,3
8
- chup,1
9
- chutia,5
10
- ghasti,4
11
- chutiya,5
12
- haraami,5
13
- haraam,5
14
- hijra,3
15
- hinjda,3
16
- jaanvar,1
17
- kutta,2
18
- kutiya,3
19
- khota,1
20
- auladheen,2
21
- jaat,1
22
- najayaz,3
23
- gandpaidaish,2
24
- saala,2
25
- kutti,2
26
- soover,3
27
- tatti,3
28
- potty,3
29
- bahenchod,9
30
- bahanchod,9
31
- bahencho,9
32
- bancho,7
33
- bahenke,6
34
- laude,7
35
- takke,7
36
- betichod,9
37
- bhaichod,8
38
- bhains,1
39
- jhalla,6
40
- jhant,7
41
- nabaal,4
42
- pissu,3
43
- kutte,2
44
- maadherchod,10
45
- madarchod,10
46
- padma,6
47
- raand,9
48
- jamai,5
49
- randwa,8
50
- randi,8
51
- bachachod,8
52
- bachichod,8
53
- soower,3
54
- bachchechod,8
55
- ullu,1
56
- pathe,1
57
- banda,7
58
- booblay,7
59
- booby,7
60
- buble,7
61
- babla,7
62
- bhonsriwala,8
63
- bhonsdiwala,8
64
- ched,8
65
- chut,8
66
- chod,7
67
- chodu,7
68
- chodra,7
69
- choochi,8
70
- chuchi,8
71
- gaandu,9
72
- gandu,9
73
- gaand,9
74
- lavda,8
75
- lawda,8
76
- lauda,8
77
- lund,8
78
- balchod,7
79
- lavander,8
80
- muth,7
81
- maacho,10
82
- mammey,8
83
- tatte,8
84
- toto,8
85
- toota,7
86
- backar,2
87
- bhandwe,7
88
- bhosadchod,9
89
- bhosad,8
90
- bumchod,9
91
- bum,8
92
- bur,9
93
- chatani,1
94
- cunt,8
95
- cuntmama,8
96
- chipkali,1
97
- pasine,1
98
- jhaat,8
99
- chodela,6
100
- bhagatchod,7
101
- chhola,7
102
- chudai,7
103
- chudaikhana,9
104
- chunni,8
105
- choot,8
106
- bhoot,1
107
- dhakkan,1
108
- bhajiye,1
109
- fateychu,9
110
- gandnatije,2
111
- lundtopi,4
112
- gaandfat,8
113
- gaandmasti,8
114
- makhanchudai,8
115
- gaandmarau,9
116
- chaatu,6
117
- beej,6
118
- choosu,7
119
- fakeerchod,8
120
- lundoos,8
121
- shorba,7
122
- binbheja,3
123
- bhadwe,6
124
- parichod,9
125
- nirodh,5
126
- pucchi,8
127
- baajer,8
128
- choud,8
129
- bhosda,9
130
- sadi,5
131
- choos,5
132
- maka,7
133
- gadde,7
134
- joon,3
135
- chullugand,4
136
- doob,1
137
- khatmal,1
138
- gandkate,6
139
- bambu,4
140
- lassan,1
141
- danda,2
142
- keera,2
143
- keeda,2
144
- hazaarchu,7
145
- paidaishikeeda,5
146
- kali,5
147
- safaid,2
148
- poot,2
149
- behendi,5
150
- chus,6
151
- machudi,10
152
- chodoonga,8
153
- baapchu,9
154
- laltern,5
155
- suhaagchudai,8
156
- raatchuda,7
157
- kaalu,6
158
- neech,7
159
- chikna,6
160
- meetha,6
161
- beechka,6
162
- chooche,8
163
- patichod,8
164
- rundi,7
165
- makkhi,1
166
- biwichod,9
167
- chodhunga,8
168
- haathi,1
169
- kute,2
170
- jhanten,8
171
- kaat,3
172
- gandi,3
173
- gadha,1
174
- bimaar,2
175
- badboodar,2
176
- dum,2
177
- raandsaala,7
178
- phudi,7
179
- chute,7
180
- kussi,7
181
- khandanchod,9
182
- ghussa,6
183
- maarey,4
184
- chipkili,1
185
- unday,1
186
- budh,7
187
- chaarpai,1
188
- chodun,5
189
- chatri,3
190
- chode,6
191
- chodho,6
192
- mullekatue,6
193
- mullikatui,6
194
- mullekebaal,6
195
- momedankatue,6
196
- katua,8
197
- chutiyapa,8
198
- bc,10
199
- mc,10
200
- chudwaya,7
201
- kutton,2
202
- jungli,2
203
- vahiyaat,4
204
- jihadi,4
205
- atankvadi,4
206
- atankwadi,4
207
- aatanki,4
 
1
+ word,severity
2
+ badir,1
3
+ badirchand,1
4
+ bakland,1
5
+ bhadva,2
6
+ bhootnika,3
7
+ chinaal,3
8
+ chup,1
9
+ chutia,5
10
+ ghasti,4
11
+ chutiya,5
12
+ haraami,5
13
+ haraam,5
14
+ hijra,3
15
+ hinjda,3
16
+ jaanvar,1
17
+ kutta,2
18
+ kutiya,3
19
+ khota,1
20
+ auladheen,2
21
+ jaat,1
22
+ najayaz,3
23
+ gandpaidaish,2
24
+ saala,2
25
+ kutti,2
26
+ soover,3
27
+ tatti,3
28
+ potty,3
29
+ bahenchod,9
30
+ bahanchod,9
31
+ bahencho,9
32
+ bancho,7
33
+ bahenke,6
34
+ laude,7
35
+ takke,7
36
+ betichod,9
37
+ bhaichod,8
38
+ bhains,1
39
+ jhalla,6
40
+ jhant,7
41
+ nabaal,4
42
+ pissu,3
43
+ kutte,2
44
+ maadherchod,10
45
+ madarchod,10
46
+ padma,6
47
+ raand,9
48
+ jamai,5
49
+ randwa,8
50
+ randi,8
51
+ bachachod,8
52
+ bachichod,8
53
+ soower,3
54
+ bachchechod,8
55
+ ullu,1
56
+ pathe,1
57
+ banda,7
58
+ booblay,7
59
+ booby,7
60
+ buble,7
61
+ babla,7
62
+ bhonsriwala,8
63
+ bhonsdiwala,8
64
+ ched,8
65
+ chut,8
66
+ chod,7
67
+ chodu,7
68
+ chodra,7
69
+ choochi,8
70
+ chuchi,8
71
+ gaandu,9
72
+ gandu,9
73
+ gaand,9
74
+ lavda,8
75
+ lawda,8
76
+ lauda,8
77
+ lund,8
78
+ balchod,7
79
+ lavander,8
80
+ muth,7
81
+ maacho,10
82
+ mammey,8
83
+ tatte,8
84
+ toto,8
85
+ toota,7
86
+ backar,2
87
+ bhandwe,7
88
+ bhosadchod,9
89
+ bhosad,8
90
+ bumchod,9
91
+ bum,8
92
+ bur,9
93
+ chatani,1
94
+ cunt,8
95
+ cuntmama,8
96
+ chipkali,1
97
+ pasine,1
98
+ jhaat,8
99
+ chodela,6
100
+ bhagatchod,7
101
+ chhola,7
102
+ chudai,7
103
+ chudaikhana,9
104
+ chunni,8
105
+ choot,8
106
+ bhoot,1
107
+ dhakkan,1
108
+ bhajiye,1
109
+ fateychu,9
110
+ gandnatije,2
111
+ lundtopi,4
112
+ gaandfat,8
113
+ gaandmasti,8
114
+ makhanchudai,8
115
+ gaandmarau,9
116
+ chaatu,6
117
+ beej,6
118
+ choosu,7
119
+ fakeerchod,8
120
+ lundoos,8
121
+ shorba,7
122
+ binbheja,3
123
+ bhadwe,6
124
+ parichod,9
125
+ nirodh,5
126
+ pucchi,8
127
+ baajer,8
128
+ choud,8
129
+ bhosda,9
130
+ sadi,5
131
+ choos,5
132
+ maka,7
133
+ gadde,7
134
+ joon,3
135
+ chullugand,4
136
+ doob,1
137
+ khatmal,1
138
+ gandkate,6
139
+ bambu,4
140
+ lassan,1
141
+ danda,2
142
+ keera,2
143
+ keeda,2
144
+ hazaarchu,7
145
+ paidaishikeeda,5
146
+ kali,5
147
+ safaid,2
148
+ poot,2
149
+ behendi,5
150
+ chus,6
151
+ machudi,10
152
+ chodoonga,8
153
+ baapchu,9
154
+ laltern,5
155
+ suhaagchudai,8
156
+ raatchuda,7
157
+ kaalu,6
158
+ neech,7
159
+ chikna,6
160
+ meetha,6
161
+ beechka,6
162
+ chooche,8
163
+ patichod,8
164
+ rundi,7
165
+ makkhi,1
166
+ biwichod,9
167
+ chodhunga,8
168
+ haathi,1
169
+ kute,2
170
+ jhanten,8
171
+ kaat,3
172
+ gandi,3
173
+ gadha,1
174
+ bimaar,2
175
+ badboodar,2
176
+ dum,2
177
+ raandsaala,7
178
+ phudi,7
179
+ chute,7
180
+ kussi,7
181
+ khandanchod,9
182
+ ghussa,6
183
+ maarey,4
184
+ chipkili,1
185
+ unday,1
186
+ budh,7
187
+ chaarpai,1
188
+ chodun,5
189
+ chatri,3
190
+ chode,6
191
+ chodho,6
192
+ mullekatue,6
193
+ mullikatui,6
194
+ mullekebaal,6
195
+ momedankatue,6
196
+ katua,8
197
+ chutiyapa,8
198
+ bc,10
199
+ mc,10
200
+ chudwaya,7
201
+ kutton,2
202
+ jungli,2
203
+ vahiyaat,4
204
+ jihadi,4
205
+ atankvadi,4
206
+ atankwadi,4
207
+ aatanki,4
predict.py CHANGED
@@ -1,105 +1,105 @@
1
- # Inference for the multi-corpus hate speech ensemble.
2
- #
3
- # from predict import HateEnsemble
4
- # model = HateEnsemble("path/to/this/folder")
5
- # model.predict_proba(["some text"]) # probability of "abusive or hateful"
6
- # model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither
7
- import json, os, re, unicodedata
8
- import numpy as np, pandas as pd, joblib
9
- from scipy.sparse import diags, hstack
10
- from scipy.special import expit
11
-
12
- RE_URL = re.compile('http\\S+|www\\.\\S+')
13
- RE_MENT = re.compile('@\\w+')
14
- RE_HTML = re.compile('<[^>]+>')
15
- RE_HASH = re.compile('#(\\w+)')
16
- RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
17
- RE_REP = re.compile('(.)\\1{2,}')
18
- RE_WS = re.compile('\\s+')
19
- TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
20
- def normalise(t, translit=True):
21
- t = unicodedata.normalize("NFKC", str(t)).lower()
22
- t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
23
- t = RE_HASH.sub(r" \1 ", t)
24
- t = RE_KEEP.sub(" ", t)
25
- t = RE_REP.sub(r"\1\1", t)
26
- if translit:
27
- for pat, rep in TRANSLIT:
28
- t = pat.sub(rep, t)
29
- return RE_WS.sub(" ", t).strip()
30
-
31
-
32
- _RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
33
- def light_clean(t):
34
- t = unicodedata.normalize("NFKC", str(t))
35
- t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
36
- return re.sub(r"\s+", " ", t).strip()
37
-
38
- class HateEnsemble:
39
- def __init__(self, path):
40
- self.path = path
41
- self.cfg = json.load(open(os.path.join(path, "config.json")))
42
- self.threshold = self.cfg["threshold"]
43
- self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
44
- for f in os.listdir(path):
45
- if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
46
- if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
47
- for l in self.cfg["learners"]:
48
- if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
49
- lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
50
- self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
51
- self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
52
- self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None
53
-
54
- def _lexicon(self, ct):
55
- out = np.zeros((len(ct), 4), dtype=np.float32)
56
- for i, t in enumerate(ct):
57
- toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
58
- out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
59
- return out
60
-
61
- def _view(self, vn, ct, cache):
62
- if vn not in cache:
63
- cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
64
- if vn == "w12+c25" else self.vec[vn].transform(ct))
65
- return cache[vn]
66
-
67
- def _transformer(self, folder, texts):
68
- import torch
69
- from transformers import AutoTokenizer, AutoModelForSequenceClassification
70
- if folder not in self._tf:
71
- p = os.path.join(self.path, folder)
72
- self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
73
- tok, model = self._tf[folder]; out = []
74
- with torch.no_grad():
75
- for i in range(0, len(texts), 32):
76
- enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
77
- out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
78
- return np.concatenate(out)
79
-
80
- def predict_proba(self, texts):
81
- texts = [str(t) for t in texts]
82
- ct = [normalise(t) for t in texts]
83
- cache, total, dense = {}, np.zeros(len(texts)), None
84
- for l in self.cfg["learners"]:
85
- k = l["kind"]
86
- if k == "tf":
87
- p = self._transformer(l["folder"], [light_clean(t) for t in texts])
88
- elif k == "lex":
89
- p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
90
- elif k in ("lgbm", "mlp"):
91
- if dense is None:
92
- X = self._view("w12+c25", ct, cache)
93
- dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
94
- Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
95
- p = self.models[l["name"]].predict_proba(Xd)[:, 1]
96
- else:
97
- X = self._view(l["view"], ct, cache)
98
- if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
99
- m = self.models[l["name"]]
100
- p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
101
- total += l["weight"] * p
102
- return total
103
-
104
- def predict(self, texts):
105
- return (self.predict_proba(texts) > self.threshold).astype(int)
 
1
+ # Inference for the multi-corpus hate speech ensemble.
2
+ #
3
+ # from predict import HateEnsemble
4
+ # model = HateEnsemble("path/to/this/folder")
5
+ # model.predict_proba(["some text"]) # probability of "abusive or hateful"
6
+ # model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither
7
+ import json, os, re, unicodedata
8
+ import numpy as np, pandas as pd, joblib
9
+ from scipy.sparse import diags, hstack
10
+ from scipy.special import expit
11
+
12
+ RE_URL = re.compile('http\\S+|www\\.\\S+')
13
+ RE_MENT = re.compile('@\\w+')
14
+ RE_HTML = re.compile('<[^>]+>')
15
+ RE_HASH = re.compile('#(\\w+)')
16
+ RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
17
+ RE_REP = re.compile('(.)\\1{2,}')
18
+ RE_WS = re.compile('\\s+')
19
+ TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
20
+ def normalise(t, translit=True):
21
+ t = unicodedata.normalize("NFKC", str(t)).lower()
22
+ t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
23
+ t = RE_HASH.sub(r" \1 ", t)
24
+ t = RE_KEEP.sub(" ", t)
25
+ t = RE_REP.sub(r"\1\1", t)
26
+ if translit:
27
+ for pat, rep in TRANSLIT:
28
+ t = pat.sub(rep, t)
29
+ return RE_WS.sub(" ", t).strip()
30
+
31
+
32
+ _RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
33
+ def light_clean(t):
34
+ t = unicodedata.normalize("NFKC", str(t))
35
+ t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
36
+ return re.sub(r"\s+", " ", t).strip()
37
+
38
+ class HateEnsemble:
39
+ def __init__(self, path):
40
+ self.path = path
41
+ self.cfg = json.load(open(os.path.join(path, "config.json")))
42
+ self.threshold = self.cfg["threshold"]
43
+ self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
44
+ for f in os.listdir(path):
45
+ if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
46
+ if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
47
+ for l in self.cfg["learners"]:
48
+ if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
49
+ lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
50
+ self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
51
+ self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
52
+ self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None
53
+
54
+ def _lexicon(self, ct):
55
+ out = np.zeros((len(ct), 4), dtype=np.float32)
56
+ for i, t in enumerate(ct):
57
+ toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
58
+ out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
59
+ return out
60
+
61
+ def _view(self, vn, ct, cache):
62
+ if vn not in cache:
63
+ cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
64
+ if vn == "w12+c25" else self.vec[vn].transform(ct))
65
+ return cache[vn]
66
+
67
+ def _transformer(self, folder, texts):
68
+ import torch
69
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification
70
+ if folder not in self._tf:
71
+ p = os.path.join(self.path, folder)
72
+ self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
73
+ tok, model = self._tf[folder]; out = []
74
+ with torch.no_grad():
75
+ for i in range(0, len(texts), 32):
76
+ enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
77
+ out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
78
+ return np.concatenate(out)
79
+
80
+ def predict_proba(self, texts):
81
+ texts = [str(t) for t in texts]
82
+ ct = [normalise(t) for t in texts]
83
+ cache, total, dense = {}, np.zeros(len(texts)), None
84
+ for l in self.cfg["learners"]:
85
+ k = l["kind"]
86
+ if k == "tf":
87
+ p = self._transformer(l["folder"], [light_clean(t) for t in texts])
88
+ elif k == "lex":
89
+ p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
90
+ elif k in ("lgbm", "mlp"):
91
+ if dense is None:
92
+ X = self._view("w12+c25", ct, cache)
93
+ dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
94
+ Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
95
+ p = self.models[l["name"]].predict_proba(Xd)[:, 1]
96
+ else:
97
+ X = self._view(l["view"], ct, cache)
98
+ if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
99
+ m = self.models[l["name"]]
100
+ p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
101
+ total += l["weight"] * p
102
+ return total
103
+
104
+ def predict(self, texts):
105
+ return (self.predict_proba(texts) > self.threshold).astype(int)
ratio_c25.npy CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:00789477540683fa5678b1b553413dc360a9ec476c6bf903160f6791dbbf4bd8
3
- size 798128
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c4dc859419e1eb1b43050faa4c12a5b1cce08ce22aee111c73d51abb95dbbc6
3
+ size 530840
ratio_w12.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d96ca206896fe890521b704d75c9795cf0e0c6e99cc816ae5685d60c4f5342a6
3
+ size 254752
requirements.txt CHANGED
@@ -1,10 +1,7 @@
1
- numpy>=2.0,<3
2
- scikit-learn==1.6.1
3
- scipy
4
- pandas
5
- joblib
6
- lightgbm==4.6.0
7
- huggingface_hub
8
- torch
9
- transformers
10
- sentencepiece
 
1
+ numpy>=2.0,<3
2
+ scikit-learn==1.8.0
3
+ scipy
4
+ pandas
5
+ joblib
6
+ lightgbm==4.6.0
7
+ huggingface_hub
 
 
 
svd.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f544c7f3d1d30ec51b14bca11f462d081128a37ef9e5f65b441ba577f2a1b547
3
- size 151416567
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8301533d687497bd6bb6fb4cfc8fe73d5901426b9015229d81eb0825b836cdd3
3
+ size 100526839
vec_c25.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:91994b4ed2b673e6bd5a615a1e3af205caf52c431762fae2cd9c3d20e166aa92
3
- size 3212764
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4e0c5f852c7b564ac7881c60e20e5def25b7fa08ebe7f02224e843e30e67b67
3
+ size 2048455
vec_w12.joblib CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:30608a5108f3ccb04df4b795898a119cfa653ff212b026eace223051ac054e04
3
- size 1657793
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f3bc465a0078cc4a26d62081f89c2f244696cd8804b0da20579aacf528f47ec
3
+ size 1081664