ensemble
Browse files- README.md +11 -14
- config.json +50 -58
- dense_scaler.joblib +2 -2
- learner_00.joblib +2 -2
- learner_01.joblib +2 -2
- learner_02.joblib +2 -2
- learner_03.joblib +2 -2
- lexicon.csv +207 -207
- predict.py +105 -105
- ratio_c25.npy +2 -2
- ratio_w12.npy +3 -0
- requirements.txt +7 -10
- svd.joblib +2 -2
- vec_c25.joblib +2 -2
- vec_w12.joblib +2 -2
README.md
CHANGED
|
@@ -11,23 +11,20 @@ The ensemble weights and threshold were chosen with HABC on validation data
|
|
| 11 |
(minimax objective).
|
| 12 |
|
| 13 |
## Test results (5,874 texts, never used for any choice)
|
| 14 |
-
accuracy 0.
|
| 15 |
|
| 16 |
| corpus | n | accuracy | macro-F1 |
|
| 17 |
|---|---|---|---|
|
| 18 |
-
| davidson2017 | 896 | 0.
|
| 19 |
-
| dynabench2021 | 900 | 0.
|
| 20 |
-
| ethos2022 | 150 | 0.
|
| 21 |
-
| hateval2019_en | 150 | 0.
|
| 22 |
-
| hateval2019_es | 75 | 0.
|
| 23 |
-
| hatexplain2021 | 899 | 0.
|
| 24 |
-
|
|
| 25 |
-
|
|
| 26 |
-
|
|
| 27 |
-
|
|
| 28 |
-
| mlma2019_en | 819 | 0.8620 | 0.5671 |
|
| 29 |
-
| mlma2019_fr | 593 | 0.7909 | 0.6611 |
|
| 30 |
-
| romanurdu2020 | 900 | 0.8456 | 0.8455 |
|
| 31 |
|
| 32 |
## Use
|
| 33 |
```python
|
|
|
|
| 11 |
(minimax objective).
|
| 12 |
|
| 13 |
## Test results (5,874 texts, never used for any choice)
|
| 14 |
+
accuracy 0.7673 | macro-F1 0.7386 | worst-corpus macro-F1 0.5312 | ROC-AUC 0.8264
|
| 15 |
|
| 16 |
| corpus | n | accuracy | macro-F1 |
|
| 17 |
|---|---|---|---|
|
| 18 |
+
| davidson2017 | 896 | 0.9118 | 0.8486 |
|
| 19 |
+
| dynabench2021 | 900 | 0.6044 | 0.6042 |
|
| 20 |
+
| ethos2022 | 150 | 0.6667 | 0.6563 |
|
| 21 |
+
| hateval2019_en | 150 | 0.6867 | 0.6791 |
|
| 22 |
+
| hateval2019_es | 75 | 0.6267 | 0.5879 |
|
| 23 |
+
| hatexplain2021 | 899 | 0.6841 | 0.6714 |
|
| 24 |
+
| mlma2019_ar | 492 | 0.7764 | 0.7180 |
|
| 25 |
+
| mlma2019_en | 819 | 0.8449 | 0.5312 |
|
| 26 |
+
| mlma2019_fr | 593 | 0.7707 | 0.6035 |
|
| 27 |
+
| romanurdu2020 | 900 | 0.8333 | 0.8333 |
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
## Use
|
| 30 |
```python
|
config.json
CHANGED
|
@@ -1,59 +1,51 @@
|
|
| 1 |
-
{
|
| 2 |
-
"threshold": 0.
|
| 3 |
-
"learners": [
|
| 4 |
-
{
|
| 5 |
-
"name": "
|
| 6 |
-
"file": "learner_00.joblib",
|
| 7 |
-
"weight": 0.
|
| 8 |
-
"kind": "
|
| 9 |
-
"view": "
|
| 10 |
-
},
|
| 11 |
-
{
|
| 12 |
-
"name": "
|
| 13 |
-
"file": "learner_01.joblib",
|
| 14 |
-
"weight": 0.
|
| 15 |
-
"kind": "
|
| 16 |
-
"view": "c25"
|
| 17 |
-
},
|
| 18 |
-
{
|
| 19 |
-
"name": "
|
| 20 |
-
"file": "learner_02.joblib",
|
| 21 |
-
"weight": 0.
|
| 22 |
-
"kind": "
|
| 23 |
-
"view": "
|
| 24 |
-
},
|
| 25 |
-
{
|
| 26 |
-
"name": "
|
| 27 |
-
"file": "learner_03.joblib",
|
| 28 |
-
"weight": 0.
|
| 29 |
-
"kind": "
|
| 30 |
-
}
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
"
|
| 39 |
-
"
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
"
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
"
|
| 48 |
-
"
|
| 49 |
-
"
|
| 50 |
-
|
| 51 |
-
"pr_auc": 0.8893
|
| 52 |
-
},
|
| 53 |
-
"versions": {
|
| 54 |
-
"python": "3.13.16",
|
| 55 |
-
"numpy": "2.1.3",
|
| 56 |
-
"sklearn": "1.6.1",
|
| 57 |
-
"lightgbm": "4.6.0"
|
| 58 |
-
}
|
| 59 |
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"threshold": 0.5293249358841979,
|
| 3 |
+
"learners": [
|
| 4 |
+
{
|
| 5 |
+
"name": "LR_w12_C4.0",
|
| 6 |
+
"file": "learner_00.joblib",
|
| 7 |
+
"weight": 0.5506429024746459,
|
| 8 |
+
"kind": "lr",
|
| 9 |
+
"view": "w12"
|
| 10 |
+
},
|
| 11 |
+
{
|
| 12 |
+
"name": "NBLR_c25_C4.0",
|
| 13 |
+
"file": "learner_01.joblib",
|
| 14 |
+
"weight": 0.010877087742393286,
|
| 15 |
+
"kind": "nblr",
|
| 16 |
+
"view": "c25"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"name": "NBLR_w12_C4.0",
|
| 20 |
+
"file": "learner_02.joblib",
|
| 21 |
+
"weight": 0.04974746258049213,
|
| 22 |
+
"kind": "nblr",
|
| 23 |
+
"view": "w12"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "TREE_LGBM",
|
| 27 |
+
"file": "learner_03.joblib",
|
| 28 |
+
"weight": 0.3887325472024688,
|
| 29 |
+
"kind": "lgbm"
|
| 30 |
+
}
|
| 31 |
+
],
|
| 32 |
+
"transformers": {},
|
| 33 |
+
"tf_max_len": 128,
|
| 34 |
+
"label_names": {
|
| 35 |
+
"0": "neither",
|
| 36 |
+
"1": "abusive or hateful"
|
| 37 |
+
},
|
| 38 |
+
"test_metrics": {
|
| 39 |
+
"accuracy": 0.7673,
|
| 40 |
+
"macro_f1": 0.7386,
|
| 41 |
+
"worst_corpus_macro_f1": 0.5312,
|
| 42 |
+
"roc_auc": 0.8264,
|
| 43 |
+
"pr_auc": 0.8984
|
| 44 |
+
},
|
| 45 |
+
"versions": {
|
| 46 |
+
"python": "3.12.6",
|
| 47 |
+
"numpy": "2.2.4",
|
| 48 |
+
"sklearn": "1.8.0",
|
| 49 |
+
"lightgbm": "4.6.0"
|
| 50 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
}
|
dense_scaler.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ef1fa1d89402e32a3d73936e055184f086ce74ff8b3e98642c74633eb31f19af
|
| 3 |
+
size 6823
|
learner_00.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2672fa85b28c12172fae881d4d7bf2ebec94add6b7eab6885dde31edbec43aee
|
| 3 |
+
size 255499
|
learner_01.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7af998fb8020b9c3b8cf3e461e52b0f1f23eb45b8ae0e9bf1659ef625e459ca2
|
| 3 |
+
size 531611
|
learner_02.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dd2a730aa94fe52f28a06e5f12a3f27e7870c7484fe4b3a98f4efa027524af7f
|
| 3 |
+
size 255499
|
learner_03.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5350829a54845581c0fc00395dd4db966d0efbd71e7e73c9222e9310e54252a5
|
| 3 |
+
size 2199780
|
lexicon.csv
CHANGED
|
@@ -1,207 +1,207 @@
|
|
| 1 |
-
word,severity
|
| 2 |
-
badir,1
|
| 3 |
-
badirchand,1
|
| 4 |
-
bakland,1
|
| 5 |
-
bhadva,2
|
| 6 |
-
bhootnika,3
|
| 7 |
-
chinaal,3
|
| 8 |
-
chup,1
|
| 9 |
-
chutia,5
|
| 10 |
-
ghasti,4
|
| 11 |
-
chutiya,5
|
| 12 |
-
haraami,5
|
| 13 |
-
haraam,5
|
| 14 |
-
hijra,3
|
| 15 |
-
hinjda,3
|
| 16 |
-
jaanvar,1
|
| 17 |
-
kutta,2
|
| 18 |
-
kutiya,3
|
| 19 |
-
khota,1
|
| 20 |
-
auladheen,2
|
| 21 |
-
jaat,1
|
| 22 |
-
najayaz,3
|
| 23 |
-
gandpaidaish,2
|
| 24 |
-
saala,2
|
| 25 |
-
kutti,2
|
| 26 |
-
soover,3
|
| 27 |
-
tatti,3
|
| 28 |
-
potty,3
|
| 29 |
-
bahenchod,9
|
| 30 |
-
bahanchod,9
|
| 31 |
-
bahencho,9
|
| 32 |
-
bancho,7
|
| 33 |
-
bahenke,6
|
| 34 |
-
laude,7
|
| 35 |
-
takke,7
|
| 36 |
-
betichod,9
|
| 37 |
-
bhaichod,8
|
| 38 |
-
bhains,1
|
| 39 |
-
jhalla,6
|
| 40 |
-
jhant,7
|
| 41 |
-
nabaal,4
|
| 42 |
-
pissu,3
|
| 43 |
-
kutte,2
|
| 44 |
-
maadherchod,10
|
| 45 |
-
madarchod,10
|
| 46 |
-
padma,6
|
| 47 |
-
raand,9
|
| 48 |
-
jamai,5
|
| 49 |
-
randwa,8
|
| 50 |
-
randi,8
|
| 51 |
-
bachachod,8
|
| 52 |
-
bachichod,8
|
| 53 |
-
soower,3
|
| 54 |
-
bachchechod,8
|
| 55 |
-
ullu,1
|
| 56 |
-
pathe,1
|
| 57 |
-
banda,7
|
| 58 |
-
booblay,7
|
| 59 |
-
booby,7
|
| 60 |
-
buble,7
|
| 61 |
-
babla,7
|
| 62 |
-
bhonsriwala,8
|
| 63 |
-
bhonsdiwala,8
|
| 64 |
-
ched,8
|
| 65 |
-
chut,8
|
| 66 |
-
chod,7
|
| 67 |
-
chodu,7
|
| 68 |
-
chodra,7
|
| 69 |
-
choochi,8
|
| 70 |
-
chuchi,8
|
| 71 |
-
gaandu,9
|
| 72 |
-
gandu,9
|
| 73 |
-
gaand,9
|
| 74 |
-
lavda,8
|
| 75 |
-
lawda,8
|
| 76 |
-
lauda,8
|
| 77 |
-
lund,8
|
| 78 |
-
balchod,7
|
| 79 |
-
lavander,8
|
| 80 |
-
muth,7
|
| 81 |
-
maacho,10
|
| 82 |
-
mammey,8
|
| 83 |
-
tatte,8
|
| 84 |
-
toto,8
|
| 85 |
-
toota,7
|
| 86 |
-
backar,2
|
| 87 |
-
bhandwe,7
|
| 88 |
-
bhosadchod,9
|
| 89 |
-
bhosad,8
|
| 90 |
-
bumchod,9
|
| 91 |
-
bum,8
|
| 92 |
-
bur,9
|
| 93 |
-
chatani,1
|
| 94 |
-
cunt,8
|
| 95 |
-
cuntmama,8
|
| 96 |
-
chipkali,1
|
| 97 |
-
pasine,1
|
| 98 |
-
jhaat,8
|
| 99 |
-
chodela,6
|
| 100 |
-
bhagatchod,7
|
| 101 |
-
chhola,7
|
| 102 |
-
chudai,7
|
| 103 |
-
chudaikhana,9
|
| 104 |
-
chunni,8
|
| 105 |
-
choot,8
|
| 106 |
-
bhoot,1
|
| 107 |
-
dhakkan,1
|
| 108 |
-
bhajiye,1
|
| 109 |
-
fateychu,9
|
| 110 |
-
gandnatije,2
|
| 111 |
-
lundtopi,4
|
| 112 |
-
gaandfat,8
|
| 113 |
-
gaandmasti,8
|
| 114 |
-
makhanchudai,8
|
| 115 |
-
gaandmarau,9
|
| 116 |
-
chaatu,6
|
| 117 |
-
beej,6
|
| 118 |
-
choosu,7
|
| 119 |
-
fakeerchod,8
|
| 120 |
-
lundoos,8
|
| 121 |
-
shorba,7
|
| 122 |
-
binbheja,3
|
| 123 |
-
bhadwe,6
|
| 124 |
-
parichod,9
|
| 125 |
-
nirodh,5
|
| 126 |
-
pucchi,8
|
| 127 |
-
baajer,8
|
| 128 |
-
choud,8
|
| 129 |
-
bhosda,9
|
| 130 |
-
sadi,5
|
| 131 |
-
choos,5
|
| 132 |
-
maka,7
|
| 133 |
-
gadde,7
|
| 134 |
-
joon,3
|
| 135 |
-
chullugand,4
|
| 136 |
-
doob,1
|
| 137 |
-
khatmal,1
|
| 138 |
-
gandkate,6
|
| 139 |
-
bambu,4
|
| 140 |
-
lassan,1
|
| 141 |
-
danda,2
|
| 142 |
-
keera,2
|
| 143 |
-
keeda,2
|
| 144 |
-
hazaarchu,7
|
| 145 |
-
paidaishikeeda,5
|
| 146 |
-
kali,5
|
| 147 |
-
safaid,2
|
| 148 |
-
poot,2
|
| 149 |
-
behendi,5
|
| 150 |
-
chus,6
|
| 151 |
-
machudi,10
|
| 152 |
-
chodoonga,8
|
| 153 |
-
baapchu,9
|
| 154 |
-
laltern,5
|
| 155 |
-
suhaagchudai,8
|
| 156 |
-
raatchuda,7
|
| 157 |
-
kaalu,6
|
| 158 |
-
neech,7
|
| 159 |
-
chikna,6
|
| 160 |
-
meetha,6
|
| 161 |
-
beechka,6
|
| 162 |
-
chooche,8
|
| 163 |
-
patichod,8
|
| 164 |
-
rundi,7
|
| 165 |
-
makkhi,1
|
| 166 |
-
biwichod,9
|
| 167 |
-
chodhunga,8
|
| 168 |
-
haathi,1
|
| 169 |
-
kute,2
|
| 170 |
-
jhanten,8
|
| 171 |
-
kaat,3
|
| 172 |
-
gandi,3
|
| 173 |
-
gadha,1
|
| 174 |
-
bimaar,2
|
| 175 |
-
badboodar,2
|
| 176 |
-
dum,2
|
| 177 |
-
raandsaala,7
|
| 178 |
-
phudi,7
|
| 179 |
-
chute,7
|
| 180 |
-
kussi,7
|
| 181 |
-
khandanchod,9
|
| 182 |
-
ghussa,6
|
| 183 |
-
maarey,4
|
| 184 |
-
chipkili,1
|
| 185 |
-
unday,1
|
| 186 |
-
budh,7
|
| 187 |
-
chaarpai,1
|
| 188 |
-
chodun,5
|
| 189 |
-
chatri,3
|
| 190 |
-
chode,6
|
| 191 |
-
chodho,6
|
| 192 |
-
mullekatue,6
|
| 193 |
-
mullikatui,6
|
| 194 |
-
mullekebaal,6
|
| 195 |
-
momedankatue,6
|
| 196 |
-
katua,8
|
| 197 |
-
chutiyapa,8
|
| 198 |
-
bc,10
|
| 199 |
-
mc,10
|
| 200 |
-
chudwaya,7
|
| 201 |
-
kutton,2
|
| 202 |
-
jungli,2
|
| 203 |
-
vahiyaat,4
|
| 204 |
-
jihadi,4
|
| 205 |
-
atankvadi,4
|
| 206 |
-
atankwadi,4
|
| 207 |
-
aatanki,4
|
|
|
|
| 1 |
+
word,severity
|
| 2 |
+
badir,1
|
| 3 |
+
badirchand,1
|
| 4 |
+
bakland,1
|
| 5 |
+
bhadva,2
|
| 6 |
+
bhootnika,3
|
| 7 |
+
chinaal,3
|
| 8 |
+
chup,1
|
| 9 |
+
chutia,5
|
| 10 |
+
ghasti,4
|
| 11 |
+
chutiya,5
|
| 12 |
+
haraami,5
|
| 13 |
+
haraam,5
|
| 14 |
+
hijra,3
|
| 15 |
+
hinjda,3
|
| 16 |
+
jaanvar,1
|
| 17 |
+
kutta,2
|
| 18 |
+
kutiya,3
|
| 19 |
+
khota,1
|
| 20 |
+
auladheen,2
|
| 21 |
+
jaat,1
|
| 22 |
+
najayaz,3
|
| 23 |
+
gandpaidaish,2
|
| 24 |
+
saala,2
|
| 25 |
+
kutti,2
|
| 26 |
+
soover,3
|
| 27 |
+
tatti,3
|
| 28 |
+
potty,3
|
| 29 |
+
bahenchod,9
|
| 30 |
+
bahanchod,9
|
| 31 |
+
bahencho,9
|
| 32 |
+
bancho,7
|
| 33 |
+
bahenke,6
|
| 34 |
+
laude,7
|
| 35 |
+
takke,7
|
| 36 |
+
betichod,9
|
| 37 |
+
bhaichod,8
|
| 38 |
+
bhains,1
|
| 39 |
+
jhalla,6
|
| 40 |
+
jhant,7
|
| 41 |
+
nabaal,4
|
| 42 |
+
pissu,3
|
| 43 |
+
kutte,2
|
| 44 |
+
maadherchod,10
|
| 45 |
+
madarchod,10
|
| 46 |
+
padma,6
|
| 47 |
+
raand,9
|
| 48 |
+
jamai,5
|
| 49 |
+
randwa,8
|
| 50 |
+
randi,8
|
| 51 |
+
bachachod,8
|
| 52 |
+
bachichod,8
|
| 53 |
+
soower,3
|
| 54 |
+
bachchechod,8
|
| 55 |
+
ullu,1
|
| 56 |
+
pathe,1
|
| 57 |
+
banda,7
|
| 58 |
+
booblay,7
|
| 59 |
+
booby,7
|
| 60 |
+
buble,7
|
| 61 |
+
babla,7
|
| 62 |
+
bhonsriwala,8
|
| 63 |
+
bhonsdiwala,8
|
| 64 |
+
ched,8
|
| 65 |
+
chut,8
|
| 66 |
+
chod,7
|
| 67 |
+
chodu,7
|
| 68 |
+
chodra,7
|
| 69 |
+
choochi,8
|
| 70 |
+
chuchi,8
|
| 71 |
+
gaandu,9
|
| 72 |
+
gandu,9
|
| 73 |
+
gaand,9
|
| 74 |
+
lavda,8
|
| 75 |
+
lawda,8
|
| 76 |
+
lauda,8
|
| 77 |
+
lund,8
|
| 78 |
+
balchod,7
|
| 79 |
+
lavander,8
|
| 80 |
+
muth,7
|
| 81 |
+
maacho,10
|
| 82 |
+
mammey,8
|
| 83 |
+
tatte,8
|
| 84 |
+
toto,8
|
| 85 |
+
toota,7
|
| 86 |
+
backar,2
|
| 87 |
+
bhandwe,7
|
| 88 |
+
bhosadchod,9
|
| 89 |
+
bhosad,8
|
| 90 |
+
bumchod,9
|
| 91 |
+
bum,8
|
| 92 |
+
bur,9
|
| 93 |
+
chatani,1
|
| 94 |
+
cunt,8
|
| 95 |
+
cuntmama,8
|
| 96 |
+
chipkali,1
|
| 97 |
+
pasine,1
|
| 98 |
+
jhaat,8
|
| 99 |
+
chodela,6
|
| 100 |
+
bhagatchod,7
|
| 101 |
+
chhola,7
|
| 102 |
+
chudai,7
|
| 103 |
+
chudaikhana,9
|
| 104 |
+
chunni,8
|
| 105 |
+
choot,8
|
| 106 |
+
bhoot,1
|
| 107 |
+
dhakkan,1
|
| 108 |
+
bhajiye,1
|
| 109 |
+
fateychu,9
|
| 110 |
+
gandnatije,2
|
| 111 |
+
lundtopi,4
|
| 112 |
+
gaandfat,8
|
| 113 |
+
gaandmasti,8
|
| 114 |
+
makhanchudai,8
|
| 115 |
+
gaandmarau,9
|
| 116 |
+
chaatu,6
|
| 117 |
+
beej,6
|
| 118 |
+
choosu,7
|
| 119 |
+
fakeerchod,8
|
| 120 |
+
lundoos,8
|
| 121 |
+
shorba,7
|
| 122 |
+
binbheja,3
|
| 123 |
+
bhadwe,6
|
| 124 |
+
parichod,9
|
| 125 |
+
nirodh,5
|
| 126 |
+
pucchi,8
|
| 127 |
+
baajer,8
|
| 128 |
+
choud,8
|
| 129 |
+
bhosda,9
|
| 130 |
+
sadi,5
|
| 131 |
+
choos,5
|
| 132 |
+
maka,7
|
| 133 |
+
gadde,7
|
| 134 |
+
joon,3
|
| 135 |
+
chullugand,4
|
| 136 |
+
doob,1
|
| 137 |
+
khatmal,1
|
| 138 |
+
gandkate,6
|
| 139 |
+
bambu,4
|
| 140 |
+
lassan,1
|
| 141 |
+
danda,2
|
| 142 |
+
keera,2
|
| 143 |
+
keeda,2
|
| 144 |
+
hazaarchu,7
|
| 145 |
+
paidaishikeeda,5
|
| 146 |
+
kali,5
|
| 147 |
+
safaid,2
|
| 148 |
+
poot,2
|
| 149 |
+
behendi,5
|
| 150 |
+
chus,6
|
| 151 |
+
machudi,10
|
| 152 |
+
chodoonga,8
|
| 153 |
+
baapchu,9
|
| 154 |
+
laltern,5
|
| 155 |
+
suhaagchudai,8
|
| 156 |
+
raatchuda,7
|
| 157 |
+
kaalu,6
|
| 158 |
+
neech,7
|
| 159 |
+
chikna,6
|
| 160 |
+
meetha,6
|
| 161 |
+
beechka,6
|
| 162 |
+
chooche,8
|
| 163 |
+
patichod,8
|
| 164 |
+
rundi,7
|
| 165 |
+
makkhi,1
|
| 166 |
+
biwichod,9
|
| 167 |
+
chodhunga,8
|
| 168 |
+
haathi,1
|
| 169 |
+
kute,2
|
| 170 |
+
jhanten,8
|
| 171 |
+
kaat,3
|
| 172 |
+
gandi,3
|
| 173 |
+
gadha,1
|
| 174 |
+
bimaar,2
|
| 175 |
+
badboodar,2
|
| 176 |
+
dum,2
|
| 177 |
+
raandsaala,7
|
| 178 |
+
phudi,7
|
| 179 |
+
chute,7
|
| 180 |
+
kussi,7
|
| 181 |
+
khandanchod,9
|
| 182 |
+
ghussa,6
|
| 183 |
+
maarey,4
|
| 184 |
+
chipkili,1
|
| 185 |
+
unday,1
|
| 186 |
+
budh,7
|
| 187 |
+
chaarpai,1
|
| 188 |
+
chodun,5
|
| 189 |
+
chatri,3
|
| 190 |
+
chode,6
|
| 191 |
+
chodho,6
|
| 192 |
+
mullekatue,6
|
| 193 |
+
mullikatui,6
|
| 194 |
+
mullekebaal,6
|
| 195 |
+
momedankatue,6
|
| 196 |
+
katua,8
|
| 197 |
+
chutiyapa,8
|
| 198 |
+
bc,10
|
| 199 |
+
mc,10
|
| 200 |
+
chudwaya,7
|
| 201 |
+
kutton,2
|
| 202 |
+
jungli,2
|
| 203 |
+
vahiyaat,4
|
| 204 |
+
jihadi,4
|
| 205 |
+
atankvadi,4
|
| 206 |
+
atankwadi,4
|
| 207 |
+
aatanki,4
|
predict.py
CHANGED
|
@@ -1,105 +1,105 @@
|
|
| 1 |
-
# Inference for the multi-corpus hate speech ensemble.
|
| 2 |
-
#
|
| 3 |
-
# from predict import HateEnsemble
|
| 4 |
-
# model = HateEnsemble("path/to/this/folder")
|
| 5 |
-
# model.predict_proba(["some text"]) # probability of "abusive or hateful"
|
| 6 |
-
# model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither
|
| 7 |
-
import json, os, re, unicodedata
|
| 8 |
-
import numpy as np, pandas as pd, joblib
|
| 9 |
-
from scipy.sparse import diags, hstack
|
| 10 |
-
from scipy.special import expit
|
| 11 |
-
|
| 12 |
-
RE_URL = re.compile('http\\S+|www\\.\\S+')
|
| 13 |
-
RE_MENT = re.compile('@\\w+')
|
| 14 |
-
RE_HTML = re.compile('<[^>]+>')
|
| 15 |
-
RE_HASH = re.compile('#(\\w+)')
|
| 16 |
-
RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
|
| 17 |
-
RE_REP = re.compile('(.)\\1{2,}')
|
| 18 |
-
RE_WS = re.compile('\\s+')
|
| 19 |
-
TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
|
| 20 |
-
def normalise(t, translit=True):
|
| 21 |
-
t = unicodedata.normalize("NFKC", str(t)).lower()
|
| 22 |
-
t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
|
| 23 |
-
t = RE_HASH.sub(r" \1 ", t)
|
| 24 |
-
t = RE_KEEP.sub(" ", t)
|
| 25 |
-
t = RE_REP.sub(r"\1\1", t)
|
| 26 |
-
if translit:
|
| 27 |
-
for pat, rep in TRANSLIT:
|
| 28 |
-
t = pat.sub(rep, t)
|
| 29 |
-
return RE_WS.sub(" ", t).strip()
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
_RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
|
| 33 |
-
def light_clean(t):
|
| 34 |
-
t = unicodedata.normalize("NFKC", str(t))
|
| 35 |
-
t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
|
| 36 |
-
return re.sub(r"\s+", " ", t).strip()
|
| 37 |
-
|
| 38 |
-
class HateEnsemble:
|
| 39 |
-
def __init__(self, path):
|
| 40 |
-
self.path = path
|
| 41 |
-
self.cfg = json.load(open(os.path.join(path, "config.json")))
|
| 42 |
-
self.threshold = self.cfg["threshold"]
|
| 43 |
-
self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
|
| 44 |
-
for f in os.listdir(path):
|
| 45 |
-
if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
|
| 46 |
-
if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
|
| 47 |
-
for l in self.cfg["learners"]:
|
| 48 |
-
if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
|
| 49 |
-
lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
|
| 50 |
-
self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
|
| 51 |
-
self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
|
| 52 |
-
self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None
|
| 53 |
-
|
| 54 |
-
def _lexicon(self, ct):
|
| 55 |
-
out = np.zeros((len(ct), 4), dtype=np.float32)
|
| 56 |
-
for i, t in enumerate(ct):
|
| 57 |
-
toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
|
| 58 |
-
out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
|
| 59 |
-
return out
|
| 60 |
-
|
| 61 |
-
def _view(self, vn, ct, cache):
|
| 62 |
-
if vn not in cache:
|
| 63 |
-
cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
|
| 64 |
-
if vn == "w12+c25" else self.vec[vn].transform(ct))
|
| 65 |
-
return cache[vn]
|
| 66 |
-
|
| 67 |
-
def _transformer(self, folder, texts):
|
| 68 |
-
import torch
|
| 69 |
-
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
| 70 |
-
if folder not in self._tf:
|
| 71 |
-
p = os.path.join(self.path, folder)
|
| 72 |
-
self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
|
| 73 |
-
tok, model = self._tf[folder]; out = []
|
| 74 |
-
with torch.no_grad():
|
| 75 |
-
for i in range(0, len(texts), 32):
|
| 76 |
-
enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
|
| 77 |
-
out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
|
| 78 |
-
return np.concatenate(out)
|
| 79 |
-
|
| 80 |
-
def predict_proba(self, texts):
|
| 81 |
-
texts = [str(t) for t in texts]
|
| 82 |
-
ct = [normalise(t) for t in texts]
|
| 83 |
-
cache, total, dense = {}, np.zeros(len(texts)), None
|
| 84 |
-
for l in self.cfg["learners"]:
|
| 85 |
-
k = l["kind"]
|
| 86 |
-
if k == "tf":
|
| 87 |
-
p = self._transformer(l["folder"], [light_clean(t) for t in texts])
|
| 88 |
-
elif k == "lex":
|
| 89 |
-
p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
|
| 90 |
-
elif k in ("lgbm", "mlp"):
|
| 91 |
-
if dense is None:
|
| 92 |
-
X = self._view("w12+c25", ct, cache)
|
| 93 |
-
dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
|
| 94 |
-
Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
|
| 95 |
-
p = self.models[l["name"]].predict_proba(Xd)[:, 1]
|
| 96 |
-
else:
|
| 97 |
-
X = self._view(l["view"], ct, cache)
|
| 98 |
-
if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
|
| 99 |
-
m = self.models[l["name"]]
|
| 100 |
-
p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
|
| 101 |
-
total += l["weight"] * p
|
| 102 |
-
return total
|
| 103 |
-
|
| 104 |
-
def predict(self, texts):
|
| 105 |
-
return (self.predict_proba(texts) > self.threshold).astype(int)
|
|
|
|
| 1 |
+
# Inference for the multi-corpus hate speech ensemble.
|
| 2 |
+
#
|
| 3 |
+
# from predict import HateEnsemble
|
| 4 |
+
# model = HateEnsemble("path/to/this/folder")
|
| 5 |
+
# model.predict_proba(["some text"]) # probability of "abusive or hateful"
|
| 6 |
+
# model.predict(["some text"]) # 1 = abusive or hateful, 0 = neither
|
| 7 |
+
import json, os, re, unicodedata
|
| 8 |
+
import numpy as np, pandas as pd, joblib
|
| 9 |
+
from scipy.sparse import diags, hstack
|
| 10 |
+
from scipy.special import expit
|
| 11 |
+
|
| 12 |
+
RE_URL = re.compile('http\\S+|www\\.\\S+')
|
| 13 |
+
RE_MENT = re.compile('@\\w+')
|
| 14 |
+
RE_HTML = re.compile('<[^>]+>')
|
| 15 |
+
RE_HASH = re.compile('#(\\w+)')
|
| 16 |
+
RE_KEEP = re.compile('[^\\w\\sऀ-ॿ\u0600-ۿ]')
|
| 17 |
+
RE_REP = re.compile('(.)\\1{2,}')
|
| 18 |
+
RE_WS = re.compile('\\s+')
|
| 19 |
+
TRANSLIT = [(re.compile('\\baa+\\b'), 'a'), (re.compile('ee+'), 'i'), (re.compile('oo+'), 'u'), (re.compile('\\bnhi\\b|\\bnahin\\b|\\bnahi+\\b'), 'nahi'), (re.compile('\\bh(a|ai|ain|e)\\b'), 'hai'), (re.compile('\\bkr\\b|\\bkar+\\b'), 'kar'), (re.compile('\\bq\\b|\\bkyu+\\b|\\bkyun\\b'), 'kyu')]
|
| 20 |
+
def normalise(t, translit=True):
|
| 21 |
+
t = unicodedata.normalize("NFKC", str(t)).lower()
|
| 22 |
+
t = RE_URL.sub(" ", t); t = RE_MENT.sub(" ", t); t = RE_HTML.sub(" ", t)
|
| 23 |
+
t = RE_HASH.sub(r" \1 ", t)
|
| 24 |
+
t = RE_KEEP.sub(" ", t)
|
| 25 |
+
t = RE_REP.sub(r"\1\1", t)
|
| 26 |
+
if translit:
|
| 27 |
+
for pat, rep in TRANSLIT:
|
| 28 |
+
t = pat.sub(rep, t)
|
| 29 |
+
return RE_WS.sub(" ", t).strip()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
_RE_URL2, _RE_MENT2 = re.compile(r"http\S+|www\.\S+"), re.compile(r"@\w+")
|
| 33 |
+
def light_clean(t):
|
| 34 |
+
t = unicodedata.normalize("NFKC", str(t))
|
| 35 |
+
t = _RE_URL2.sub(" URL ", t); t = _RE_MENT2.sub(" @user ", t)
|
| 36 |
+
return re.sub(r"\s+", " ", t).strip()
|
| 37 |
+
|
| 38 |
+
class HateEnsemble:
|
| 39 |
+
def __init__(self, path):
|
| 40 |
+
self.path = path
|
| 41 |
+
self.cfg = json.load(open(os.path.join(path, "config.json")))
|
| 42 |
+
self.threshold = self.cfg["threshold"]
|
| 43 |
+
self.vec, self.ratio, self.models, self._tf = {}, {}, {}, {}
|
| 44 |
+
for f in os.listdir(path):
|
| 45 |
+
if f.startswith("vec_"): self.vec[f[4:-7]] = joblib.load(os.path.join(path, f))
|
| 46 |
+
if f.startswith("ratio_"): self.ratio[f[6:-4]] = np.load(os.path.join(path, f))
|
| 47 |
+
for l in self.cfg["learners"]:
|
| 48 |
+
if "file" in l: self.models[l["name"]] = joblib.load(os.path.join(path, l["file"]))
|
| 49 |
+
lex = pd.read_csv(os.path.join(path, "lexicon.csv"))
|
| 50 |
+
self.sev = dict(zip(lex.word.astype(str), lex.severity.astype(int)))
|
| 51 |
+
self.svd = joblib.load(os.path.join(path, "svd.joblib")) if os.path.exists(os.path.join(path, "svd.joblib")) else None
|
| 52 |
+
self.scaler = joblib.load(os.path.join(path, "dense_scaler.joblib")) if self.svd is not None else None
|
| 53 |
+
|
| 54 |
+
def _lexicon(self, ct):
|
| 55 |
+
out = np.zeros((len(ct), 4), dtype=np.float32)
|
| 56 |
+
for i, t in enumerate(ct):
|
| 57 |
+
toks = t.split(); hits = [self.sev[w] for w in toks if w in self.sev]
|
| 58 |
+
out[i] = (len(hits), sum(hits), max(hits) if hits else 0, len(hits) / (len(toks) or 1))
|
| 59 |
+
return out
|
| 60 |
+
|
| 61 |
+
def _view(self, vn, ct, cache):
|
| 62 |
+
if vn not in cache:
|
| 63 |
+
cache[vn] = (hstack([self._view("w12", ct, cache), self._view("c25", ct, cache)]).tocsr()
|
| 64 |
+
if vn == "w12+c25" else self.vec[vn].transform(ct))
|
| 65 |
+
return cache[vn]
|
| 66 |
+
|
| 67 |
+
def _transformer(self, folder, texts):
|
| 68 |
+
import torch
|
| 69 |
+
from transformers import AutoTokenizer, AutoModelForSequenceClassification
|
| 70 |
+
if folder not in self._tf:
|
| 71 |
+
p = os.path.join(self.path, folder)
|
| 72 |
+
self._tf[folder] = (AutoTokenizer.from_pretrained(p), AutoModelForSequenceClassification.from_pretrained(p).eval())
|
| 73 |
+
tok, model = self._tf[folder]; out = []
|
| 74 |
+
with torch.no_grad():
|
| 75 |
+
for i in range(0, len(texts), 32):
|
| 76 |
+
enc = tok(texts[i:i + 32], padding=True, truncation=True, max_length=self.cfg["tf_max_len"], return_tensors="pt")
|
| 77 |
+
out.append(torch.softmax(model(**enc).logits.float(), -1)[:, 1].numpy())
|
| 78 |
+
return np.concatenate(out)
|
| 79 |
+
|
| 80 |
+
def predict_proba(self, texts):
|
| 81 |
+
texts = [str(t) for t in texts]
|
| 82 |
+
ct = [normalise(t) for t in texts]
|
| 83 |
+
cache, total, dense = {}, np.zeros(len(texts)), None
|
| 84 |
+
for l in self.cfg["learners"]:
|
| 85 |
+
k = l["kind"]
|
| 86 |
+
if k == "tf":
|
| 87 |
+
p = self._transformer(l["folder"], [light_clean(t) for t in texts])
|
| 88 |
+
elif k == "lex":
|
| 89 |
+
p = self.models[l["name"]].predict_proba(self._lexicon(ct))[:, 1]
|
| 90 |
+
elif k in ("lgbm", "mlp"):
|
| 91 |
+
if dense is None:
|
| 92 |
+
X = self._view("w12+c25", ct, cache)
|
| 93 |
+
dense = np.hstack([self.svd.transform(X), self._lexicon(ct)]).astype(np.float32)
|
| 94 |
+
Xd = dense if k == "lgbm" else self.scaler.transform(dense).astype(np.float32)
|
| 95 |
+
p = self.models[l["name"]].predict_proba(Xd)[:, 1]
|
| 96 |
+
else:
|
| 97 |
+
X = self._view(l["view"], ct, cache)
|
| 98 |
+
if k in ("nblr", "nbsvm"): X = X @ diags(self.ratio[l["view"]])
|
| 99 |
+
m = self.models[l["name"]]
|
| 100 |
+
p = expit(m.decision_function(X)) if k == "nbsvm" else m.predict_proba(X)[:, 1]
|
| 101 |
+
total += l["weight"] * p
|
| 102 |
+
return total
|
| 103 |
+
|
| 104 |
+
def predict(self, texts):
|
| 105 |
+
return (self.predict_proba(texts) > self.threshold).astype(int)
|
ratio_c25.npy
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5c4dc859419e1eb1b43050faa4c12a5b1cce08ce22aee111c73d51abb95dbbc6
|
| 3 |
+
size 530840
|
ratio_w12.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d96ca206896fe890521b704d75c9795cf0e0c6e99cc816ae5685d60c4f5342a6
|
| 3 |
+
size 254752
|
requirements.txt
CHANGED
|
@@ -1,10 +1,7 @@
|
|
| 1 |
-
numpy>=2.0,<3
|
| 2 |
-
scikit-learn==1.
|
| 3 |
-
scipy
|
| 4 |
-
pandas
|
| 5 |
-
joblib
|
| 6 |
-
lightgbm==4.6.0
|
| 7 |
-
huggingface_hub
|
| 8 |
-
torch
|
| 9 |
-
transformers
|
| 10 |
-
sentencepiece
|
|
|
|
| 1 |
+
numpy>=2.0,<3
|
| 2 |
+
scikit-learn==1.8.0
|
| 3 |
+
scipy
|
| 4 |
+
pandas
|
| 5 |
+
joblib
|
| 6 |
+
lightgbm==4.6.0
|
| 7 |
+
huggingface_hub
|
|
|
|
|
|
|
|
|
svd.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8301533d687497bd6bb6fb4cfc8fe73d5901426b9015229d81eb0825b836cdd3
|
| 3 |
+
size 100526839
|
vec_c25.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f4e0c5f852c7b564ac7881c60e20e5def25b7fa08ebe7f02224e843e30e67b67
|
| 3 |
+
size 2048455
|
vec_w12.joblib
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4f3bc465a0078cc4a26d62081f89c2f244696cd8804b0da20579aacf528f47ec
|
| 3 |
+
size 1081664
|