File size: 14,683 Bytes
c1e5dc4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 | {
"Horizon-Labs/multilingual-hate-speech-base": {
"labels": [
"hate",
"target_race",
"target_religion",
"target_origin",
"target_gender",
"target_sexuality",
"target_age",
"target_disability"
],
"col": "hate",
"neg": false,
"ethos": {
"n": 998,
"auc": 0.9317235177502095,
"f1_05": 0.7790549169859514,
"fpr_05": 0.07964601769911504
},
"hatexplain": {
"n": 1924,
"auc": 0.8468165362902205,
"f1_05": 0.6658130601792573,
"fpr_05": 0.3368421052631579
},
"hs18": {
"n": 10534,
"auc": 0.903714273347328,
"f1_05": 0.5688575899843505,
"fpr_05": 0.06818668379362021
},
"kmhas": {
"n": 5000,
"auc": 0.8075370536546457,
"f1_05": 0.3082405345211581,
"fpr_05": 0.029420970266040687
},
"mhc_ara": {
"n": 3570,
"auc": 0.7762656397659214,
"f1_05": 0.6992628992628993,
"fpr_05": 0.1421933085501859,
"func_acc": {
"counter_quote_nh": 0.8013698630136986,
"counter_ref_nh": 0.8082191780821918,
"derog_dehum_h": 0.7214285714285714,
"derog_impl_h": 0.5103448275862069,
"derog_neg_attrib_h": 0.7642857142857142,
"derog_neg_emote_h": 0.6071428571428571,
"ident_neutral_nh": 0.9785714285714285,
"ident_pos_nh": 0.7523809523809524,
"negate_neg_nh": 0.7428571428571429,
"negate_pos_h": 0.7142857142857143,
"phrase_opinion_h": 0.75,
"phrase_question_h": 0.5071428571428571,
"profanity_h": 0.7785714285714286,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.6857142857142857,
"ref_subs_sent_h": 0.7142857142857143,
"slur_h": 0.02197802197802198,
"spell_char_eng_replace_h": 0.4489795918367347,
"spell_char_repeat_h": 0.410958904109589,
"spell_sent_arabazi_h": 0.0,
"spell_space_add_h": 0.3493150684931507,
"spell_word_variation_h": 0.6164383561643836,
"target_group_nh": 0.9230769230769231,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.6071428571428571,
"threat_norm_h": 0.8642857142857143
}
},
"mhc_cmn": {
"n": 3378,
"auc": 0.9220649399332393,
"f1_05": 0.8688060405851817,
"fpr_05": 0.11859838274932614,
"func_acc": {
"counter_quote_nh": 0.8170731707317073,
"counter_ref_nh": 0.8109756097560976,
"derog_dehum_h": 0.9714285714285714,
"derog_impl_h": 0.5857142857142857,
"derog_neg_attrib_h": 0.9571428571428572,
"derog_neg_emote_h": 0.8071428571428572,
"ident_neutral_nh": 0.9571428571428572,
"ident_pos_nh": 0.8285714285714286,
"negate_neg_nh": 0.85,
"negate_pos_h": 0.8571428571428571,
"phrase_opinion_h": 0.9571428571428572,
"phrase_question_h": 0.9071428571428571,
"profanity_h": 0.9642857142857143,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9142857142857143,
"ref_subs_sent_h": 0.95,
"slur_h": 0.21333333333333335,
"spell_char_comp_h": 0.6575342465753424,
"spell_homoph_h": 0.7571428571428571,
"spell_pinyin_h": 0.6577181208053692,
"target_group_nh": 0.8769230769230769,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.9428571428571428,
"threat_norm_h": 0.9642857142857143
}
},
"mhc_deu": {
"n": 3645,
"auc": 0.9100735965619124,
"f1_05": 0.8775681341719077,
"fpr_05": 0.11598173515981736,
"func_acc": {
"counter_quote_nh": 0.8774193548387097,
"counter_ref_nh": 0.967741935483871,
"derog_dehum_h": 0.9928571428571429,
"derog_impl_h": 0.7241379310344828,
"derog_neg_attrib_h": 1.0,
"derog_neg_emote_h": 0.7142857142857143,
"ident_neutral_nh": 0.95,
"ident_pos_nh": 0.7952380952380952,
"negate_neg_nh": 0.6928571428571428,
"negate_pos_h": 0.85,
"phrase_opinion_h": 1.0,
"phrase_question_h": 0.9,
"profanity_h": 0.95,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9571428571428572,
"ref_subs_sent_h": 0.9214285714285714,
"slur_h": 0.4166666666666667,
"spell_char_del_h": 0.7,
"spell_char_swap_h": 0.8571428571428571,
"spell_leet_h": 0.7032258064516129,
"spell_space_add_h": 0.41935483870967744,
"spell_space_del_h": 0.8193548387096774,
"target_group_nh": 0.8615384615384616,
"target_indiv_nh": 0.9846153846153847,
"target_obj_nh": 1.0,
"threat_dir_h": 0.9,
"threat_norm_h": 0.95
}
},
"mhc_eng": {
"n": 3728,
"auc": 0.9233422474668399,
"f1_05": 0.864244741873805,
"fpr_05": 0.0944206008583691,
"func_acc": {
"counter_quote_nh": 0.7861271676300579,
"counter_ref_nh": 0.900709219858156,
"derog_dehum_h": 0.9714285714285714,
"derog_impl_h": 0.6214285714285714,
"derog_neg_attrib_h": 1.0,
"derog_neg_emote_h": 0.8214285714285714,
"ident_neutral_nh": 1.0,
"ident_pos_nh": 0.91005291005291,
"negate_neg_nh": 0.8646616541353384,
"negate_pos_h": 0.9357142857142857,
"phrase_opinion_h": 0.9548872180451128,
"phrase_question_h": 0.9214285714285714,
"profanity_h": 0.9428571428571428,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9571428571428572,
"ref_subs_sent_h": 0.9624060150375939,
"slur_h": 0.5486111111111112,
"slur_homonym_nh": 0.7666666666666667,
"slur_reclaimed_nh": 0.8641975308641975,
"spell_char_del_h": 0.4928571428571429,
"spell_char_swap_h": 0.7744360902255639,
"spell_leet_h": 0.5260115606936416,
"spell_space_add_h": 0.4046242774566474,
"spell_space_del_h": 0.7730496453900709,
"target_group_nh": 0.9193548387096774,
"target_indiv_nh": 1.0,
"target_obj_nh": 0.9846153846153847,
"threat_dir_h": 0.8947368421052632,
"threat_norm_h": 0.9642857142857143
}
},
"mhc_fra": {
"n": 3718,
"auc": 0.8889973510389432,
"f1_05": 0.85150955021565,
"fpr_05": 0.17531305903398928,
"func_acc": {
"counter_quote_nh": 0.7485029940119761,
"counter_ref_nh": 0.8083832335329342,
"derog_dehum_h": 1.0,
"derog_impl_h": 0.7642857142857142,
"derog_neg_attrib_h": 0.9642857142857143,
"derog_neg_emote_h": 0.8428571428571429,
"ident_neutral_nh": 0.95,
"ident_pos_nh": 0.7511961722488039,
"negate_neg_nh": 0.6285714285714286,
"negate_pos_h": 0.95,
"phrase_opinion_h": 1.0,
"phrase_question_h": 0.9104477611940298,
"profanity_h": 0.9473684210526315,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9924812030075187,
"ref_subs_sent_h": 0.9856115107913669,
"slur_h": 0.29375,
"spell_char_del_h": 0.5928571428571429,
"spell_char_swap_h": 0.8642857142857143,
"spell_leet_h": 0.6287425149700598,
"spell_space_add_h": 0.40119760479041916,
"spell_space_del_h": 0.6227544910179641,
"target_group_nh": 0.8307692307692308,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.9,
"threat_norm_h": 0.9285714285714286
}
},
"mhc_hin": {
"n": 3565,
"auc": 0.6772136711181171,
"f1_05": 0.4907300115874855,
"fpr_05": 0.10863509749303621,
"func_acc": {
"counter_quote_nh": 0.8972602739726028,
"counter_ref_nh": 0.8356164383561644,
"derog_dehum_h": 0.3142857142857143,
"derog_impl_h": 0.42857142857142855,
"derog_neg_attrib_h": 0.40714285714285714,
"derog_neg_emote_h": 0.37142857142857144,
"ident_neutral_nh": 0.9214285714285714,
"ident_pos_nh": 0.7952380952380952,
"negate_neg_nh": 0.8428571428571429,
"negate_pos_h": 0.39285714285714285,
"phrase_opinion_h": 0.4357142857142857,
"phrase_question_h": 0.36428571428571427,
"profanity_h": 0.4142857142857143,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.5,
"ref_subs_sent_h": 0.5285714285714286,
"slur_h": 0.011111111111111112,
"spell_char_del_h": 0.20714285714285716,
"spell_char_swap_h": 0.3,
"spell_leet_h": 0.2671232876712329,
"spell_space_add_h": 0.1095890410958904,
"spell_space_del_h": 0.1917808219178082,
"target_group_nh": 0.9692307692307692,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.3142857142857143,
"threat_norm_h": 0.4714285714285714
}
},
"mhc_ita": {
"n": 3690,
"auc": 0.9011112894713926,
"f1_05": 0.8500105775333192,
"fpr_05": 0.12195121951219512,
"func_acc": {
"counter_quote_nh": 0.8136645962732919,
"counter_ref_nh": 0.9130434782608695,
"derog_dehum_h": 1.0,
"derog_impl_h": 0.6714285714285714,
"derog_neg_attrib_h": 1.0,
"derog_neg_emote_h": 0.7357142857142858,
"ident_neutral_nh": 0.95,
"ident_pos_nh": 0.7952380952380952,
"negate_neg_nh": 0.7785714285714286,
"negate_pos_h": 0.8857142857142857,
"phrase_opinion_h": 0.9714285714285714,
"phrase_question_h": 0.8928571428571429,
"profanity_h": 0.9785714285714285,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.95,
"ref_subs_sent_h": 0.9214285714285714,
"slur_h": 0.37857142857142856,
"spell_char_del_h": 0.5714285714285714,
"spell_char_swap_h": 0.7928571428571428,
"spell_leet_h": 0.5031055900621118,
"spell_space_add_h": 0.37888198757763975,
"spell_space_del_h": 0.7267080745341615,
"target_group_nh": 0.8461538461538461,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.8571428571428571,
"threat_norm_h": 0.8928571428571429
}
},
"mhc_nld": {
"n": 3765,
"auc": 0.8709887205387206,
"f1_05": 0.8359810191871261,
"fpr_05": 0.1608888888888889,
"func_acc": {
"counter_quote_nh": 0.8176470588235294,
"counter_ref_nh": 0.8764705882352941,
"derog_dehum_h": 0.9714285714285714,
"derog_impl_h": 0.7357142857142858,
"derog_neg_attrib_h": 1.0,
"derog_neg_emote_h": 0.7571428571428571,
"ident_neutral_nh": 0.9571428571428572,
"ident_pos_nh": 0.6761904761904762,
"negate_neg_nh": 0.6857142857142857,
"negate_pos_h": 0.9,
"phrase_opinion_h": 0.95,
"phrase_question_h": 0.9142857142857143,
"profanity_h": 0.9928571428571429,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9428571428571428,
"ref_subs_sent_h": 0.9285714285714286,
"slur_h": 0.12941176470588237,
"spell_char_del_h": 0.6357142857142857,
"spell_char_swap_h": 0.7928571428571428,
"spell_leet_h": 0.5294117647058824,
"spell_space_add_h": 0.3352941176470588,
"spell_space_del_h": 0.6941176470588235,
"target_group_nh": 0.8461538461538461,
"target_indiv_nh": 1.0,
"target_obj_nh": 0.9846153846153847,
"threat_dir_h": 0.95,
"threat_norm_h": 0.95
}
},
"mhc_pol": {
"n": 3815,
"auc": 0.8597691342138918,
"f1_05": 0.8170655567117586,
"fpr_05": 0.1442392260334213,
"func_acc": {
"counter_quote_nh": 0.875,
"counter_ref_nh": 0.8579545454545454,
"derog_dehum_h": 0.95,
"derog_impl_h": 0.6357142857142857,
"derog_neg_attrib_h": 0.9428571428571428,
"derog_neg_emote_h": 0.7214285714285714,
"ident_neutral_nh": 0.95,
"ident_pos_nh": 0.6952380952380952,
"negate_neg_nh": 0.7428571428571429,
"negate_pos_h": 0.8928571428571429,
"phrase_opinion_h": 0.9357142857142857,
"phrase_question_h": 0.8142857142857143,
"profanity_h": 0.9285714285714286,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9071428571428571,
"ref_subs_sent_h": 0.9857142857142858,
"slur_h": 0.19473684210526315,
"spell_char_del_h": 0.5785714285714286,
"spell_char_swap_h": 0.7642857142857142,
"spell_leet_h": 0.5454545454545454,
"spell_space_add_h": 0.30113636363636365,
"spell_space_del_h": 0.6534090909090909,
"target_group_nh": 0.8615384615384616,
"target_indiv_nh": 0.9846153846153847,
"target_obj_nh": 1.0,
"threat_dir_h": 0.8785714285714286,
"threat_norm_h": 0.9357142857142857
}
},
"mhc_por": {
"n": 3691,
"auc": 0.900126880076512,
"f1_05": 0.8486520908511993,
"fpr_05": 0.11801801801801802,
"func_acc": {
"counter_quote_nh": 0.8198757763975155,
"counter_ref_nh": 0.906832298136646,
"derog_dehum_h": 1.0,
"derog_impl_h": 0.7028985507246377,
"derog_neg_attrib_h": 0.9714285714285714,
"derog_neg_emote_h": 0.7785714285714286,
"ident_neutral_nh": 0.9090909090909091,
"ident_pos_nh": 0.8380952380952381,
"negate_neg_nh": 0.7714285714285715,
"negate_pos_h": 0.8857142857142857,
"phrase_opinion_h": 0.9642857142857143,
"phrase_question_h": 0.8428571428571429,
"profanity_h": 0.9,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9214285714285714,
"ref_subs_sent_h": 0.9357142857142857,
"slur_h": 0.21428571428571427,
"spell_char_del_h": 0.6285714285714286,
"spell_char_swap_h": 0.8285714285714286,
"spell_leet_h": 0.5838509316770186,
"spell_space_add_h": 0.40993788819875776,
"spell_space_del_h": 0.6459627329192547,
"target_group_nh": 0.8769230769230769,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.9357142857142857,
"threat_norm_h": 0.8928571428571429
}
},
"mhc_spa": {
"n": 3745,
"auc": 0.8934337082886146,
"f1_05": 0.8514155817317628,
"fpr_05": 0.1320754716981132,
"func_acc": {
"counter_quote_nh": 0.8048780487804879,
"counter_ref_nh": 0.7987804878048781,
"derog_dehum_h": 1.0,
"derog_impl_h": 0.7142857142857143,
"derog_neg_attrib_h": 1.0,
"derog_neg_emote_h": 0.7714285714285715,
"ident_neutral_nh": 0.9714285714285714,
"ident_pos_nh": 0.8095238095238095,
"negate_neg_nh": 0.7857142857142857,
"negate_pos_h": 0.8896551724137931,
"phrase_opinion_h": 0.9241379310344827,
"phrase_question_h": 0.8928571428571429,
"profanity_h": 0.9142857142857143,
"profanity_nh": 1.0,
"ref_subs_clause_h": 0.9310344827586207,
"ref_subs_sent_h": 0.9379310344827586,
"slur_h": 0.2866666666666667,
"spell_char_del_h": 0.5642857142857143,
"spell_char_swap_h": 0.8571428571428571,
"spell_leet_h": 0.5443786982248521,
"spell_space_add_h": 0.47560975609756095,
"spell_space_del_h": 0.676829268292683,
"target_group_nh": 0.8769230769230769,
"target_indiv_nh": 1.0,
"target_obj_nh": 1.0,
"threat_dir_h": 0.8857142857142857,
"threat_norm_h": 0.9517241379310345
}
},
"mhs": {
"n": 6000,
"auc": 0.7565266801456891,
"f1_05": 0.5119189511323003,
"fpr_05": 0.21192350956130485,
"target_auc": {
"race": 0.890887542401214,
"religion": 0.9565567067442536,
"origin": 0.9227262673507151,
"gender": 0.9034750126856563,
"sexuality": 0.9199812728486189,
"age": 0.8252287581699347,
"disability": 0.9437653464289527
}
},
"olidbr": {
"n": 1738,
"auc": 0.7214719896223983,
"f1_05": 0.22429906542056074,
"fpr_05": 0.029392117568470273
},
"_mhc_mean_auc": 0.8657624707703732,
"_mhc_mean_f1": 0.805022382155645,
"_real_mean_auc": 0.8036267566968518,
"_heldout_real_mean_auc": 0.8411117085936454
}
} |