{ "task": "moderation", "threshold": 0.91, "per_language": { "az": { "support": 57, "precision": 0.9815, "recall": 0.9298, "f1": 0.955, "fpr": 0.0169, "accuracy": 0.9052, "n": 116 }, "bg": { "support": 52, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.9825, "n": 114 }, "cs": { "support": 60, "precision": 0.9828, "recall": 0.95, "f1": 0.9661, "fpr": 0.0172, "accuracy": 0.9492, "n": 118 }, "da": { "support": 51, "precision": 1.0, "recall": 0.9804, "f1": 0.9901, "fpr": 0.0, "accuracy": 0.9828, "n": 116 }, "de": { "support": 54, "precision": 1.0, "recall": 0.9815, "f1": 0.9907, "fpr": 0.0, "accuracy": 0.9667, "n": 120 }, "el": { "support": 57, "precision": 1.0, "recall": 0.9825, "f1": 0.9912, "fpr": 0.0, "accuracy": 0.9667, "n": 120 }, "en": { "support": 62, "precision": 0.9839, "recall": 0.9839, "f1": 0.9839, "fpr": 0.0179, "accuracy": 0.9576, "n": 118 }, "es": { "support": 52, "precision": 1.0, "recall": 0.9231, "f1": 0.96, "fpr": 0.0, "accuracy": 0.9569, "n": 116 }, "et": { "support": 59, "precision": 1.0, "recall": 0.9661, "f1": 0.9828, "fpr": 0.0, "accuracy": 0.9375, "n": 112 }, "fi": { "support": 50, "precision": 1.0, "recall": 0.96, "f1": 0.9796, "fpr": 0.0, "accuracy": 0.9487, "n": 117 }, "fr": { "support": 60, "precision": 1.0, "recall": 0.95, "f1": 0.9744, "fpr": 0.0, "accuracy": 0.9664, "n": 119 }, "ga": { "support": 50, "precision": 0.913, "recall": 0.84, "f1": 0.875, "fpr": 0.0597, "accuracy": 0.7607, "n": 117 }, "hr": { "support": 60, "precision": 0.9672, "recall": 0.9833, "f1": 0.9752, "fpr": 0.0357, "accuracy": 0.9483, "n": 116 }, "hu": { "support": 63, "precision": 0.9688, "recall": 0.9841, "f1": 0.9764, "fpr": 0.0351, "accuracy": 0.9583, "n": 120 }, "it": { "support": 61, "precision": 0.9839, "recall": 1.0, "f1": 0.9919, "fpr": 0.0167, "accuracy": 0.9504, "n": 121 }, "lt": { "support": 52, "precision": 1.0, "recall": 0.9808, "f1": 0.9903, "fpr": 0.0, "accuracy": 0.9224, "n": 116 }, "lv": { "support": 55, "precision": 1.0, "recall": 0.9455, "f1": 0.972, "fpr": 0.0, "accuracy": 0.9402, "n": 117 }, "mt": { "support": 48, "precision": 0.8723, "recall": 0.8542, "f1": 0.8632, "fpr": 0.0845, "accuracy": 0.7731, "n": 119 }, "nl": { "support": 56, "precision": 0.9821, "recall": 0.9821, "f1": 0.9821, "fpr": 0.0175, "accuracy": 0.9469, "n": 113 }, "pl": { "support": 61, "precision": 1.0, "recall": 0.9836, "f1": 0.9917, "fpr": 0.0, "accuracy": 0.959, "n": 122 }, "pt": { "support": 52, "precision": 0.9804, "recall": 0.9615, "f1": 0.9709, "fpr": 0.0156, "accuracy": 0.931, "n": 116 }, "ro": { "support": 63, "precision": 1.0, "recall": 0.9524, "f1": 0.9756, "fpr": 0.0, "accuracy": 0.9658, "n": 117 }, "sk": { "support": 48, "precision": 0.9792, "recall": 0.9792, "f1": 0.9792, "fpr": 0.0141, "accuracy": 0.9664, "n": 119 }, "sl": { "support": 59, "precision": 1.0, "recall": 0.9831, "f1": 0.9915, "fpr": 0.0, "accuracy": 0.9483, "n": 116 }, "sv": { "support": 52, "precision": 0.9623, "recall": 0.9808, "f1": 0.9714, "fpr": 0.0317, "accuracy": 0.9565, "n": 115 }, "tr": { "support": 60, "precision": 0.9365, "recall": 0.9833, "f1": 0.9593, "fpr": 0.0702, "accuracy": 0.9316, "n": 117 } }, "per_register": { "cyber_intrusion": { "support": 119, "precision": 1.0, "recall": 0.958, "f1": 0.9785, "fpr": 0.0, "accuracy": 0.8739, "n": 119 }, "cyber_intrusion_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 79 }, "defamation": { "support": 111, "precision": 1.0, "recall": 1.0, "f1": 1.0, "fpr": 0.0, "accuracy": 0.982, "n": 111 }, "defamation_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 56 }, "election_integrity": { "support": 116, "precision": 1.0, "recall": 0.9741, "f1": 0.9869, "fpr": 0.0, "accuracy": 0.9655, "n": 116 }, "election_integrity_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 57 }, "extremism": { "support": 130, "precision": 1.0, "recall": 0.9077, "f1": 0.9516, "fpr": 0.0, "accuracy": 0.8154, "n": 130 }, "extremism_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0685, "accuracy": 0.9315, "n": 73 }, "fraud_deception": { "support": 124, "precision": 1.0, "recall": 0.9516, "f1": 0.9752, "fpr": 0.0, "accuracy": 0.8306, "n": 124 }, "fraud_deception_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0508, "accuracy": 0.9492, "n": 59 }, "hate_incitement": { "support": 106, "precision": 1.0, "recall": 0.9906, "f1": 0.9953, "fpr": 0.0, "accuracy": 0.9528, "n": 106 }, "hate_incitement_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 75 }, "illicit_drugs": { "support": 141, "precision": 1.0, "recall": 0.9574, "f1": 0.9783, "fpr": 0.0, "accuracy": 0.8723, "n": 141 }, "illicit_drugs_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 68 }, "mundane_account_access": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0244, "accuracy": 0.9756, "n": 164 }, "mundane_informational": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0046, "accuracy": 0.9954, "n": 218 }, "mundane_operational": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 222 }, "mundane_transactional": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 209 }, "property_crime": { "support": 129, "precision": 1.0, "recall": 0.9535, "f1": 0.9762, "fpr": 0.0, "accuracy": 0.7984, "n": 129 }, "property_crime_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0735, "accuracy": 0.9265, "n": 68 }, "self_harm": { "support": 99, "precision": 1.0, "recall": 0.9495, "f1": 0.9741, "fpr": 0.0, "accuracy": 0.9394, "n": 99 }, "self_harm_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.1111, "accuracy": 0.8889, "n": 63 }, "sexual_exploitation": { "support": 119, "precision": 1.0, "recall": 0.9832, "f1": 0.9915, "fpr": 0.0, "accuracy": 0.9496, "n": 119 }, "sexual_exploitation_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0189, "accuracy": 0.9811, "n": 53 }, "violent_facilitation": { "support": 127, "precision": 1.0, "recall": 0.9606, "f1": 0.9799, "fpr": 0.0, "accuracy": 0.8031, "n": 127 }, "violent_facilitation_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0179, "accuracy": 0.9821, "n": 56 }, "weapons_cbrn": { "support": 133, "precision": 1.0, "recall": 0.9774, "f1": 0.9886, "fpr": 0.0, "accuracy": 0.9173, "n": 133 }, "weapons_cbrn_near_miss": { "support": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "fpr": 0.0, "accuracy": 1.0, "n": 73 } }, "per_label": { "cyber_intrusion": { "support": 119, "precision": 0.8455, "recall": 0.8739, "f1": 0.8595, "fpr": 0.0065, "accuracy": 0.0, "n": 0 }, "defamation": { "support": 111, "precision": 0.9732, "recall": 0.982, "f1": 0.9776, "fpr": 0.001, "accuracy": 0.0, "n": 0 }, "election_integrity": { "support": 116, "precision": 0.9912, "recall": 0.9655, "f1": 0.9782, "fpr": 0.0003, "accuracy": 0.0, "n": 0 }, "extremism": { "support": 130, "precision": 0.8618, "recall": 0.8154, "f1": 0.8379, "fpr": 0.0058, "accuracy": 0.0, "n": 0 }, "fraud_deception": { "support": 124, "precision": 0.8125, "recall": 0.8387, "f1": 0.8254, "fpr": 0.0082, "accuracy": 0.0, "n": 0 }, "hate_incitement": { "support": 106, "precision": 0.9439, "recall": 0.9528, "f1": 0.9484, "fpr": 0.002, "accuracy": 0.0, "n": 0 }, "illicit_drugs": { "support": 141, "precision": 0.9609, "recall": 0.8723, "f1": 0.9145, "fpr": 0.0017, "accuracy": 0.0, "n": 0 }, "property_crime": { "support": 129, "precision": 0.8607, "recall": 0.814, "f1": 0.8367, "fpr": 0.0058, "accuracy": 0.0, "n": 0 }, "self_harm": { "support": 99, "precision": 0.8304, "recall": 0.9394, "f1": 0.8815, "fpr": 0.0064, "accuracy": 0.0, "n": 0 }, "sexual_exploitation": { "support": 119, "precision": 0.9826, "recall": 0.9496, "f1": 0.9658, "fpr": 0.0007, "accuracy": 0.0, "n": 0 }, "violent_facilitation": { "support": 127, "precision": 0.8879, "recall": 0.811, "f1": 0.8477, "fpr": 0.0045, "accuracy": 0.0, "n": 0 }, "weapons_cbrn": { "support": 133, "precision": 0.9248, "recall": 0.9248, "f1": 0.9248, "fpr": 0.0034, "accuracy": 0.0, "n": 0 } }, "worst_languages": [ [ "mt", 0.8631578947368421 ], [ "ga", 0.8749999999999999 ], [ "az", 0.954954954954955 ], [ "tr", 0.9593495934959351 ], [ "es", 0.9600000000000001 ] ], "calibration": { "threshold": 0.91, "objective": "macro_f1", "value_on_val": 0.9037, "default_threshold": 0.5, "default_value_on_val": 0.8658, "improvement": 0.0379, "chosen_on": "validation split, never test", "curve": [ { "threshold": 0.02, "macro_f1": 0.1503, "any_label_recall": 1.0, "any_label_fpr": 0.1051 }, { "threshold": 0.03, "macro_f1": 0.1631, "any_label_recall": 1.0, "any_label_fpr": 0.0813 }, { "threshold": 0.04, "macro_f1": 0.2075, "any_label_recall": 1.0, "any_label_fpr": 0.0747 }, { "threshold": 0.05, "macro_f1": 0.2941, "any_label_recall": 0.9993, "any_label_fpr": 0.0701 }, { "threshold": 0.06, "macro_f1": 0.3936, "any_label_recall": 0.9993, "any_label_fpr": 0.0628 }, { "threshold": 0.07, "macro_f1": 0.4777, "any_label_recall": 0.9993, "any_label_fpr": 0.0588 }, { "threshold": 0.08, "macro_f1": 0.5395, "any_label_recall": 0.9993, "any_label_fpr": 0.0582 }, { "threshold": 0.09, "macro_f1": 0.5885, "any_label_recall": 0.9993, "any_label_fpr": 0.0562 }, { "threshold": 0.1, "macro_f1": 0.6306, "any_label_recall": 0.9993, "any_label_fpr": 0.0549 }, { "threshold": 0.11, "macro_f1": 0.6592, "any_label_recall": 0.9993, "any_label_fpr": 0.0549 }, { "threshold": 0.12, "macro_f1": 0.6812, "any_label_recall": 0.9993, "any_label_fpr": 0.0535 }, { "threshold": 0.13, "macro_f1": 0.7026, "any_label_recall": 0.9993, "any_label_fpr": 0.0529 }, { "threshold": 0.14, "macro_f1": 0.7188, "any_label_recall": 0.9993, "any_label_fpr": 0.0516 }, { "threshold": 0.15, "macro_f1": 0.7321, "any_label_recall": 0.9993, "any_label_fpr": 0.0516 }, { "threshold": 0.16, "macro_f1": 0.7422, "any_label_recall": 0.9993, "any_label_fpr": 0.0502 }, { "threshold": 0.17, "macro_f1": 0.7524, "any_label_recall": 0.9993, "any_label_fpr": 0.0489 }, { "threshold": 0.18, "macro_f1": 0.7614, "any_label_recall": 0.9993, "any_label_fpr": 0.0489 }, { "threshold": 0.19, "macro_f1": 0.7698, "any_label_recall": 0.9993, "any_label_fpr": 0.0476 }, { "threshold": 0.2, "macro_f1": 0.7787, "any_label_recall": 0.9993, "any_label_fpr": 0.0469 }, { "threshold": 0.21, "macro_f1": 0.7856, "any_label_recall": 0.9987, "any_label_fpr": 0.0463 }, { "threshold": 0.22, "macro_f1": 0.7921, "any_label_recall": 0.9987, "any_label_fpr": 0.0463 }, { "threshold": 0.23, "macro_f1": 0.7957, "any_label_recall": 0.9987, "any_label_fpr": 0.0463 }, { "threshold": 0.24, "macro_f1": 0.8001, "any_label_recall": 0.9987, "any_label_fpr": 0.0456 }, { "threshold": 0.25, "macro_f1": 0.8041, "any_label_recall": 0.9987, "any_label_fpr": 0.0456 }, { "threshold": 0.26, "macro_f1": 0.8076, "any_label_recall": 0.9987, "any_label_fpr": 0.0449 }, { "threshold": 0.27, "macro_f1": 0.8114, "any_label_recall": 0.9987, "any_label_fpr": 0.0443 }, { "threshold": 0.28, "macro_f1": 0.8158, "any_label_recall": 0.9987, "any_label_fpr": 0.0443 }, { "threshold": 0.29, "macro_f1": 0.8198, "any_label_recall": 0.9987, "any_label_fpr": 0.043 }, { "threshold": 0.3, "macro_f1": 0.8237, "any_label_recall": 0.9987, "any_label_fpr": 0.043 }, { "threshold": 0.31, "macro_f1": 0.8281, "any_label_recall": 0.9987, "any_label_fpr": 0.043 }, { "threshold": 0.32, "macro_f1": 0.8308, "any_label_recall": 0.9987, "any_label_fpr": 0.0423 }, { "threshold": 0.33, "macro_f1": 0.8345, "any_label_recall": 0.9987, "any_label_fpr": 0.0423 }, { "threshold": 0.34, "macro_f1": 0.836, "any_label_recall": 0.9987, "any_label_fpr": 0.0416 }, { "threshold": 0.35, "macro_f1": 0.8386, "any_label_recall": 0.9987, "any_label_fpr": 0.0416 }, { "threshold": 0.36, "macro_f1": 0.8408, "any_label_recall": 0.9987, "any_label_fpr": 0.0416 }, { "threshold": 0.37, "macro_f1": 0.8434, "any_label_recall": 0.9987, "any_label_fpr": 0.0403 }, { "threshold": 0.38, "macro_f1": 0.846, "any_label_recall": 0.9987, "any_label_fpr": 0.0403 }, { "threshold": 0.39, "macro_f1": 0.849, "any_label_recall": 0.9987, "any_label_fpr": 0.0403 }, { "threshold": 0.4, "macro_f1": 0.8504, "any_label_recall": 0.9987, "any_label_fpr": 0.0397 }, { "threshold": 0.41, "macro_f1": 0.8522, "any_label_recall": 0.9987, "any_label_fpr": 0.0383 }, { "threshold": 0.42, "macro_f1": 0.8547, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.43, "macro_f1": 0.856, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.44, "macro_f1": 0.8583, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.45, "macro_f1": 0.8603, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.46, "macro_f1": 0.8623, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.47, "macro_f1": 0.8636, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.48, "macro_f1": 0.8648, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.49, "macro_f1": 0.8662, "any_label_recall": 0.9987, "any_label_fpr": 0.0377 }, { "threshold": 0.5, "macro_f1": 0.8658, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.51, "macro_f1": 0.8676, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.52, "macro_f1": 0.8699, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.53, "macro_f1": 0.8708, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.54, "macro_f1": 0.8738, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.55, "macro_f1": 0.8748, "any_label_recall": 0.9987, "any_label_fpr": 0.037 }, { "threshold": 0.56, "macro_f1": 0.877, "any_label_recall": 0.9987, "any_label_fpr": 0.0364 }, { "threshold": 0.57, "macro_f1": 0.8779, "any_label_recall": 0.9987, "any_label_fpr": 0.0364 }, { "threshold": 0.58, "macro_f1": 0.8785, "any_label_recall": 0.9987, "any_label_fpr": 0.0357 }, { "threshold": 0.59, "macro_f1": 0.8805, "any_label_recall": 0.9987, "any_label_fpr": 0.035 }, { "threshold": 0.6, "macro_f1": 0.882, "any_label_recall": 0.9987, "any_label_fpr": 0.0337 }, { "threshold": 0.61, "macro_f1": 0.8823, "any_label_recall": 0.998, "any_label_fpr": 0.033 }, { "threshold": 0.62, "macro_f1": 0.8833, "any_label_recall": 0.998, "any_label_fpr": 0.033 }, { "threshold": 0.63, "macro_f1": 0.8837, "any_label_recall": 0.9974, "any_label_fpr": 0.033 }, { "threshold": 0.64, "macro_f1": 0.8845, "any_label_recall": 0.9974, "any_label_fpr": 0.033 }, { "threshold": 0.65, "macro_f1": 0.8838, "any_label_recall": 0.9974, "any_label_fpr": 0.033 }, { "threshold": 0.66, "macro_f1": 0.8844, "any_label_recall": 0.9974, "any_label_fpr": 0.033 }, { "threshold": 0.67, "macro_f1": 0.8853, "any_label_recall": 0.9974, "any_label_fpr": 0.0324 }, { "threshold": 0.68, "macro_f1": 0.8877, "any_label_recall": 0.9974, "any_label_fpr": 0.0317 }, { "threshold": 0.69, "macro_f1": 0.8889, "any_label_recall": 0.9974, "any_label_fpr": 0.0304 }, { "threshold": 0.7, "macro_f1": 0.8894, "any_label_recall": 0.9974, "any_label_fpr": 0.0304 }, { "threshold": 0.71, "macro_f1": 0.8901, "any_label_recall": 0.9974, "any_label_fpr": 0.0297 }, { "threshold": 0.72, "macro_f1": 0.8908, "any_label_recall": 0.9974, "any_label_fpr": 0.0291 }, { "threshold": 0.73, "macro_f1": 0.8924, "any_label_recall": 0.9974, "any_label_fpr": 0.0291 }, { "threshold": 0.74, "macro_f1": 0.8931, "any_label_recall": 0.9974, "any_label_fpr": 0.0291 }, { "threshold": 0.75, "macro_f1": 0.8943, "any_label_recall": 0.9974, "any_label_fpr": 0.0284 }, { "threshold": 0.76, "macro_f1": 0.8962, "any_label_recall": 0.9974, "any_label_fpr": 0.0284 }, { "threshold": 0.77, "macro_f1": 0.8973, "any_label_recall": 0.9967, "any_label_fpr": 0.0278 }, { "threshold": 0.78, "macro_f1": 0.8977, "any_label_recall": 0.9961, "any_label_fpr": 0.0278 }, { "threshold": 0.79, "macro_f1": 0.8986, "any_label_recall": 0.9954, "any_label_fpr": 0.0271 }, { "threshold": 0.8, "macro_f1": 0.8999, "any_label_recall": 0.9954, "any_label_fpr": 0.0271 }, { "threshold": 0.81, "macro_f1": 0.8997, "any_label_recall": 0.9948, "any_label_fpr": 0.0271 }, { "threshold": 0.82, "macro_f1": 0.8996, "any_label_recall": 0.9935, "any_label_fpr": 0.0264 }, { "threshold": 0.83, "macro_f1": 0.8996, "any_label_recall": 0.9928, "any_label_fpr": 0.0258 }, { "threshold": 0.84, "macro_f1": 0.8996, "any_label_recall": 0.9915, "any_label_fpr": 0.0258 }, { "threshold": 0.85, "macro_f1": 0.8989, "any_label_recall": 0.9902, "any_label_fpr": 0.0251 }, { "threshold": 0.86, "macro_f1": 0.8995, "any_label_recall": 0.9863, "any_label_fpr": 0.0251 }, { "threshold": 0.87, "macro_f1": 0.9, "any_label_recall": 0.9831, "any_label_fpr": 0.0245 }, { "threshold": 0.88, "macro_f1": 0.9004, "any_label_recall": 0.9804, "any_label_fpr": 0.0245 }, { "threshold": 0.89, "macro_f1": 0.9018, "any_label_recall": 0.9785, "any_label_fpr": 0.0231 }, { "threshold": 0.9, "macro_f1": 0.9032, "any_label_recall": 0.9759, "any_label_fpr": 0.0212 }, { "threshold": 0.91, "macro_f1": 0.9037, "any_label_recall": 0.9687, "any_label_fpr": 0.0185 }, { "threshold": 0.92, "macro_f1": 0.9027, "any_label_recall": 0.9583, "any_label_fpr": 0.0165 }, { "threshold": 0.93, "macro_f1": 0.8994, "any_label_recall": 0.9472, "any_label_fpr": 0.0152 }, { "threshold": 0.94, "macro_f1": 0.8972, "any_label_recall": 0.9329, "any_label_fpr": 0.0126 }, { "threshold": 0.95, "macro_f1": 0.8979, "any_label_recall": 0.914, "any_label_fpr": 0.0106 } ], "notes": [] } }