| { |
| "model": { |
| "accuracy": 0.6445830085736555, |
| "macro_f1": 0.6350839549911429, |
| "f1_fake": 0.6939597315436241, |
| "precision_fake": 0.775112443778111, |
| "recall_fake": 0.6281895504252734, |
| "roc_auc": 0.6959308468487506 |
| }, |
| "baseline_majority": { |
| "accuracy": 0.6414653156664069, |
| "macro_f1": 0.39078822412155745, |
| "f1_fake": 0.7815764482431149, |
| "precision_fake": 0.6414653156664069, |
| "recall_fake": 1.0 |
| }, |
| "baseline_tfidf_logreg": { |
| "accuracy": 0.6219797349961029, |
| "macro_f1": 0.6025029496631283, |
| "f1_fake": 0.6904913848117422, |
| "precision_fake": 0.7271505376344086, |
| "recall_fake": 0.6573511543134872, |
| "roc_auc": 0.6495694437107086 |
| }, |
| "confusion_matrix": [ |
| [ |
| 310, |
| 150 |
| ], |
| [ |
| 306, |
| 517 |
| ] |
| ], |
| "ece": 0.04245237071162911, |
| "per_class": [ |
| { |
| "label_name": "false", |
| "n": 250, |
| "accuracy": 0.652, |
| "mean_p_fake": 0.5863052606582642, |
| "binary": "FAKE" |
| }, |
| { |
| "label_name": "half-true", |
| "n": 267, |
| "accuracy": 0.5318352059925093, |
| "mean_p_fake": 0.5340710878372192, |
| "binary": "FAKE" |
| }, |
| { |
| "label_name": "mostly-true", |
| "n": 249, |
| "accuracy": 0.6626506024096386, |
| "mean_p_fake": 0.45594820380210876, |
| "binary": "REAL" |
| }, |
| { |
| "label_name": "true", |
| "n": 211, |
| "accuracy": 0.6872037914691943, |
| "mean_p_fake": 0.44890156388282776, |
| "binary": "REAL" |
| }, |
| { |
| "label_name": "barely-true", |
| "n": 214, |
| "accuracy": 0.6682242990654206, |
| "mean_p_fake": 0.5944609642028809, |
| "binary": "FAKE" |
| }, |
| { |
| "label_name": "pants-fire", |
| "n": 92, |
| "accuracy": 0.75, |
| "mean_p_fake": 0.6390070915222168, |
| "binary": "FAKE" |
| } |
| ], |
| "thresholds": [ |
| { |
| "threshold": 0.05, |
| "precision": 0.6414653156664069, |
| "recall": 1.0, |
| "f1": 0.7815764482431149, |
| "accuracy": 0.6414653156664069, |
| "flagged_rate": 1.0 |
| }, |
| { |
| "threshold": 0.1, |
| "precision": 0.6414653156664069, |
| "recall": 1.0, |
| "f1": 0.7815764482431149, |
| "accuracy": 0.6414653156664069, |
| "flagged_rate": 1.0 |
| }, |
| { |
| "threshold": 0.15, |
| "precision": 0.6414653156664069, |
| "recall": 1.0, |
| "f1": 0.7815764482431149, |
| "accuracy": 0.6414653156664069, |
| "flagged_rate": 1.0 |
| }, |
| { |
| "threshold": 0.2, |
| "precision": 0.641965678627145, |
| "recall": 1.0, |
| "f1": 0.7819477434679335, |
| "accuracy": 0.642244738893219, |
| "flagged_rate": 0.9992205767731879 |
| }, |
| { |
| "threshold": 0.25, |
| "precision": 0.649164677804296, |
| "recall": 0.991494532199271, |
| "f1": 0.7846153846153846, |
| "accuracy": 0.6508183943881528, |
| "flagged_rate": 0.9797349961028838 |
| }, |
| { |
| "threshold": 0.3, |
| "precision": 0.6692640692640692, |
| "recall": 0.9392466585662211, |
| "f1": 0.78159757330637, |
| "accuracy": 0.6632891660171473, |
| "flagged_rate": 0.9002338269680437 |
| }, |
| { |
| "threshold": 0.35, |
| "precision": 0.7057057057057057, |
| "recall": 0.8566221142162819, |
| "f1": 0.7738748627881449, |
| "accuracy": 0.6788776305533905, |
| "flagged_rate": 0.7786438035853468 |
| }, |
| { |
| "threshold": 0.4, |
| "precision": 0.7332576617480137, |
| "recall": 0.7849331713244229, |
| "f1": 0.7582159624413145, |
| "accuracy": 0.6788776305533905, |
| "flagged_rate": 0.686671862821512 |
| }, |
| { |
| "threshold": 0.45, |
| "precision": 0.7464239271781534, |
| "recall": 0.6974483596597812, |
| "f1": 0.7211055276381909, |
| "accuracy": 0.6539360872954014, |
| "flagged_rate": 0.5993764614185503 |
| }, |
| { |
| "threshold": 0.5, |
| "precision": 0.775112443778111, |
| "recall": 0.6281895504252734, |
| "f1": 0.6939597315436241, |
| "accuracy": 0.6445830085736555, |
| "flagged_rate": 0.5198752922837101 |
| }, |
| { |
| "threshold": 0.55, |
| "precision": 0.7837370242214533, |
| "recall": 0.5504252733900364, |
| "f1": 0.6466809421841542, |
| "accuracy": 0.6141855027279813, |
| "flagged_rate": 0.4505066250974279 |
| }, |
| { |
| "threshold": 0.6, |
| "precision": 0.7964071856287425, |
| "recall": 0.4848116646415553, |
| "f1": 0.6027190332326284, |
| "accuracy": 0.5900233826968043, |
| "flagged_rate": 0.39049103663289164 |
| }, |
| { |
| "threshold": 0.65, |
| "precision": 0.8107416879795396, |
| "recall": 0.38517618469015796, |
| "f1": 0.5222405271828665, |
| "accuracy": 0.5479345284489477, |
| "flagged_rate": 0.3047544816835542 |
| }, |
| { |
| "threshold": 0.7, |
| "precision": 0.835016835016835, |
| "recall": 0.30133657351154314, |
| "f1": 0.44285714285714284, |
| "accuracy": 0.5136399064692128, |
| "flagged_rate": 0.23148869836321123 |
| }, |
| { |
| "threshold": 0.75, |
| "precision": 0.8564356435643564, |
| "recall": 0.21020656136087484, |
| "f1": 0.3375609756097561, |
| "accuracy": 0.47077162899454406, |
| "flagged_rate": 0.15744349181605613 |
| }, |
| { |
| "threshold": 0.8, |
| "precision": 0.8837209302325582, |
| "recall": 0.1385176184690158, |
| "f1": 0.23949579831932774, |
| "accuracy": 0.4356975837879969, |
| "flagged_rate": 0.10054559625876851 |
| }, |
| { |
| "threshold": 0.85, |
| "precision": 0.873015873015873, |
| "recall": 0.06682867557715674, |
| "f1": 0.12415349887133183, |
| "accuracy": 0.3951675759937646, |
| "flagged_rate": 0.049103663289166016 |
| }, |
| { |
| "threshold": 0.9, |
| "precision": 1.0, |
| "recall": 0.014580801944106925, |
| "f1": 0.02874251497005988, |
| "accuracy": 0.36788776305533905, |
| "flagged_rate": 0.009353078721745909 |
| }, |
| { |
| "threshold": 0.95, |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "accuracy": 0.35853468433359315, |
| "flagged_rate": 0.0 |
| } |
| ], |
| "label_map": { |
| "false": 1, |
| "half-true": 1, |
| "mostly-true": 0, |
| "true": 0, |
| "barely-true": 1, |
| "pants-fire": 1 |
| }, |
| "config": { |
| "base_model": "roberta-base", |
| "max_len": 128, |
| "epochs": 3, |
| "lr": 2e-05, |
| "batch_size": 16, |
| "seed": 42, |
| "class_weights": [ |
| 1.4071, |
| 0.7756 |
| ], |
| "train_minutes": 5.5, |
| "hardware": "Tesla T4", |
| "data_source": "hub:ucsbnlp/liar" |
| } |
| } |
