File size: 3,970 Bytes
2b21f2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | {
"target": "volatility",
"n_calls": 1578,
"date_range": [
"2000-04-20",
"2024-08-07"
],
"rungs": {
"A_tfidf_logreg": {
"rung": "A_tfidf_logreg",
"target": "volatility",
"macro_f1": {
"mean": 0.4654222031712612,
"std": 0.020658810508762372,
"ci_low": 0.43977090191681734,
"ci_high": 0.4910735044257051,
"n_folds": 5
},
"weighted_f1": {
"mean": 0.4728655149663347,
"std": 0.02683929107190984,
"ci_low": 0.4395401335050325,
"ci_high": 0.5061908964276369,
"n_folds": 5
},
"auroc_ovr": 0.6498221079830196
},
"B_audio_only_tabular": {
"rung": "B_audio_only_tabular",
"target": "volatility",
"macro_f1": {
"mean": 0.36234587450346983,
"std": 0.02206090666124767,
"ci_low": 0.33495364093446495,
"ci_high": 0.3897381080724747,
"n_folds": 5
},
"weighted_f1": {
"mean": 0.3673854200004104,
"std": 0.01407394242228341,
"ci_low": 0.34991031238186837,
"ci_high": 0.38486052761895245,
"n_folds": 5
},
"auroc_ovr": 0.5413070442007489
},
"C_finbert_attn_text": {
"rung": "C_finbert_attn_text",
"target": "volatility",
"macro_f1": {
"mean": 0.33702124520627996,
"std": 0.019925616306633233,
"ci_low": 0.31228032479631157,
"ci_high": 0.36176216561624835,
"n_folds": 5
},
"weighted_f1": {
"mean": 0.34116823790180584,
"std": 0.029504222200554094,
"ci_low": 0.3045339074003735,
"ci_high": 0.37780256840323817,
"n_folds": 5
},
"auroc_ovr": 0.5098772127518304
},
"D_concat_fusion": {
"rung": "D_concat_fusion",
"target": "volatility",
"macro_f1": {
"mean": 0.30830499262932437,
"std": 0.020128038676930405,
"ci_low": 0.2833127316497269,
"ci_high": 0.3332972536089218,
"n_folds": 5
},
"weighted_f1": {
"mean": 0.32191919492763543,
"std": 0.01616148233394642,
"ci_low": 0.301852064155968,
"ci_high": 0.3419863256993029,
"n_folds": 5
},
"auroc_ovr": 0.5203965258687623
},
"E_gated_fusion": {
"rung": "E_gated_fusion",
"target": "volatility",
"macro_f1": {
"mean": 0.294887158773062,
"std": 0.056394692567913456,
"ci_low": 0.22486389932171433,
"ci_high": 0.3649104182244096,
"n_folds": 5
},
"weighted_f1": {
"mean": 0.2970488993705678,
"std": 0.07276907518926472,
"ci_low": 0.20669415852547485,
"ci_high": 0.3874036402156607,
"n_folds": 5
},
"auroc_ovr": 0.4944706147864221
}
},
"comparisons": {
"E_vs_C": {
"a": "E_gated_fusion",
"b": "C_finbert_attn_text",
"macro_f1_diff": -0.04205235655002032,
"ci_low": -0.0771429424789849,
"ci_high": -0.007902882426394888
},
"E_vs_D": {
"a": "E_gated_fusion",
"b": "D_concat_fusion",
"macro_f1_diff": -0.030745406375682238,
"ci_low": -0.06949999160221496,
"ci_high": 0.006397956158978752
},
"D_vs_C": {
"a": "D_concat_fusion",
"b": "C_finbert_attn_text",
"macro_f1_diff": -0.011306950174338082,
"ci_low": -0.04659330854857793,
"ci_high": 0.02710306777885451
},
"C_vs_A": {
"a": "C_finbert_attn_text",
"b": "A_tfidf_logreg",
"macro_f1_diff": -0.12136716714872242,
"ci_low": -0.16103085394674593,
"ci_high": -0.0796842699919007
},
"E_vs_B": {
"a": "E_gated_fusion",
"b": "B_audio_only_tabular",
"macro_f1_diff": -0.05437444207956699,
"ci_low": -0.09531472506342593,
"ci_high": -0.015387183694287699
}
},
"verdict": "DOES NOT beat text-only baseline",
"convincing_multimodal_result": false
} |