File size: 3,099 Bytes
2b21f2d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | {
"target": "sentiment",
"task": "3-class turn-level late-fusion sentiment classification",
"classes": [
"negative",
"neutral",
"positive"
],
"flagship_rung": "E_gated_fusion",
"n_calls": 1038,
"date_range": [
"2018-01-29",
"2024-08-07"
],
"population_filter": "call_has_any_audio == true AND sentiment_triclass in {0,1,2}",
"cv": {
"method": "walk_forward_time_splits",
"n_splits": 5,
"min_train_frac": 0.35,
"expanding_window": true,
"source": "src/evaluation/cv.py:walk_forward_time_splits"
},
"oof_coverage_pct": 65.0,
"max_turns": 114,
"n_truncated_calls": 54,
"label_caveat": "sentiment_triclass is derived from FinBERT tercile-bucketed text sentiment scores (src/modeling/targets.py:call_level_sentiment_label), not independently human-labeled ground truth \u2014 treat 'true_class' as a text-sentiment-model-derived reference label, not verified ground truth.",
"headline_result": {
"macro_f1_gated_fusion": 0.7413003578162497,
"macro_f1_text_only": 0.6988462296532032,
"macro_f1_audio_only": 0.32692407698816484,
"diff_vs_text_only": {
"point": 0.05092381444555949,
"ci_low": 0.015264113594670557,
"ci_high": 0.08594527181351999
}
},
"export_run_walk_forward_macro_f1": 0.7696012817198535,
"export_run_drift_from_frozen": 0.028300923903603836,
"behavioral_dimensions": {
"confidence": "clip(0.5*((sentiment_score+1)/2) + 0.5*(1-uncertainty_score), 0, 1)",
"specificity": "clip(0.7*specificity_score + 0.3*(forward_looking_score/0.667), 0, 1)",
"stress": "audio-covered turns: mean(sentiment_pitch_divergence, sentiment_energy_divergence, hedging_prosody_divergence, squash(speech_rate_anomaly)); no-audio turns (text-only fallback): clip(0.6*uncertainty_score + 0.4*hedging_frequency, 0, 1)",
"alignment_drift": "within-call min-max normalized Euclidean distance of (sentiment_score, uncertainty_score, specificity_score) from that call's own prepared_remarks-turn baseline mean; null when a call has fewer than 2 prepared_remarks turns (alignment_drift_available=false)"
},
"risk_event_rule": "friction = 0.5*stress + 0.5*fillna(alignment_drift, 0); is_risk_event when friction >= max(within-call 90th percentile, 0.6), capped at the top 5 flagged turns per call",
"alignment_drift_unavailable_pct": 9.7,
"risk_tier_cutoffs": {
"method": "population tertiles of avg_stress across the aligned population",
"low_high_cutoff": 0.15765269684765826,
"medium_high_cutoff": 0.2286551220882104
},
"company_views": {
"min_calls_for_trend_view": 2,
"min_oof_calls_for_comparison_view": 2,
"note": "Company trend/comparison views only include tickers meeting these thresholds; thinly-covered tickers are absent, not zero-filled."
},
"workspace_caveat": "Per-turn Timeline, Evidence panel, and risk-event markers are available only for the 675 OOF-covered calls (same population as attention-weight detail). Non-OOF calls in calls_index.json/companies.json carry call-level behavioral summary scores only, no turns[] detail."
} |