| { |
| "selected_step": 700, |
| "validation_improved": true, |
| "metric": "uncalibrated macro-source NLL; improvement must exceed 1e-6", |
| "base": { |
| "n": 786, |
| "nll": 0.8177371901208593, |
| "brier": 0.3928857406198933, |
| "argmax_agreement": 0.6424936386768448, |
| "soft_target_fraction": 0.10941475826972011, |
| "by_source": { |
| "KSE-RESEARCH-Group/UAReviews": { |
| "n": 200, |
| "nll": 0.3008418807003928, |
| "brier": 0.12895151835801125, |
| "argmax_agreement": 0.915, |
| "soft_target_fraction": 0.0 |
| }, |
| "nvidia/Aegis-AI-Content-Safety-Dataset-2.0": { |
| "n": 200, |
| "nll": 0.5550802189421661, |
| "brier": 0.3796225903166879, |
| "argmax_agreement": 0.695, |
| "soft_target_fraction": 0.0 |
| }, |
| "nvidia/HelpSteer3": { |
| "n": 200, |
| "nll": 1.377823997184949, |
| "brier": 0.5669972128285794, |
| "argmax_agreement": 0.43, |
| "soft_target_fraction": 0.255 |
| }, |
| "openai/coval": { |
| "n": 36, |
| "nll": 1.3788071523285632, |
| "brier": 0.17675586768508195, |
| "argmax_agreement": 0.5, |
| "soft_target_fraction": 0.9722222222222222 |
| }, |
| "rabuahmad/climatecheck": { |
| "n": 150, |
| "nll": 0.9757010305711111, |
| "brier": 0.5822047772661176, |
| "argmax_agreement": 0.5266666666666666, |
| "soft_target_fraction": 0.0 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 468, |
| "nll": 0.7127335941312066, |
| "brier": 0.3476115614992396, |
| "argmax_agreement": 0.7115384615384616, |
| "soft_target_fraction": 0.0811965811965812 |
| }, |
| "noul": { |
| "n": 200, |
| "nll": 0.5550802189421661, |
| "brier": 0.3796225903166879, |
| "argmax_agreement": 0.695, |
| "soft_target_fraction": 0.0 |
| }, |
| "score": { |
| "n": 118, |
| "nll": 1.6793734372301579, |
| "brier": 0.594927654934361, |
| "argmax_agreement": 0.2796610169491525, |
| "soft_target_fraction": 0.4067796610169492 |
| } |
| }, |
| "macro_source_nll": 0.9176508559454364, |
| "note": "Accuracy is argmax agreement with annotation targets; ties use first-index argmax. Soft targets represent finite human votes or subjective preferences, not objective truth. Brier is sum of squared candidate-probability errors; its scale varies with candidate count." |
| }, |
| "candidate": { |
| "n": 786, |
| "nll": 0.5571027568779557, |
| "brier": 0.2618064378625637, |
| "argmax_agreement": 0.7544529262086515, |
| "soft_target_fraction": 0.10941475826972011, |
| "by_source": { |
| "KSE-RESEARCH-Group/UAReviews": { |
| "n": 200, |
| "nll": 0.16488824965745785, |
| "brier": 0.08122107897078124, |
| "argmax_agreement": 0.955, |
| "soft_target_fraction": 0.0 |
| }, |
| "nvidia/Aegis-AI-Content-Safety-Dataset-2.0": { |
| "n": 200, |
| "nll": 0.32759895917618914, |
| "brier": 0.2073110988938831, |
| "argmax_agreement": 0.855, |
| "soft_target_fraction": 0.0 |
| }, |
| "nvidia/HelpSteer3": { |
| "n": 200, |
| "nll": 1.0104743724896212, |
| "brier": 0.42899340445147355, |
| "argmax_agreement": 0.52, |
| "soft_target_fraction": 0.255 |
| }, |
| "openai/coval": { |
| "n": 36, |
| "nll": 1.1635381012599184, |
| "brier": 0.07591942035684947, |
| "argmax_agreement": 0.6666666666666666, |
| "soft_target_fraction": 0.9722222222222222 |
| }, |
| "rabuahmad/climatecheck": { |
| "n": 150, |
| "nll": 0.6360205266404155, |
| "brier": 0.39694429709267093, |
| "argmax_agreement": 0.6866666666666666, |
| "soft_target_fraction": 0.0 |
| } |
| }, |
| "by_kind": { |
| "choice": { |
| "n": 468, |
| "nll": 0.450252531502061, |
| "brier": 0.2214117972369437, |
| "argmax_agreement": 0.8141025641025641, |
| "soft_target_fraction": 0.0811965811965812 |
| }, |
| "noul": { |
| "n": 200, |
| "nll": 0.32759895917618914, |
| "brier": 0.2073110988938831, |
| "argmax_agreement": 0.855, |
| "soft_target_fraction": 0.0 |
| }, |
| "score": { |
| "n": 118, |
| "nll": 1.3698711044734813, |
| "brier": 0.5143806718161742, |
| "argmax_agreement": 0.3474576271186441, |
| "soft_target_fraction": 0.4067796610169492 |
| } |
| }, |
| "macro_source_nll": 0.6605040418447203, |
| "note": "Accuracy is argmax agreement with annotation targets; ties use first-index argmax. Soft targets represent finite human votes or subjective preferences, not objective truth. Brier is sum of squared candidate-probability errors; its scale varies with candidate count." |
| }, |
| "fallback": null |
| } |
