BaseDecision-Dyno / calibration.json
onlyaady's picture
Publish calibrated BaseDecision-Dyno with portable HTTP serving
3e88dcc verified
Raw History Blame Contribute Delete
97.2 kB
{ "source_release_id": "2818c9c47a86170ce9192c4cabd0715cb26f4c89b0ad557f8c3f15358b7f012a", "data_counts": { "fit": 7632, "selection": 2368, "audit_family": 2000, "audit_group": 8000 }, "data_identity_sha256": "676e3c06bb9d3858875515c3c2da5f25b2b478fa2b9253ad11fa8f6b196b69ed", "data_scope": "Reused historical holdout, newly screened and rescored. Not a fresh sealed test.", "selection_and_audit_scoring": "Singleton, exact native serving numerical path; fit batching screened separately", "baseline_temperature": 1.0, "modes": { "default": { "fit_and_selection": { "temperature": 1.2626014578759173, "selected_fit_loss": "nll", "candidates": [ { "temperature": 1.0, "fit_loss": "identity", "at_search_boundary": false, "selection_nll": 0.7275512028121913, "selection_brier": 0.327622453770135, "selection_score": 0.33580234422981187, "eligible": true }, { "temperature": 1.2626014578759173, "fit_loss": "nll", "fit_objective": 0.7168196817779551, "at_search_boundary": false, "bounds": [ 0.05, 20.0 ], "selection_nll": 0.712893716489562, "selection_brier": 0.32333401069095247, "selection_score": 0.3308736186536595, "eligible": true }, { "temperature": 1.27003745705363, "fit_loss": "brier", "fit_objective": 0.31996540663853446, "at_search_boundary": false, "bounds": [ 0.05, 20.0 ], "selection_nll": 0.712955260655177, "selection_brier": 0.32334033241842747, "selection_score": 0.3309141805567075, "eligible": true } ], "policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization." }, "audit": { "pooled": { "passed": true, "baseline_T1": { "n": 10000, "nll": 0.852946444060366, "brier_sum_over_options": 0.3749600967063925, "source_balanced_nll": 0.6999376751969619, "source_balanced_brier": 0.30462924474543973, "hard_label_n": 9266, "hard_accuracy": 0.703971508741636, "ece15_hard": 0.07901558557138746, "ece15_soft_target_top_label": 0.07927800766590869, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10000, "nll": 0.8154129693881843, "brier_sum_over_options": 0.3636438958749284, "source_balanced_nll": 0.6835431229745554, "source_balanced_brier": 0.2998612344780709, "hard_label_n": 9266, "hard_accuracy": 0.703971508741636, "ece15_hard": 0.038995968182245856, "ece15_soft_target_top_label": 0.036747079044559566, "argmax_changes_vs_T1": 0 }, "source_families": 84, "family_cluster_bootstrap": { "nll_delta_95": [ -0.03605991235662911, -0.0013140520669599177 ], "brier_delta_95": [ -0.009391151837829768, -0.0006976623009813496 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." }, "audit_group": { "passed": true, "baseline_T1": { "n": 8000, "nll": 0.888022765832513, "brier_sum_over_options": 0.3811418691518153, "source_balanced_nll": 0.7267924319475662, "source_balanced_brier": 0.3125574139333116, "hard_label_n": 7499, "hard_accuracy": 0.6998266435524737, "ece15_hard": 0.07459343476372879, "ece15_soft_target_top_label": 0.07301347448112568, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 8000, "nll": 0.8499313997523429, "brier_sum_over_options": 0.371009631033076, "source_balanced_nll": 0.7097259735906477, "source_balanced_brier": 0.3083857511306294, "hard_label_n": 7499, "hard_accuracy": 0.6998266435524737, "ece15_hard": 0.03078003605959842, "ece15_soft_target_top_label": 0.02856035757287184, "argmax_changes_vs_T1": 0 }, "source_families": 71, "family_cluster_bootstrap": { "nll_delta_95": [ -0.038382270574252315, 0.0003448221832350238 ], "brier_delta_95": [ -0.00891044962013338, 0.000358978773719409 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." }, "audit_family": { "passed": true, "baseline_T1": { "n": 2000, "nll": 0.7126411569717783, "brier_sum_over_options": 0.3502330069247016, "source_balanced_nll": 0.5532693883282767, "source_balanced_brier": 0.26132924379629274, "hard_label_n": 1767, "hard_accuracy": 0.7215619694397284, "ece15_hard": 0.0977828232095496, "ece15_soft_target_top_label": 0.10433614040504041, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2000, "nll": 0.6773392479315501, "brier_sum_over_options": 0.3341809552423381, "source_balanced_nll": 0.5405444773020512, "source_balanced_brier": 0.25330425891409736, "hard_label_n": 1767, "hard_accuracy": 0.7215619694397284, "ece15_hard": 0.07800818374407796, "ece15_soft_target_top_label": 0.07447692885921983, "argmax_changes_vs_T1": 0 }, "source_families": 13, "family_cluster_bootstrap": { "nll_delta_95": [ -0.04730023306508244, 0.01337183299465118 ], "brier_delta_95": [ -0.02130890503696608, 0.001674475477007773 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." } }, "diagnostics": { "type": { "choice": { "baseline_T1": { "n": 9585, "nll": 0.8736567431397109, "brier_sum_over_options": 0.3877836569453464, "source_balanced_nll": 0.7113112552360633, "source_balanced_brier": 0.3134031210640267, "hard_label_n": 9124, "hard_accuracy": 0.6998027181060938, "ece15_hard": 0.08082652934851452, "ece15_soft_target_top_label": 0.08301943064283653, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 9585, "nll": 0.8340462097507013, "brier_sum_over_options": 0.37579718896533437, "source_balanced_nll": 0.6937124217595521, "source_balanced_brier": 0.30821745985835847, "hard_label_n": 9124, "hard_accuracy": 0.6998027181060938, "ece15_hard": 0.03971737650838311, "ece15_soft_target_top_label": 0.039635874875866624, "argmax_changes_vs_T1": 0 } }, "noul": { "baseline_T1": { "n": 415, "nll": 0.3746133918301991, "brier_sum_over_options": 0.07878220540428955, "source_balanced_nll": 0.3561816083069769, "source_balanced_brier": 0.07147533612369922, "hard_label_n": 142, "hard_accuracy": 0.971830985915493, "ece15_hard": 0.05048296869603257, "ece15_soft_target_top_label": 0.028886424994996142, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 415, "nll": 0.38505246607559207, "brier_sum_over_options": 0.08294675305193852, "source_balanced_nll": 0.3686868791865422, "source_balanced_brier": 0.07672669673180794, "hard_label_n": 142, "hard_accuracy": 0.971830985915493, "ece15_hard": 0.08497080667781354, "ece15_soft_target_top_label": 0.04488882620673573, "argmax_changes_vs_T1": 0 } } }, "option_count": { "2": { "baseline_T1": { "n": 3739, "nll": 0.3332287517972207, "brier_sum_over_options": 0.1835811451336453, "source_balanced_nll": 0.34593070447439217, "source_balanced_brier": 0.17391155278332907, "hard_label_n": 3402, "hard_accuracy": 0.8644914756025868, "ece15_hard": 0.02245159561941003, "ece15_soft_target_top_label": 0.023019111682206526, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3739, "nll": 0.3324121227893266, "brier_sum_over_options": 0.18317845144837797, "source_balanced_nll": 0.3509764214190602, "source_balanced_brier": 0.17633861775459045, "hard_label_n": 3402, "hard_accuracy": 0.8644914756025868, "ece15_hard": 0.018485063230447577, "ece15_soft_target_top_label": 0.017437953045210633, "argmax_changes_vs_T1": 0 } }, "4": { "baseline_T1": { "n": 1907, "nll": 0.9022618813320882, "brier_sum_over_options": 0.4423083809261321, "source_balanced_nll": 0.7193985221889149, "source_balanced_brier": 0.33394772375263204, "hard_label_n": 1712, "hard_accuracy": 0.6378504672897196, "ece15_hard": 0.1047506033581767, "ece15_soft_target_top_label": 0.11000249335029898, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1907, "nll": 0.8681695016556669, "brier_sum_over_options": 0.42566309616876935, "source_balanced_nll": 0.7044589075652965, "source_balanced_brier": 0.32323273079786935, "hard_label_n": 1712, "hard_accuracy": 0.6378504672897196, "ece15_hard": 0.06361082805090897, "ece15_soft_target_top_label": 0.06251297995699959, "argmax_changes_vs_T1": 0 } }, "3": { "baseline_T1": { "n": 1624, "nll": 0.7716510297442911, "brier_sum_over_options": 0.4526422204640664, "source_balanced_nll": 0.5180689872780773, "source_balanced_brier": 0.2984113713437683, "hard_label_n": 1622, "hard_accuracy": 0.6701602959309494, "ece15_hard": 0.11125267540216917, "ece15_soft_target_top_label": 0.11116108052713702, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1624, "nll": 0.7361560390543282, "brier_sum_over_options": 0.4351161008978419, "source_balanced_nll": 0.523276249215232, "source_balanced_brier": 0.2996238087253387, "hard_label_n": 1622, "hard_accuracy": 0.6701602959309494, "ece15_hard": 0.06738145921212663, "ece15_soft_target_top_label": 0.06726816249045443, "argmax_changes_vs_T1": 0 } }, "5": { "baseline_T1": { "n": 950, "nll": 1.0321579332718551, "brier_sum_over_options": 0.45636761085648353, "source_balanced_nll": 0.7373689648085122, "source_balanced_brier": 0.2650205609566993, "hard_label_n": 809, "hard_accuracy": 0.6007416563658838, "ece15_hard": 0.09083865295497663, "ece15_soft_target_top_label": 0.09551498865798673, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 950, "nll": 1.0045923357674995, "brier_sum_over_options": 0.44522656892983503, "source_balanced_nll": 0.7313919900168762, "source_balanced_brier": 0.26329914692243295, "hard_label_n": 809, "hard_accuracy": 0.6007416563658838, "ece15_hard": 0.041113167177559154, "ece15_soft_target_top_label": 0.039897717676097656, "argmax_changes_vs_T1": 0 } }, "7": { "baseline_T1": { "n": 251, "nll": 1.3152865948450587, "brier_sum_over_options": 0.5748395813866473, "source_balanced_nll": 1.1292949784944664, "source_balanced_brier": 0.23963345216270032, "hard_label_n": 210, "hard_accuracy": 0.5190476190476191, "ece15_hard": 0.17706544190724635, "ece15_soft_target_top_label": 0.1548424924379715, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 251, "nll": 1.2478735234414642, "brier_sum_over_options": 0.5498257461063198, "source_balanced_nll": 1.1229184732025017, "source_balanced_brier": 0.23985396884350701, "hard_label_n": 210, "hard_accuracy": 0.5190476190476191, "ece15_hard": 0.15229650266605774, "ece15_soft_target_top_label": 0.12258486410872624, "argmax_changes_vs_T1": 0 } }, "10": { "baseline_T1": { "n": 177, "nll": 0.3206380389579705, "brier_sum_over_options": 0.14555325025959065, "source_balanced_nll": 0.8944668652808186, "source_balanced_brier": 0.41634359681708066, "hard_label_n": 176, "hard_accuracy": 0.9147727272727273, "ece15_hard": 0.050593007747918244, "ece15_soft_target_top_label": 0.04944359431042387, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 177, "nll": 0.33478579212073806, "brier_sum_over_options": 0.14833699565831765, "source_balanced_nll": 0.8508665765404974, "source_balanced_brier": 0.40010888757583646, "hard_label_n": 176, "hard_accuracy": 0.9147727272727273, "ece15_hard": 0.06139121810069679, "ece15_soft_target_top_label": 0.060381498793323045, "argmax_changes_vs_T1": 0 } }, "9": { "baseline_T1": { "n": 9, "nll": 0.4476933610233881, "brier_sum_over_options": 0.21395305071412862, "source_balanced_nll": 0.4029329139445354, "source_balanced_brier": 0.19255774859625302, "hard_label_n": 9, "hard_accuracy": 0.8888888888888888, "ece15_hard": 0.10807980843377268, "ece15_soft_target_top_label": 0.10807980843377268, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 9, "nll": 0.38291973817084257, "brier_sum_over_options": 0.2054226490983549, "source_balanced_nll": 0.34470287752893114, "source_balanced_brier": 0.18488054585265584, "hard_label_n": 9, "hard_accuracy": 0.8888888888888888, "ece15_hard": 0.11729725515014063, "ece15_soft_target_top_label": 0.11729725515014063, "argmax_changes_vs_T1": 0 } }, "17": { "baseline_T1": { "n": 205, "nll": 1.6132380005191462, "brier_sum_over_options": 0.5970722049808997, "source_balanced_nll": 0.9233621719373097, "source_balanced_brier": 0.349952645388343, "hard_label_n": 205, "hard_accuracy": 0.5365853658536586, "ece15_hard": 0.1328836293853757, "ece15_soft_target_top_label": 0.1328836293853757, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 205, "nll": 1.5793890122940422, "brier_sum_over_options": 0.597448295599059, "source_balanced_nll": 0.9348759607764361, "source_balanced_brier": 0.36374541657296194, "hard_label_n": 205, "hard_accuracy": 0.5365853658536586, "ece15_hard": 0.10492652707851656, "ece15_soft_target_top_label": 0.10492652707851656, "argmax_changes_vs_T1": 0 } }, "11": { "baseline_T1": { "n": 39, "nll": 1.7329636387541798, "brier_sum_over_options": 0.667152374977268, "source_balanced_nll": 0.8892872556929191, "source_balanced_brier": 0.34235450824438984, "hard_label_n": 38, "hard_accuracy": 0.5263157894736842, "ece15_hard": 0.2929348492210114, "ece15_soft_target_top_label": 0.2901902065207956, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 39, "nll": 1.5641598543735449, "brier_sum_over_options": 0.6349368441311507, "source_balanced_nll": 0.8027125855206154, "source_balanced_brier": 0.3258228615687921, "hard_label_n": 38, "hard_accuracy": 0.5263157894736842, "ece15_hard": 0.2295083563890572, "ece15_soft_target_top_label": 0.22773843639126312, "argmax_changes_vs_T1": 0 } }, "16": { "baseline_T1": { "n": 66, "nll": 1.385363536431591, "brier_sum_over_options": 0.6357574280908284, "source_balanced_nll": 1.3853635364315908, "source_balanced_brier": 0.6357574280908282, "hard_label_n": 66, "hard_accuracy": 0.4696969696969697, "ece15_hard": 0.18863946073552296, "ece15_soft_target_top_label": 0.18863946073552296, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 66, "nll": 1.3778733417442919, "brier_sum_over_options": 0.6324273088852371, "source_balanced_nll": 1.3778733417442919, "source_balanced_brier": 0.6324273088852372, "hard_label_n": 66, "hard_accuracy": 0.4696969696969697, "ece15_hard": 0.16052618463860757, "ece15_soft_target_top_label": 0.16052618463860757, "argmax_changes_vs_T1": 0 } }, "24": { "baseline_T1": { "n": 199, "nll": 2.52009298708954, "brier_sum_over_options": 0.8993141646552938, "source_balanced_nll": 2.52009298708954, "source_balanced_brier": 0.8993141646552935, "hard_label_n": 199, "hard_accuracy": 0.2663316582914573, "ece15_hard": 0.20258006075459445, "ece15_soft_target_top_label": 0.20258006075459445, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 199, "nll": 2.39154090351234, "brier_sum_over_options": 0.8603586852754271, "source_balanced_nll": 2.3915409035123396, "source_balanced_brier": 0.8603586852754269, "hard_label_n": 199, "hard_accuracy": 0.2663316582914573, "ece15_hard": 0.12110174706098106, "ece15_soft_target_top_label": 0.12110174706098106, "argmax_changes_vs_T1": 0 } }, "19": { "baseline_T1": { "n": 250, "nll": 2.000001528751946, "brier_sum_over_options": 0.764220994427989, "source_balanced_nll": 1.391390760354478, "source_balanced_brier": 0.5522001864064683, "hard_label_n": 250, "hard_accuracy": 0.448, "ece15_hard": 0.26716842748351055, "ece15_soft_target_top_label": 0.26716842748351055, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 250, "nll": 1.7969496853885047, "brier_sum_over_options": 0.7189072714462958, "source_balanced_nll": 1.224825530838087, "source_balanced_brier": 0.5156647723552193, "hard_label_n": 250, "hard_accuracy": 0.448, "ece15_hard": 0.18910919644120522, "ece15_soft_target_top_label": 0.18910919644120522, "argmax_changes_vs_T1": 0 } }, "6": { "baseline_T1": { "n": 42, "nll": 0.7952857752023642, "brier_sum_over_options": 0.25679041419443505, "source_balanced_nll": 1.3649593159260112, "source_balanced_brier": 0.49742160192872276, "hard_label_n": 28, "hard_accuracy": 0.7857142857142857, "ece15_hard": 0.12202502285315012, "ece15_soft_target_top_label": 0.10849406411904323, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 42, "nll": 0.7946724429502305, "brier_sum_over_options": 0.2508575171249201, "source_balanced_nll": 1.303336818211357, "source_balanced_brier": 0.4687985480720983, "hard_label_n": 28, "hard_accuracy": 0.7857142857142857, "ece15_hard": 0.0990461413124916, "ece15_soft_target_top_label": 0.08061339596167542, "argmax_changes_vs_T1": 0 } }, "46": { "baseline_T1": { "n": 28, "nll": 0.26639228936146403, "brier_sum_over_options": 0.12504826564601207, "source_balanced_nll": 0.2663922893614641, "source_balanced_brier": 0.1250482656460121, "hard_label_n": 28, "hard_accuracy": 0.9285714285714286, "ece15_hard": 0.1005493388651708, "ece15_soft_target_top_label": 0.1005493388651708, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 28, "nll": 0.2733773807804469, "brier_sum_over_options": 0.13021424100801882, "source_balanced_nll": 0.27337738078044704, "source_balanced_brier": 0.13021424100801882, "hard_label_n": 28, "hard_accuracy": 0.9285714285714286, "ece15_hard": 0.08000988992707549, "ece15_soft_target_top_label": 0.08000988992707549, "argmax_changes_vs_T1": 0 } }, "151": { "baseline_T1": { "n": 27, "nll": 0.01318742037085779, "brier_sum_over_options": 0.003061869506362786, "source_balanced_nll": 0.013187420370857789, "source_balanced_brier": 0.003061869506362787, "hard_label_n": 27, "hard_accuracy": 1.0, "ece15_hard": 0.012221792881074414, "ece15_soft_target_top_label": 0.012221792881074414, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 27, "nll": 0.0324108944722723, "brier_sum_over_options": 0.005900679165396043, "source_balanced_nll": 0.0324108944722723, "source_balanced_brier": 0.005900679165396044, "hard_label_n": 27, "hard_accuracy": 1.0, "ece15_hard": 0.030124594022812777, "ece15_soft_target_top_label": 0.030124594022812777, "argmax_changes_vs_T1": 0 } }, "174": { "baseline_T1": { "n": 199, "nll": 6.158517993150494, "brier_sum_over_options": 1.1277309962228477, "source_balanced_nll": 6.158517993150494, "source_balanced_brier": 1.1277309962228474, "hard_label_n": 199, "hard_accuracy": 0.04522613065326633, "ece15_hard": 0.2688069486260517, "ece15_soft_target_top_label": 0.2688069486260517, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 199, "nll": 5.580793881646713, "brier_sum_over_options": 1.0614228245534538, "source_balanced_nll": 5.580793881646713, "source_balanced_brier": 1.0614228245534538, "hard_label_n": 199, "hard_accuracy": 0.04522613065326633, "ece15_hard": 0.16975478917023556, "ece15_soft_target_top_label": 0.16975478917023556, "argmax_changes_vs_T1": 0 } }, "8": { "baseline_T1": { "n": 128, "nll": 0.6207579846866409, "brier_sum_over_options": 0.28413876758825996, "source_balanced_nll": 1.0954452550702964, "source_balanced_brier": 0.2678070927481252, "hard_label_n": 127, "hard_accuracy": 0.7795275590551181, "ece15_hard": 0.08215042145998787, "ece15_soft_target_top_label": 0.08231420366151865, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 128, "nll": 0.5890299247160777, "brier_sum_over_options": 0.2733743371441415, "source_balanced_nll": 1.0525137725327651, "source_balanced_brier": 0.2585817551466508, "hard_label_n": 127, "hard_accuracy": 0.7795275590551181, "ece15_hard": 0.06598347246885168, "ece15_soft_target_top_label": 0.06621948240680248, "argmax_changes_vs_T1": 0 } }, "26": { "baseline_T1": { "n": 10, "nll": 0.5025231596563897, "brier_sum_over_options": 0.24677783766682865, "source_balanced_nll": 0.36044603902995376, "source_balanced_brier": 0.1528364128764804, "hard_label_n": 10, "hard_accuracy": 0.8, "ece15_hard": 0.11644059096268489, "ece15_soft_target_top_label": 0.11644059096268489, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10, "nll": 0.609105311087961, "brier_sum_over_options": 0.2651768630573715, "source_balanced_nll": 0.49112756697666576, "source_balanced_brier": 0.19258974008385663, "hard_label_n": 10, "hard_accuracy": 0.8, "ece15_hard": 0.19986552305602281, "ece15_soft_target_top_label": 0.19986552305602281, "argmax_changes_vs_T1": 0 } }, "60": { "baseline_T1": { "n": 30, "nll": 0.5548174319487342, "brier_sum_over_options": 0.2479334677808331, "source_balanced_nll": 0.5548174319487341, "source_balanced_brier": 0.2479334677808331, "hard_label_n": 30, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.10795255537426568, "ece15_soft_target_top_label": 0.10795255537426568, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 30, "nll": 0.537288122706222, "brier_sum_over_options": 0.24392409515753188, "source_balanced_nll": 0.537288122706222, "source_balanced_brier": 0.2439240951575319, "hard_label_n": 30, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.13914162287049503, "ece15_soft_target_top_label": 0.13914162287049503, "argmax_changes_vs_T1": 0 } }, "77": { "baseline_T1": { "n": 23, "nll": 0.24889848198509765, "brier_sum_over_options": 0.10007059041852256, "source_balanced_nll": 0.24889848198509765, "source_balanced_brier": 0.10007059041852255, "hard_label_n": 23, "hard_accuracy": 0.9565217391304348, "ece15_hard": 0.10821931280222388, "ece15_soft_target_top_label": 0.10821931280222388, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 23, "nll": 0.29383817968107395, "brier_sum_over_options": 0.10984460726191307, "source_balanced_nll": 0.293838179681074, "source_balanced_brier": 0.10984460726191303, "hard_label_n": 23, "hard_accuracy": 0.9565217391304348, "ece15_hard": 0.1510796294711406, "ece15_soft_target_top_label": 0.1510796294711406, "argmax_changes_vs_T1": 0 } }, "14": { "baseline_T1": { "n": 41, "nll": 1.0006219052775647, "brier_sum_over_options": 0.4079138427183757, "source_balanced_nll": 0.5926213025283489, "source_balanced_brier": 0.25326978504067077, "hard_label_n": 41, "hard_accuracy": 0.7317073170731707, "ece15_hard": 0.14411558905984945, "ece15_soft_target_top_label": 0.14411558905984945, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 41, "nll": 0.9985517823084674, "brier_sum_over_options": 0.40942780379684524, "source_balanced_nll": 0.616780589952582, "source_balanced_brier": 0.254690053332879, "hard_label_n": 41, "hard_accuracy": 0.7317073170731707, "ece15_hard": 0.1639515813029694, "ece15_soft_target_top_label": 0.1639515813029694, "argmax_changes_vs_T1": 0 } }, "25": { "baseline_T1": { "n": 1, "nll": 3.0424463714125904e-05, "brier_sum_over_options": 1.1082748128296532e-09, "source_balanced_nll": 3.0424463714125904e-05, "source_balanced_brier": 1.1082748128296532e-09, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 3.042400089481312e-05, "ece15_soft_target_top_label": 3.042400089481312e-05, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1, "nll": 0.00044176654284283076, "brier_sum_over_options": 2.1910825266858835e-07, "source_balanced_nll": 0.00044176654284283076, "source_balanced_brier": 2.1910825266858835e-07, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.000441668978371057, "ece15_soft_target_top_label": 0.000441668978371057, "argmax_changes_vs_T1": 0 } }, "55": { "baseline_T1": { "n": 26, "nll": 0.5464120909376707, "brier_sum_over_options": 0.3016258560842057, "source_balanced_nll": 0.5464120909376707, "source_balanced_brier": 0.30162585608420583, "hard_label_n": 26, "hard_accuracy": 0.8076923076923077, "ece15_hard": 0.19163817005247774, "ece15_soft_target_top_label": 0.19163817005247774, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 26, "nll": 0.5395469799824957, "brier_sum_over_options": 0.2967736418453159, "source_balanced_nll": 0.5395469799824957, "source_balanced_brier": 0.29677364184531596, "hard_label_n": 26, "hard_accuracy": 0.8076923076923077, "ece15_hard": 0.19311131159010958, "ece15_soft_target_top_label": 0.19311131159010958, "argmax_changes_vs_T1": 0 } }, "100": { "baseline_T1": { "n": 10, "nll": 1.2505443561707321, "brier_sum_over_options": 0.47732782682331704, "source_balanced_nll": 1.2505443561707321, "source_balanced_brier": 0.47732782682331715, "hard_label_n": 10, "hard_accuracy": 0.7, "ece15_hard": 0.18308357087203325, "ece15_soft_target_top_label": 0.18308357087203325, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10, "nll": 1.1278725658570645, "brier_sum_over_options": 0.45785612712555296, "source_balanced_nll": 1.1278725658570645, "source_balanced_brier": 0.4578561271255529, "hard_label_n": 10, "hard_accuracy": 0.7, "ece15_hard": 0.16359466837116274, "ece15_soft_target_top_label": 0.16359466837116274, "argmax_changes_vs_T1": 0 } }, "12": { "baseline_T1": { "n": 4, "nll": 0.22633905322119835, "brier_sum_over_options": 0.16548968556165386, "source_balanced_nll": 0.22633905322119835, "source_balanced_brier": 0.16548968556165386, "hard_label_n": 4, "hard_accuracy": 0.75, "ece15_hard": 0.15544531288246594, "ece15_soft_target_top_label": 0.15544531288246594, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 4, "nll": 0.24186842126102173, "brier_sum_over_options": 0.1589598280414067, "source_balanced_nll": 0.24186842126102173, "source_balanced_brier": 0.1589598280414067, "hard_label_n": 4, "hard_accuracy": 0.75, "ece15_hard": 0.17253393357035618, "ece15_soft_target_top_label": 0.17253393357035618, "argmax_changes_vs_T1": 0 } }, "96": { "baseline_T1": { "n": 7, "nll": 0.623225138950921, "brier_sum_over_options": 0.16453340208446093, "source_balanced_nll": 0.6232251389509212, "source_balanced_brier": 0.16453340208446096, "hard_label_n": 6, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.055500330162458915, "ece15_soft_target_top_label": 0.11378535016952016, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 7, "nll": 0.619194953153693, "brier_sum_over_options": 0.16303123677100362, "source_balanced_nll": 0.619194953153693, "source_balanced_brier": 0.16303123677100367, "hard_label_n": 6, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.06957412809356438, "ece15_soft_target_top_label": 0.11649380646134164, "argmax_changes_vs_T1": 0 } }, "13": { "baseline_T1": { "n": 3, "nll": 0.00010359158169846062, "brier_sum_over_options": 2.3205667611402075e-08, "source_balanced_nll": 0.00010359158169846062, "source_balanced_brier": 2.3205667611402072e-08, "hard_label_n": 3, "hard_accuracy": 1.0, "ece15_hard": 0.00010358306109203852, "ece15_soft_target_top_label": 0.00010358306109203852, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3, "nll": 0.0009203684616632247, "brier_sum_over_options": 1.555861995072091e-06, "source_balanced_nll": 0.0009203684616632247, "source_balanced_brier": 1.555861995072091e-06, "hard_label_n": 3, "hard_accuracy": 1.0, "ece15_hard": 0.0009197509787280955, "ece15_soft_target_top_label": 0.0009197509787280955, "argmax_changes_vs_T1": 0 } }, "18": { "baseline_T1": { "n": 2, "nll": 0.0002237176497633426, "brier_sum_over_options": 1.0071760210689769e-07, "source_balanced_nll": 0.0002237176497633426, "source_balanced_brier": 1.0071760210689769e-07, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.00022368672505856946, "ece15_soft_target_top_label": 0.00022368672505856946, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2, "nll": 0.0017783344770914305, "brier_sum_over_options": 4.616268324223769e-06, "source_balanced_nll": 0.0017783344770914305, "source_balanced_brier": 4.616268324223769e-06, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.0017766436394408558, "ece15_soft_target_top_label": 0.0017766436394408558, "argmax_changes_vs_T1": 0 } }, "20": { "baseline_T1": { "n": 1, "nll": 7.0881026342218e-05, "brier_sum_over_options": 5.737917643872293e-09, "source_balanced_nll": 7.0881026342218e-05, "source_balanced_brier": 5.737917643872293e-09, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 7.087851434162129e-05, "ece15_soft_target_top_label": 7.087851434162129e-05, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1, "nll": 0.0008676947730364941, "brier_sum_over_options": 8.306677880213897e-07, "source_balanced_nll": 0.0008676947730364941, "source_balanced_brier": 8.306677880213897e-07, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.0008673184347837148, "ece15_soft_target_top_label": 0.0008673184347837148, "argmax_changes_vs_T1": 0 } }, "15": { "baseline_T1": { "n": 2, "nll": 0.21302172843123862, "brier_sum_over_options": 0.08973998493512275, "source_balanced_nll": 0.21302172843123862, "source_balanced_brier": 0.08973998493512275, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.17346277685360906, "ece15_soft_target_top_label": 0.17346277685360906, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2, "nll": 0.28318648946835734, "brier_sum_over_options": 0.1311346167392156, "source_balanced_nll": 0.28318648946835734, "source_balanced_brier": 0.1311346167392156, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.21632645505646092, "ece15_soft_target_top_label": 0.21632645505646092, "argmax_changes_vs_T1": 0 } } }, "length_bucket": { "256": { "baseline_T1": { "n": 2795, "nll": 0.47037257600293053, "brier_sum_over_options": 0.22541632435245837, "source_balanced_nll": 0.43267872716046035, "source_balanced_brier": 0.19970012750356125, "hard_label_n": 2438, "hard_accuracy": 0.8351107465135357, "ece15_hard": 0.023655126639394466, "ece15_soft_target_top_label": 0.031499768143108876, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2795, "nll": 0.4643596142831439, "brier_sum_over_options": 0.22369417931993363, "source_balanced_nll": 0.43952051425113703, "source_balanced_brier": 0.2019426030434196, "hard_label_n": 2438, "hard_accuracy": 0.8351107465135357, "ece15_hard": 0.018615318161063654, "ece15_soft_target_top_label": 0.011551091968849354, "argmax_changes_vs_T1": 0 } }, "512": { "baseline_T1": { "n": 3633, "nll": 0.7961120431431014, "brier_sum_over_options": 0.3866354321805597, "source_balanced_nll": 0.6723262697528442, "source_balanced_brier": 0.3246270647083185, "hard_label_n": 3381, "hard_accuracy": 0.6876663708961845, "ece15_hard": 0.07185174553774616, "ece15_soft_target_top_label": 0.07249608384810159, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3633, "nll": 0.7767235169627644, "brier_sum_over_options": 0.3775918950131493, "source_balanced_nll": 0.6589270592749751, "source_balanced_brier": 0.3176360770357968, "hard_label_n": 3381, "hard_accuracy": 0.6876663708961845, "ece15_hard": 0.03326765310622277, "ece15_soft_target_top_label": 0.03159729681646724, "argmax_changes_vs_T1": 0 } }, "1024": { "baseline_T1": { "n": 1854, "nll": 1.0735632875231345, "brier_sum_over_options": 0.4711795599195009, "source_balanced_nll": 0.8809112187585068, "source_balanced_brier": 0.3926998694092402, "hard_label_n": 1767, "hard_accuracy": 0.6327108092812677, "ece15_hard": 0.12052197658515143, "ece15_soft_target_top_label": 0.11736387380649344, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1854, "nll": 1.0154387575225823, "brier_sum_over_options": 0.4524833167245523, "source_balanced_nll": 0.8361058718242633, "source_balanced_brier": 0.37985981229200694, "hard_label_n": 1767, "hard_accuracy": 0.6327108092812677, "ece15_hard": 0.06732815180475929, "ece15_soft_target_top_label": 0.06464806043367109, "argmax_changes_vs_T1": 0 } }, "2048": { "baseline_T1": { "n": 663, "nll": 0.6541135388322734, "brier_sum_over_options": 0.34538001341076485, "source_balanced_nll": 0.7069701243738188, "source_balanced_brier": 0.3692422084606182, "hard_label_n": 628, "hard_accuracy": 0.7404458598726115, "ece15_hard": 0.08093880400978243, "ece15_soft_target_top_label": 0.08004513152560036, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 663, "nll": 0.6278369365495612, "brier_sum_over_options": 0.334457831205003, "source_balanced_nll": 0.6834964588529152, "source_balanced_brier": 0.3622779376606939, "hard_label_n": 628, "hard_accuracy": 0.7404458598726115, "ece15_hard": 0.048098854116713524, "ece15_soft_target_top_label": 0.04538127122956523, "argmax_changes_vs_T1": 0 } }, "4096": { "baseline_T1": { "n": 486, "nll": 2.9953129558521128, "brier_sum_over_options": 0.728661071956894, "source_balanced_nll": 0.9605047895483201, "source_balanced_brier": 0.35616517994562186, "hard_label_n": 483, "hard_accuracy": 0.41200828157349895, "ece15_hard": 0.19798039561026823, "ece15_soft_target_top_label": 0.1989211076770336, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 486, "nll": 2.7227974288105425, "brier_sum_over_options": 0.6854139842379053, "source_balanced_nll": 0.8889657813488471, "source_balanced_brier": 0.3341220032284308, "hard_label_n": 483, "hard_accuracy": 0.41200828157349895, "ece15_hard": 0.1369973550975109, "ece15_soft_target_top_label": 0.13779306928471627, "argmax_changes_vs_T1": 0 } }, "8192": { "baseline_T1": { "n": 220, "nll": 0.8261859145282336, "brier_sum_over_options": 0.5000690949390265, "source_balanced_nll": 0.7186466661923275, "source_balanced_brier": 0.42251676814245676, "hard_label_n": 220, "hard_accuracy": 0.6181818181818182, "ece15_hard": 0.17249925138328506, "ece15_soft_target_top_label": 0.17249925138328506, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 220, "nll": 0.7679840071494353, "brier_sum_over_options": 0.4714047900952329, "source_balanced_nll": 0.6695459040455484, "source_balanced_brier": 0.4027365452863051, "hard_label_n": 220, "hard_accuracy": 0.6181818181818182, "ece15_hard": 0.13520540837183176, "ece15_soft_target_top_label": 0.13520540837183176, "argmax_changes_vs_T1": 0 } }, "16384": { "baseline_T1": { "n": 349, "nll": 0.7477118306090438, "brier_sum_over_options": 0.42469192133840344, "source_balanced_nll": 0.8343087539394536, "source_balanced_brier": 0.46741366218669134, "hard_label_n": 349, "hard_accuracy": 0.6991404011461319, "ece15_hard": 0.13091306333007388, "ece15_soft_target_top_label": 0.13091306333007388, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 349, "nll": 0.6971136317498758, "brier_sum_over_options": 0.4067407206604246, "source_balanced_nll": 0.7792284684133695, "source_balanced_brier": 0.4517577915031896, "hard_label_n": 349, "hard_accuracy": 0.6991404011461319, "ece15_hard": 0.08009399151033489, "ece15_soft_target_top_label": 0.08009399151033489, "argmax_changes_vs_T1": 0 } } } }, "accepted": true }, "flash": { "fit_and_selection": { "temperature": 1.2206820938802239, "selected_fit_loss": "nll", "candidates": [ { "temperature": 1.0, "fit_loss": "identity", "at_search_boundary": false, "selection_nll": 0.737196373394814, "selection_brier": 0.33375038462744916, "selection_score": 0.3410571662820997, "eligible": true }, { "temperature": 1.2206820938802239, "fit_loss": "nll", "fit_objective": 0.7515176189861205, "at_search_boundary": false, "bounds": [ 0.05, 20.0 ], "selection_nll": 0.730124808135844, "selection_brier": 0.3315942599645005, "selection_score": 0.3392871454965463, "eligible": true }, { "temperature": 1.227575800140271, "fit_loss": "brier", "fit_objective": 0.3389004130008895, "at_search_boundary": false, "bounds": [ 0.05, 20.0 ], "selection_nll": 0.7302632405973527, "selection_brier": 0.33162998870737326, "selection_score": 0.3393661199072915, "eligible": true } ], "policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization." }, "audit": { "pooled": { "passed": true, "baseline_T1": { "n": 10000, "nll": 0.8570002323848743, "brier_sum_over_options": 0.37919432626901206, "source_balanced_nll": 0.7118519293468761, "source_balanced_brier": 0.3106978930512085, "hard_label_n": 9266, "hard_accuracy": 0.6965249298510684, "ece15_hard": 0.062451612223343875, "ece15_soft_target_top_label": 0.06243402250680544, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10000, "nll": 0.8321960092542425, "brier_sum_over_options": 0.37160163535588864, "source_balanced_nll": 0.7027353839673045, "source_balanced_brier": 0.30851932327767795, "hard_label_n": 9266, "hard_accuracy": 0.6965249298510684, "ece15_hard": 0.030980441201261717, "ece15_soft_target_top_label": 0.02841899539582912, "argmax_changes_vs_T1": 0 }, "source_families": 84, "family_cluster_bootstrap": { "nll_delta_95": [ -0.026075691245677348, 0.0036036286640317217 ], "brier_delta_95": [ -0.006356109913979643, 0.0013150842620000239 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." }, "audit_group": { "passed": true, "baseline_T1": { "n": 8000, "nll": 0.9032170652755189, "brier_sum_over_options": 0.39186413206604753, "source_balanced_nll": 0.7427867356466757, "source_balanced_brier": 0.32151723440277274, "hard_label_n": 7499, "hard_accuracy": 0.6884917989065209, "ece15_hard": 0.06392807226641213, "ece15_soft_target_top_label": 0.061682067108722864, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 8000, "nll": 0.8762545067845626, "brier_sum_over_options": 0.38424953593396205, "source_balanced_nll": 0.7327617378039142, "source_balanced_brier": 0.3194039506091354, "hard_label_n": 7499, "hard_accuracy": 0.6884917989065209, "ece15_hard": 0.028459940989966365, "ece15_soft_target_top_label": 0.02494309557290096, "argmax_changes_vs_T1": 0 }, "source_families": 71, "family_cluster_bootstrap": { "nll_delta_95": [ -0.02793543836392883, 0.004751466455646055 ], "brier_delta_95": [ -0.0066107228100673, 0.0020246791014271407 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." }, "audit_family": { "passed": true, "baseline_T1": { "n": 2000, "nll": 0.672132900822296, "brier_sum_over_options": 0.3285151030808702, "source_balanced_nll": 0.5429002949402779, "source_balanced_brier": 0.25160764413112674, "hard_label_n": 1767, "hard_accuracy": 0.7306168647425014, "ece15_hard": 0.05762287331091378, "ece15_soft_target_top_label": 0.06613371317283424, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2000, "nll": 0.6559620191329626, "brier_sum_over_options": 0.32101003304359504, "source_balanced_nll": 0.538745297628898, "source_balanced_brier": 0.2490725124674108, "hard_label_n": 1767, "hard_accuracy": 0.7306168647425014, "ece15_hard": 0.053041698299378603, "ece15_soft_target_top_label": 0.0460495879467492, "argmax_changes_vs_T1": 0 }, "source_families": 13, "family_cluster_bootstrap": { "nll_delta_95": [ -0.0224885374355191, 0.013856519161634075 ], "brier_delta_95": [ -0.009731275705932721, 0.00442424973831137 ] }, "interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift." } }, "diagnostics": { "type": { "choice": { "baseline_T1": { "n": 9585, "nll": 0.8772217338998575, "brier_sum_over_options": 0.3917876319294491, "source_balanced_nll": 0.723263364856754, "source_balanced_brier": 0.31935528714685957, "hard_label_n": 9124, "hard_accuracy": 0.6924594476106971, "ece15_hard": 0.06400940996750015, "ece15_soft_target_top_label": 0.06551855092083372, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 9585, "nll": 0.8509885943159481, "brier_sum_over_options": 0.38371255767134194, "source_balanced_nll": 0.7133445651926446, "source_balanced_brier": 0.31689551555221435, "hard_label_n": 9124, "hard_accuracy": 0.6924594476106971, "ece15_hard": 0.03143555594499423, "ece15_soft_target_top_label": 0.030205136993845923, "argmax_changes_vs_T1": 0 } }, "noul": { "baseline_T1": { "n": 415, "nll": 0.38995663715327494, "brier_sum_over_options": 0.08833448348518269, "source_balanced_nll": 0.37363381275371293, "source_balanced_brier": 0.0829826996918807, "hard_label_n": 142, "hard_accuracy": 0.9577464788732394, "ece15_hard": 0.06288195970798233, "ece15_soft_target_top_label": 0.030598094376512987, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 415, "nll": 0.39815521933509257, "brier_sum_over_options": 0.09188310428692517, "source_balanced_nll": 0.383695176077421, "source_balanced_brier": 0.08754055668370506, "hard_label_n": 142, "hard_accuracy": 0.9577464788732394, "ece15_hard": 0.07643712522321415, "ece15_soft_target_top_label": 0.03813514826276408, "argmax_changes_vs_T1": 0 } } }, "option_count": { "2": { "baseline_T1": { "n": 3739, "nll": 0.34456045849965816, "brier_sum_over_options": 0.19168513862671624, "source_balanced_nll": 0.364383448081177, "source_balanced_brier": 0.1870597652761825, "hard_label_n": 3402, "hard_accuracy": 0.8641975308641975, "ece15_hard": 0.014521320701320558, "ece15_soft_target_top_label": 0.014485028706927148, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3739, "nll": 0.347994903587601, "brier_sum_over_options": 0.19347148230754105, "source_balanced_nll": 0.3708302552762176, "source_balanced_brier": 0.19057903058028713, "hard_label_n": 3402, "hard_accuracy": 0.8641975308641975, "ece15_hard": 0.03370223301330605, "ece15_soft_target_top_label": 0.03081853496373301, "argmax_changes_vs_T1": 0 } }, "4": { "baseline_T1": { "n": 1907, "nll": 0.8780784289258254, "brier_sum_over_options": 0.42942297750497044, "source_balanced_nll": 0.717786973346652, "source_balanced_brier": 0.3305826004928006, "hard_label_n": 1712, "hard_accuracy": 0.634929906542056, "ece15_hard": 0.07560011676712683, "ece15_soft_target_top_label": 0.08055535862735413, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1907, "nll": 0.8607848007212155, "brier_sum_over_options": 0.42013227775435097, "source_balanced_nll": 0.7131964519501658, "source_balanced_brier": 0.3249144022396803, "hard_label_n": 1712, "hard_accuracy": 0.634929906542056, "ece15_hard": 0.05901139914756741, "ece15_soft_target_top_label": 0.0535352508262639, "argmax_changes_vs_T1": 0 } }, "3": { "baseline_T1": { "n": 1624, "nll": 0.785835337758448, "brier_sum_over_options": 0.47004015333173127, "source_balanced_nll": 0.5631316810018027, "source_balanced_brier": 0.33191132836767095, "hard_label_n": 1622, "hard_accuracy": 0.6368680641183724, "ece15_hard": 0.12745012975209907, "ece15_soft_target_top_label": 0.12733742597917397, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1624, "nll": 0.7579783200548996, "brier_sum_over_options": 0.45355116090254954, "source_balanced_nll": 0.5641112187421008, "source_balanced_brier": 0.328408604630681, "hard_label_n": 1622, "hard_accuracy": 0.6368680641183724, "ece15_hard": 0.09248554910019118, "ece15_soft_target_top_label": 0.09234089293515488, "argmax_changes_vs_T1": 0 } }, "5": { "baseline_T1": { "n": 950, "nll": 1.0198279459718724, "brier_sum_over_options": 0.4537877484898534, "source_balanced_nll": 0.7411751486322502, "source_balanced_brier": 0.2627755636080211, "hard_label_n": 809, "hard_accuracy": 0.6093943139678616, "ece15_hard": 0.0739815683708236, "ece15_soft_target_top_label": 0.0752115324492788, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 950, "nll": 1.0058757211412626, "brier_sum_over_options": 0.4454651778503967, "source_balanced_nll": 0.7446224058860016, "source_balanced_brier": 0.2646854264518146, "hard_label_n": 809, "hard_accuracy": 0.6093943139678616, "ece15_hard": 0.043150569558027416, "ece15_soft_target_top_label": 0.04000418776261011, "argmax_changes_vs_T1": 0 } }, "7": { "baseline_T1": { "n": 251, "nll": 1.2660457526018802, "brier_sum_over_options": 0.5568022902060177, "source_balanced_nll": 1.114109267653641, "source_balanced_brier": 0.2303607680244388, "hard_label_n": 210, "hard_accuracy": 0.5476190476190477, "ece15_hard": 0.15114649965418456, "ece15_soft_target_top_label": 0.11907381211915682, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 251, "nll": 1.2447503100682535, "brier_sum_over_options": 0.5440920981041713, "source_balanced_nll": 1.1284429223392554, "source_balanced_brier": 0.23633839305922688, "hard_label_n": 210, "hard_accuracy": 0.5476190476190477, "ece15_hard": 0.13768108972995582, "ece15_soft_target_top_label": 0.11032470220575231, "argmax_changes_vs_T1": 0 } }, "10": { "baseline_T1": { "n": 177, "nll": 0.3876108425278409, "brier_sum_over_options": 0.17220266211499055, "source_balanced_nll": 1.0141946727194626, "source_balanced_brier": 0.46444338337530816, "hard_label_n": 176, "hard_accuracy": 0.8806818181818182, "ece15_hard": 0.06343742806851521, "ece15_soft_target_top_label": 0.06477182598055643, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 177, "nll": 0.4061828262405223, "brier_sum_over_options": 0.17601555173169234, "source_balanced_nll": 0.9728960016273719, "source_balanced_brier": 0.444851552314224, "hard_label_n": 176, "hard_accuracy": 0.8806818181818182, "ece15_hard": 0.0840859170202171, "ece15_soft_target_top_label": 0.08512382515535123, "argmax_changes_vs_T1": 0 } }, "9": { "baseline_T1": { "n": 9, "nll": 0.5181241857958109, "brier_sum_over_options": 0.22723160082465763, "source_balanced_nll": 0.46633998503336405, "source_balanced_brier": 0.2045084586306961, "hard_label_n": 9, "hard_accuracy": 0.8888888888888888, "ece15_hard": 0.12437858887005425, "ece15_soft_target_top_label": 0.12437858887005425, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 9, "nll": 0.45631116645501074, "brier_sum_over_options": 0.22506445202827713, "source_balanced_nll": 0.41083668871677403, "source_balanced_brier": 0.20255847018619647, "hard_label_n": 9, "hard_accuracy": 0.8888888888888888, "ece15_hard": 0.151315777849797, "ece15_soft_target_top_label": 0.151315777849797, "argmax_changes_vs_T1": 0 } }, "17": { "baseline_T1": { "n": 205, "nll": 1.6704944121216594, "brier_sum_over_options": 0.6114686696715282, "source_balanced_nll": 0.9534717316424565, "source_balanced_brier": 0.3486444747645515, "hard_label_n": 205, "hard_accuracy": 0.5219512195121951, "ece15_hard": 0.1436878908421718, "ece15_soft_target_top_label": 0.1436878908421718, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 205, "nll": 1.6679925852627786, "brier_sum_over_options": 0.619127888892377, "source_balanced_nll": 0.9860940081523526, "source_balanced_brier": 0.36896947799357965, "hard_label_n": 205, "hard_accuracy": 0.5219512195121951, "ece15_hard": 0.12894796792450236, "ece15_soft_target_top_label": 0.12894796792450236, "argmax_changes_vs_T1": 0 } }, "11": { "baseline_T1": { "n": 39, "nll": 1.7691923654930493, "brier_sum_over_options": 0.6467617561982862, "source_balanced_nll": 0.9078878985181886, "source_balanced_brier": 0.3318909017035189, "hard_label_n": 38, "hard_accuracy": 0.5, "ece15_hard": 0.179558657341575, "ece15_soft_target_top_label": 0.1866719201381481, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 39, "nll": 1.644164531310609, "brier_sum_over_options": 0.6318981820120347, "source_balanced_nll": 0.8438272165868839, "source_balanced_brier": 0.32426357491109076, "hard_label_n": 38, "hard_accuracy": 0.5, "ece15_hard": 0.2102966208414764, "ece15_soft_target_top_label": 0.21525189865464361, "argmax_changes_vs_T1": 0 } }, "16": { "baseline_T1": { "n": 66, "nll": 1.5103265510876325, "brier_sum_over_options": 0.6909272488355744, "source_balanced_nll": 1.5103265510876318, "source_balanced_brier": 0.6909272488355747, "hard_label_n": 66, "hard_accuracy": 0.4393939393939394, "ece15_hard": 0.16550696094702416, "ece15_soft_target_top_label": 0.16550696094702416, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 66, "nll": 1.4945455778603822, "brier_sum_over_options": 0.675276809393677, "source_balanced_nll": 1.4945455778603824, "source_balanced_brier": 0.675276809393677, "hard_label_n": 66, "hard_accuracy": 0.4393939393939394, "ece15_hard": 0.17113553935093878, "ece15_soft_target_top_label": 0.17113553935093878, "argmax_changes_vs_T1": 0 } }, "24": { "baseline_T1": { "n": 199, "nll": 2.4924340025418417, "brier_sum_over_options": 0.8903662726669508, "source_balanced_nll": 2.4924340025418408, "source_balanced_brier": 0.8903662726669506, "hard_label_n": 199, "hard_accuracy": 0.23618090452261306, "ece15_hard": 0.19438784148009122, "ece15_soft_target_top_label": 0.19438784148009122, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 199, "nll": 2.4120819565089704, "brier_sum_over_options": 0.8612717774028041, "source_balanced_nll": 2.4120819565089704, "source_balanced_brier": 0.8612717774028039, "hard_label_n": 199, "hard_accuracy": 0.23618090452261306, "ece15_hard": 0.11830923555486911, "ece15_soft_target_top_label": 0.11830923555486911, "argmax_changes_vs_T1": 0 } }, "19": { "baseline_T1": { "n": 250, "nll": 2.0952916773884236, "brier_sum_over_options": 0.7711542523586539, "source_balanced_nll": 1.4374930375559447, "source_balanced_brier": 0.5555214202693165, "hard_label_n": 250, "hard_accuracy": 0.428, "ece15_hard": 0.2610132899083695, "ece15_soft_target_top_label": 0.2610132899083695, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 250, "nll": 1.91575608646441, "brier_sum_over_options": 0.7345501633191516, "source_balanced_nll": 1.2983558513387234, "source_balanced_brier": 0.5274825565155761, "hard_label_n": 250, "hard_accuracy": 0.428, "ece15_hard": 0.1939003334998453, "ece15_soft_target_top_label": 0.1939003334998453, "argmax_changes_vs_T1": 0 } }, "6": { "baseline_T1": { "n": 42, "nll": 0.8288988397766355, "brier_sum_over_options": 0.27777704347580634, "source_balanced_nll": 1.4039889742757714, "source_balanced_brier": 0.47600245332885854, "hard_label_n": 28, "hard_accuracy": 0.7857142857142857, "ece15_hard": 0.19954755091637338, "ece15_soft_target_top_label": 0.18117467075035296, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 42, "nll": 0.8314159941361794, "brier_sum_over_options": 0.2756424929725876, "source_balanced_nll": 1.3491880345668303, "source_balanced_brier": 0.4640535072101043, "hard_label_n": 28, "hard_accuracy": 0.7857142857142857, "ece15_hard": 0.11473882152610511, "ece15_soft_target_top_label": 0.09996606237372609, "argmax_changes_vs_T1": 0 } }, "46": { "baseline_T1": { "n": 28, "nll": 0.22063032128614127, "brier_sum_over_options": 0.1149215211280737, "source_balanced_nll": 0.22063032128614127, "source_balanced_brier": 0.1149215211280737, "hard_label_n": 28, "hard_accuracy": 0.9285714285714286, "ece15_hard": 0.03850126263478336, "ece15_soft_target_top_label": 0.03850126263478336, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 28, "nll": 0.2321652667445951, "brier_sum_over_options": 0.11680010408691169, "source_balanced_nll": 0.2321652667445952, "source_balanced_brier": 0.11680010408691172, "hard_label_n": 28, "hard_accuracy": 0.9285714285714286, "ece15_hard": 0.07196001785616364, "ece15_soft_target_top_label": 0.07196001785616364, "argmax_changes_vs_T1": 0 } }, "151": { "baseline_T1": { "n": 27, "nll": 0.004604715853748762, "brier_sum_over_options": 0.00012750703767116354, "source_balanced_nll": 0.004604715853748763, "source_balanced_brier": 0.0001275070376711636, "hard_label_n": 27, "hard_accuracy": 1.0, "ece15_hard": 0.004568269164576533, "ece15_soft_target_top_label": 0.004568269164576533, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 27, "nll": 0.014876589347728297, "brier_sum_over_options": 0.0006647270336140146, "source_balanced_nll": 0.014876589347728297, "source_balanced_brier": 0.0006647270336140146, "hard_label_n": 27, "hard_accuracy": 1.0, "ece15_hard": 0.014646935063854838, "ece15_soft_target_top_label": 0.014646935063854838, "argmax_changes_vs_T1": 0 } }, "174": { "baseline_T1": { "n": 199, "nll": 6.089131697100824, "brier_sum_over_options": 1.1393897207933026, "source_balanced_nll": 6.089131697100823, "source_balanced_brier": 1.1393897207933021, "hard_label_n": 199, "hard_accuracy": 0.05527638190954774, "ece15_hard": 0.2599587761255519, "ece15_soft_target_top_label": 0.2599587761255519, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 199, "nll": 5.5972312315964174, "brier_sum_over_options": 1.079531224764791, "source_balanced_nll": 5.5972312315964174, "source_balanced_brier": 1.0795312247647908, "hard_label_n": 199, "hard_accuracy": 0.05527638190954774, "ece15_hard": 0.17813420139289587, "ece15_soft_target_top_label": 0.17813420139289587, "argmax_changes_vs_T1": 0 } }, "8": { "baseline_T1": { "n": 128, "nll": 0.6217994993057765, "brier_sum_over_options": 0.28379704746901147, "source_balanced_nll": 1.13244544662642, "source_balanced_brier": 0.28692143547730253, "hard_label_n": 127, "hard_accuracy": 0.7795275590551181, "ece15_hard": 0.07812389647018171, "ece15_soft_target_top_label": 0.0783288755985589, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 128, "nll": 0.6027463276612296, "brier_sum_over_options": 0.2772966153890374, "source_balanced_nll": 1.0967937460023398, "source_balanced_brier": 0.27956312011942985, "hard_label_n": 127, "hard_accuracy": 0.7795275590551181, "ece15_hard": 0.06140453183929856, "ece15_soft_target_top_label": 0.061690747026549635, "argmax_changes_vs_T1": 0 } }, "26": { "baseline_T1": { "n": 10, "nll": 0.6782343122294427, "brier_sum_over_options": 0.31888276696776036, "source_balanced_nll": 0.6839362107385382, "source_balanced_brier": 0.3092345143753137, "hard_label_n": 10, "hard_accuracy": 0.8, "ece15_hard": 0.15755902064415173, "ece15_soft_target_top_label": 0.15755902064415173, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10, "nll": 0.7667196391891683, "brier_sum_over_options": 0.3324455276288491, "source_balanced_nll": 0.821816955421676, "source_balanced_brier": 0.36423887110224457, "hard_label_n": 10, "hard_accuracy": 0.8, "ece15_hard": 0.16071276209491955, "ece15_soft_target_top_label": 0.16071276209491955, "argmax_changes_vs_T1": 0 } }, "60": { "baseline_T1": { "n": 30, "nll": 0.6068868724152872, "brier_sum_over_options": 0.23492470460340126, "source_balanced_nll": 0.6068868724152869, "source_balanced_brier": 0.23492470460340123, "hard_label_n": 30, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.1213210928743784, "ece15_soft_target_top_label": 0.1213210928743784, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 30, "nll": 0.574208985443045, "brier_sum_over_options": 0.22809119128151892, "source_balanced_nll": 0.574208985443045, "source_balanced_brier": 0.2280911912815189, "hard_label_n": 30, "hard_accuracy": 0.8333333333333334, "ece15_hard": 0.12498927840457866, "ece15_soft_target_top_label": 0.12498927840457866, "argmax_changes_vs_T1": 0 } }, "77": { "baseline_T1": { "n": 23, "nll": 0.17979156775813146, "brier_sum_over_options": 0.07855611335844186, "source_balanced_nll": 0.1797915677581315, "source_balanced_brier": 0.07855611335844186, "hard_label_n": 23, "hard_accuracy": 0.9565217391304348, "ece15_hard": 0.08498243179602454, "ece15_soft_target_top_label": 0.08498243179602454, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 23, "nll": 0.2166132535367895, "brier_sum_over_options": 0.08438267023213168, "source_balanced_nll": 0.2166132535367895, "source_balanced_brier": 0.08438267023213171, "hard_label_n": 23, "hard_accuracy": 0.9565217391304348, "ece15_hard": 0.1159196567852966, "ece15_soft_target_top_label": 0.1159196567852966, "argmax_changes_vs_T1": 0 } }, "14": { "baseline_T1": { "n": 41, "nll": 1.0668751741666684, "brier_sum_over_options": 0.4378387318531714, "source_balanced_nll": 0.6278262886817039, "source_balanced_brier": 0.2651493353026454, "hard_label_n": 41, "hard_accuracy": 0.6585365853658537, "ece15_hard": 0.14833965614136363, "ece15_soft_target_top_label": 0.14833965614136363, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 41, "nll": 1.0805650109548282, "brier_sum_over_options": 0.440672101920061, "source_balanced_nll": 0.6655367164535373, "source_balanced_brier": 0.2739978949113573, "hard_label_n": 41, "hard_accuracy": 0.6585365853658537, "ece15_hard": 0.19374436248848903, "ece15_soft_target_top_label": 0.19374436248848903, "argmax_changes_vs_T1": 0 } }, "25": { "baseline_T1": { "n": 1, "nll": 0.00021037501346573512, "brier_sum_over_options": 6.988059189787206e-08, "source_balanced_nll": 0.00021037501346573512, "source_balanced_brier": 6.988059189787206e-08, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.00021035288619430936, "ece15_soft_target_top_label": 0.00021035288619430936, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1, "nll": 0.0012814849995642606, "brier_sum_over_options": 2.2566101897773096e-06, "source_balanced_nll": 0.0012814849995642606, "source_balanced_brier": 2.2566101897773096e-06, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.0012806642482930952, "ece15_soft_target_top_label": 0.0012806642482930952, "argmax_changes_vs_T1": 0 } }, "55": { "baseline_T1": { "n": 26, "nll": 0.5595560475712149, "brier_sum_over_options": 0.31853276986126855, "source_balanced_nll": 0.559556047571215, "source_balanced_brier": 0.3185327698612685, "hard_label_n": 26, "hard_accuracy": 0.8076923076923077, "ece15_hard": 0.13763858319692607, "ece15_soft_target_top_label": 0.13763858319692607, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 26, "nll": 0.5612721713452827, "brier_sum_over_options": 0.3149761008589313, "source_balanced_nll": 0.5612721713452827, "source_balanced_brier": 0.3149761008589313, "hard_label_n": 26, "hard_accuracy": 0.8076923076923077, "ece15_hard": 0.19223441714332415, "ece15_soft_target_top_label": 0.19223441714332415, "argmax_changes_vs_T1": 0 } }, "100": { "baseline_T1": { "n": 10, "nll": 1.4187601674943622, "brier_sum_over_options": 0.5283951877402946, "source_balanced_nll": 1.4187601674943622, "source_balanced_brier": 0.5283951877402946, "hard_label_n": 10, "hard_accuracy": 0.7, "ece15_hard": 0.2810215214838676, "ece15_soft_target_top_label": 0.2810215214838676, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 10, "nll": 1.2693268402366198, "brier_sum_over_options": 0.5146513335916041, "source_balanced_nll": 1.2693268402366196, "source_balanced_brier": 0.514651333591604, "hard_label_n": 10, "hard_accuracy": 0.7, "ece15_hard": 0.2791317548095704, "ece15_soft_target_top_label": 0.2791317548095704, "argmax_changes_vs_T1": 0 } }, "12": { "baseline_T1": { "n": 4, "nll": 0.19016785862226854, "brier_sum_over_options": 0.11915582199701051, "source_balanced_nll": 0.19016785862226854, "source_balanced_brier": 0.11915582199701051, "hard_label_n": 4, "hard_accuracy": 1.0, "ece15_hard": 0.14435070393884972, "ece15_soft_target_top_label": 0.14435070393884972, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 4, "nll": 0.22933654270800785, "brier_sum_over_options": 0.12460462824330547, "source_balanced_nll": 0.22933654270800785, "source_balanced_brier": 0.12460462824330547, "hard_label_n": 4, "hard_accuracy": 1.0, "ece15_hard": 0.17996493569310962, "ece15_soft_target_top_label": 0.17996493569310962, "argmax_changes_vs_T1": 0 } }, "96": { "baseline_T1": { "n": 7, "nll": 0.6344054687477539, "brier_sum_over_options": 0.14243373382021554, "source_balanced_nll": 0.634405468747754, "source_balanced_brier": 0.14243373382021557, "hard_label_n": 6, "hard_accuracy": 1.0, "ece15_hard": 0.12852053366837077, "ece15_soft_target_top_label": 0.11544498217299713, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 7, "nll": 0.6260825930519244, "brier_sum_over_options": 0.14654161450034958, "source_balanced_nll": 0.6260825930519244, "source_balanced_brier": 0.1465416145003496, "hard_label_n": 6, "hard_accuracy": 1.0, "ece15_hard": 0.1503075372779968, "ece15_soft_target_top_label": 0.13526183634693043, "argmax_changes_vs_T1": 0 } }, "13": { "baseline_T1": { "n": 3, "nll": 0.055850372198111974, "brier_sum_over_options": 0.014579928048139838, "source_balanced_nll": 0.05585037219811197, "source_balanced_brier": 0.014579928048139838, "hard_label_n": 3, "hard_accuracy": 1.0, "ece15_hard": 0.051428429669643005, "ece15_soft_target_top_label": 0.051428429669643005, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3, "nll": 0.07925563588489214, "brier_sum_over_options": 0.026024010571508705, "source_balanced_nll": 0.07925563588489212, "source_balanced_brier": 0.026024010571508705, "hard_label_n": 3, "hard_accuracy": 1.0, "ece15_hard": 0.0706061979456305, "ece15_soft_target_top_label": 0.0706061979456305, "argmax_changes_vs_T1": 0 } }, "18": { "baseline_T1": { "n": 2, "nll": 0.00015321243970713315, "brier_sum_over_options": 5.812257036566845e-08, "source_balanced_nll": 0.00015321243970713315, "source_balanced_brier": 5.812257036566845e-08, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.00015319539995872766, "ece15_soft_target_top_label": 0.00015319539995872766, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2, "nll": 0.0009299787065514727, "brier_sum_over_options": 1.582427728624678e-06, "source_balanced_nll": 0.0009299787065514727, "source_balanced_brier": 1.582427728624678e-06, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.0009294456581273591, "ece15_soft_target_top_label": 0.0009294456581273591, "argmax_changes_vs_T1": 0 } }, "20": { "baseline_T1": { "n": 1, "nll": 0.0003486236345549233, "brier_sum_over_options": 1.3883399864364483e-07, "source_balanced_nll": 0.0003486236345549233, "source_balanced_brier": 1.3883399864364483e-07, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.0003485628723969292, "ece15_soft_target_top_label": 0.0003485628723969292, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1, "nll": 0.002337668970627584, "brier_sum_over_options": 6.0080194953139875e-06, "source_balanced_nll": 0.002337668970627584, "source_balanced_brier": 6.0080194953139875e-06, "hard_label_n": 1, "hard_accuracy": 1.0, "ece15_hard": 0.0023349387503842634, "ece15_soft_target_top_label": 0.0023349387503842634, "argmax_changes_vs_T1": 0 } }, "15": { "baseline_T1": { "n": 2, "nll": 0.3751794034386599, "brier_sum_over_options": 0.20958336260990054, "source_balanced_nll": 0.3751794034386599, "source_balanced_brier": 0.20958336260990054, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.26391215423193704, "ece15_soft_target_top_label": 0.26391215423193704, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2, "nll": 0.421759116090247, "brier_sum_over_options": 0.2319420283401216, "source_balanced_nll": 0.421759116090247, "source_balanced_brier": 0.2319420283401216, "hard_label_n": 2, "hard_accuracy": 1.0, "ece15_hard": 0.28501566512025645, "ece15_soft_target_top_label": 0.28501566512025645, "argmax_changes_vs_T1": 0 } } }, "length_bucket": { "128": { "baseline_T1": { "n": 2795, "nll": 0.47894481602307243, "brier_sum_over_options": 0.2329793196852082, "source_balanced_nll": 0.43759908640966344, "source_balanced_brier": 0.20097543908466445, "hard_label_n": 2438, "hard_accuracy": 0.8264971287940935, "ece15_hard": 0.013381028589857631, "ece15_soft_target_top_label": 0.018470153965915073, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 2795, "nll": 0.47843872330270093, "brier_sum_over_options": 0.23320563196516056, "source_balanced_nll": 0.4478829139106674, "source_balanced_brier": 0.2049984771990012, "hard_label_n": 2438, "hard_accuracy": 0.8264971287940935, "ece15_hard": 0.03341391723188097, "ece15_soft_target_top_label": 0.024613462816985565, "argmax_changes_vs_T1": 0 } }, "256": { "baseline_T1": { "n": 3633, "nll": 0.8082300419985322, "brier_sum_over_options": 0.3924719370871102, "source_balanced_nll": 0.7027756248316769, "source_balanced_brier": 0.33934571675261077, "hard_label_n": 3381, "hard_accuracy": 0.6805678793256433, "ece15_hard": 0.053401579972982206, "ece15_soft_target_top_label": 0.05343383213420854, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 3633, "nll": 0.7997880763542585, "brier_sum_over_options": 0.38734689731454064, "source_balanced_nll": 0.6938142892251751, "source_balanced_brier": 0.3351683380384077, "hard_label_n": 3381, "hard_accuracy": 0.6805678793256433, "ece15_hard": 0.03154354282947759, "ece15_soft_target_top_label": 0.02792805372156109, "argmax_changes_vs_T1": 0 } }, "512": { "baseline_T1": { "n": 1854, "nll": 1.0837142432662947, "brier_sum_over_options": 0.47187547077627484, "source_balanced_nll": 0.873295200388099, "source_balanced_brier": 0.38996230717858815, "hard_label_n": 1767, "hard_accuracy": 0.619128466327108, "ece15_hard": 0.10746793409973858, "ece15_soft_target_top_label": 0.10391363109551448, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 1854, "nll": 1.0427599018966052, "brier_sum_over_options": 0.4583898241454803, "source_balanced_nll": 0.8466079492872749, "source_balanced_brier": 0.38203669707621724, "hard_label_n": 1767, "hard_accuracy": 0.619128466327108, "ece15_hard": 0.06000675148381363, "ece15_soft_target_top_label": 0.05644598288132306, "argmax_changes_vs_T1": 0 } }, "1024": { "baseline_T1": { "n": 663, "nll": 0.6370531661532262, "brier_sum_over_options": 0.34126908148587454, "source_balanced_nll": 0.7172978633192849, "source_balanced_brier": 0.38140375127676807, "hard_label_n": 628, "hard_accuracy": 0.75, "ece15_hard": 0.06338353188088795, "ece15_soft_target_top_label": 0.05893613871948881, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 663, "nll": 0.6251593312320077, "brier_sum_over_options": 0.3354781540029996, "source_balanced_nll": 0.6996748713870239, "source_balanced_brier": 0.37360134750248913, "hard_label_n": 628, "hard_accuracy": 0.75, "ece15_hard": 0.03617791572725478, "ece15_soft_target_top_label": 0.037583535790362256, "argmax_changes_vs_T1": 0 } }, "2048": { "baseline_T1": { "n": 486, "nll": 2.934371753220357, "brier_sum_over_options": 0.7212951137452513, "source_balanced_nll": 0.9372840096614883, "source_balanced_brier": 0.34372131190130417, "hard_label_n": 483, "hard_accuracy": 0.4057971014492754, "ece15_hard": 0.18174196601739398, "ece15_soft_target_top_label": 0.18173537988281505, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 486, "nll": 2.7125354846028364, "brier_sum_over_options": 0.6861812323728023, "source_balanced_nll": 0.8875682417282856, "source_balanced_brier": 0.3308451224553599, "hard_label_n": 483, "hard_accuracy": 0.4057971014492754, "ece15_hard": 0.13102365024567325, "ece15_soft_target_top_label": 0.13083062698046766, "argmax_changes_vs_T1": 0 } }, "4096": { "baseline_T1": { "n": 220, "nll": 0.8239227941876504, "brier_sum_over_options": 0.50357074573417, "source_balanced_nll": 0.6915513187665021, "source_balanced_brier": 0.4315091788503526, "hard_label_n": 220, "hard_accuracy": 0.6136363636363636, "ece15_hard": 0.17151284595484564, "ece15_soft_target_top_label": 0.17151284595484564, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 220, "nll": 0.7783430711541587, "brier_sum_over_options": 0.47929985097472433, "source_balanced_nll": 0.6679516357179132, "source_balanced_brier": 0.41560952370795534, "hard_label_n": 220, "hard_accuracy": 0.6136363636363636, "ece15_hard": 0.14466059920242538, "ece15_soft_target_top_label": 0.14466059920242538, "argmax_changes_vs_T1": 0 } }, "8192": { "baseline_T1": { "n": 349, "nll": 0.7338443482003226, "brier_sum_over_options": 0.4368538783565455, "source_balanced_nll": 0.8297455107236107, "source_balanced_brier": 0.48946470854565366, "hard_label_n": 349, "hard_accuracy": 0.6934097421203438, "ece15_hard": 0.14190864490761737, "ece15_soft_target_top_label": 0.14190864490761737, "argmax_changes_vs_T1": 0 }, "candidate": { "n": 349, "nll": 0.6928612138175814, "brier_sum_over_options": 0.41767489414099057, "source_balanced_nll": 0.7825939870960537, "source_balanced_brier": 0.47023329484963994, "hard_label_n": 349, "hard_accuracy": 0.6934097421203438, "ece15_hard": 0.11256430198077799, "ece15_soft_target_top_label": 0.11256430198077799, "argmax_changes_vs_T1": 0 } } } }, "accepted": true } }, "accepted": true, "policy": "Select on selection only, then audit once; pooled and each audit panel must not worsen pooled or equal-family NLL/Brier versus T=1.0. No audit refitting or public/sealed benchmark fitting." }
{
"source_release_id": "2818c9c47a86170ce9192c4cabd0715cb26f4c89b0ad557f8c3f15358b7f012a",
"data_counts": {
"fit": 7632,
"selection": 2368,
"audit_family": 2000,
"audit_group": 8000
},
"data_identity_sha256": "676e3c06bb9d3858875515c3c2da5f25b2b478fa2b9253ad11fa8f6b196b69ed",
"data_scope": "Reused historical holdout, newly screened and rescored. Not a fresh sealed test.",
"selection_and_audit_scoring": "Singleton, exact native serving numerical path; fit batching screened separately",
"baseline_temperature": 1.0,
"modes": {
"default": {
"fit_and_selection": {
"temperature": 1.2626014578759173,
"selected_fit_loss": "nll",
"candidates": [
{
"temperature": 1.0,
"fit_loss": "identity",
"at_search_boundary": false,
"selection_nll": 0.7275512028121913,
"selection_brier": 0.327622453770135,
"selection_score": 0.33580234422981187,
"eligible": true
},
{
"temperature": 1.2626014578759173,
"fit_loss": "nll",
"fit_objective": 0.7168196817779551,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712893716489562,
"selection_brier": 0.32333401069095247,
"selection_score": 0.3308736186536595,
"eligible": true
},
{
"temperature": 1.27003745705363,
"fit_loss": "brier",
"fit_objective": 0.31996540663853446,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712955260655177,
"selection_brier": 0.32334033241842747,
"selection_score": 0.3309141805567075,
"eligible": true
}
],
"policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization."
},
"audit": {
"pooled": {
"passed": true,
"baseline_T1": {
"n": 10000,
"nll": 0.852946444060366,
"brier_sum_over_options": 0.3749600967063925,
"source_balanced_nll": 0.6999376751969619,
"source_balanced_brier": 0.30462924474543973,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.07901558557138746,
"ece15_soft_target_top_label": 0.07927800766590869,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10000,
"nll": 0.8154129693881843,
"brier_sum_over_options": 0.3636438958749284,
"source_balanced_nll": 0.6835431229745554,
"source_balanced_brier": 0.2998612344780709,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.038995968182245856,
"ece15_soft_target_top_label": 0.036747079044559566,
"argmax_changes_vs_T1": 0
},
"source_families": 84,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.03605991235662911,
-0.0013140520669599177
],
"brier_delta_95": [
-0.009391151837829768,
-0.0006976623009813496
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_group": {
"passed": true,