Text Classification
Transformers
Safetensors
English
basedecision_dyno
feature-extraction
decision-model
system-one
decision-index
multiple-choice
calibration
read-twice
custom-code
lora-merged
custom_code
Instructions to use onlyaady/BaseDecision-Dyno with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use onlyaady/BaseDecision-Dyno with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="onlyaady/BaseDecision-Dyno", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("onlyaady/BaseDecision-Dyno", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download calibration.json from onlyaady/BaseDecision-Dyno: direct link, hf CLI and curl.
- Browser
- Download file 97.2 kB
-
https://huggingface.co/onlyaady/BaseDecision-Dyno/resolve/main/calibration.json
- Command line
-
hf download hf://onlyaady/BaseDecision-Dyno/calibration.json
-
curl -L -o calibration.json https://huggingface.co/onlyaady/BaseDecision-Dyno/resolve/main/calibration.json
97.2 kB
{
"source_release_id": "2818c9c47a86170ce9192c4cabd0715cb26f4c89b0ad557f8c3f15358b7f012a",
"data_counts": {
"fit": 7632,
"selection": 2368,
"audit_family": 2000,
"audit_group": 8000
},
"data_identity_sha256": "676e3c06bb9d3858875515c3c2da5f25b2b478fa2b9253ad11fa8f6b196b69ed",
"data_scope": "Reused historical holdout, newly screened and rescored. Not a fresh sealed test.",
"selection_and_audit_scoring": "Singleton, exact native serving numerical path; fit batching screened separately",
"baseline_temperature": 1.0,
"modes": {
"default": {
"fit_and_selection": {
"temperature": 1.2626014578759173,
"selected_fit_loss": "nll",
"candidates": [
{
"temperature": 1.0,
"fit_loss": "identity",
"at_search_boundary": false,
"selection_nll": 0.7275512028121913,
"selection_brier": 0.327622453770135,
"selection_score": 0.33580234422981187,
"eligible": true
},
{
"temperature": 1.2626014578759173,
"fit_loss": "nll",
"fit_objective": 0.7168196817779551,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712893716489562,
"selection_brier": 0.32333401069095247,
"selection_score": 0.3308736186536595,
"eligible": true
},
{
"temperature": 1.27003745705363,
"fit_loss": "brier",
"fit_objective": 0.31996540663853446,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712955260655177,
"selection_brier": 0.32334033241842747,
"selection_score": 0.3309141805567075,
"eligible": true
}
],
"policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization."
},
"audit": {
"pooled": {
"passed": true,
"baseline_T1": {
"n": 10000,
"nll": 0.852946444060366,
"brier_sum_over_options": 0.3749600967063925,
"source_balanced_nll": 0.6999376751969619,
"source_balanced_brier": 0.30462924474543973,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.07901558557138746,
"ece15_soft_target_top_label": 0.07927800766590869,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10000,
"nll": 0.8154129693881843,
"brier_sum_over_options": 0.3636438958749284,
"source_balanced_nll": 0.6835431229745554,
"source_balanced_brier": 0.2998612344780709,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.038995968182245856,
"ece15_soft_target_top_label": 0.036747079044559566,
"argmax_changes_vs_T1": 0
},
"source_families": 84,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.03605991235662911,
-0.0013140520669599177
],
"brier_delta_95": [
-0.009391151837829768,
-0.0006976623009813496
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_group": {
"passed": true,
"baseline_T1": {
"n": 8000,
"nll": 0.888022765832513,
"brier_sum_over_options": 0.3811418691518153,
"source_balanced_nll": 0.7267924319475662,
"source_balanced_brier": 0.3125574139333116,
"hard_label_n": 7499,
"hard_accuracy": 0.6998266435524737,
"ece15_hard": 0.07459343476372879,
"ece15_soft_target_top_label": 0.07301347448112568,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 8000,
"nll": 0.8499313997523429,
"brier_sum_over_options": 0.371009631033076,
"source_balanced_nll": 0.7097259735906477,
"source_balanced_brier": 0.3083857511306294,
"hard_label_n": 7499,
"hard_accuracy": 0.6998266435524737,
"ece15_hard": 0.03078003605959842,
"ece15_soft_target_top_label": 0.02856035757287184,
"argmax_changes_vs_T1": 0
},
"source_families": 71,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.038382270574252315,
0.0003448221832350238
],
"brier_delta_95": [
-0.00891044962013338,
0.000358978773719409
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_family": {
"passed": true,
"baseline_T1": {
"n": 2000,
"nll": 0.7126411569717783,
"brier_sum_over_options": 0.3502330069247016,
"source_balanced_nll": 0.5532693883282767,
"source_balanced_brier": 0.26132924379629274,
"hard_label_n": 1767,
"hard_accuracy": 0.7215619694397284,
"ece15_hard": 0.0977828232095496,
"ece15_soft_target_top_label": 0.10433614040504041,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2000,
"nll": 0.6773392479315501,
"brier_sum_over_options": 0.3341809552423381,
"source_balanced_nll": 0.5405444773020512,
"source_balanced_brier": 0.25330425891409736,
"hard_label_n": 1767,
"hard_accuracy": 0.7215619694397284,
"ece15_hard": 0.07800818374407796,
"ece15_soft_target_top_label": 0.07447692885921983,
"argmax_changes_vs_T1": 0
},
"source_families": 13,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.04730023306508244,
0.01337183299465118
],
"brier_delta_95": [
-0.02130890503696608,
0.001674475477007773
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
}
},
"diagnostics": {
"type": {
"choice": {
"baseline_T1": {
"n": 9585,
"nll": 0.8736567431397109,
"brier_sum_over_options": 0.3877836569453464,
"source_balanced_nll": 0.7113112552360633,
"source_balanced_brier": 0.3134031210640267,
"hard_label_n": 9124,
"hard_accuracy": 0.6998027181060938,
"ece15_hard": 0.08082652934851452,
"ece15_soft_target_top_label": 0.08301943064283653,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 9585,
"nll": 0.8340462097507013,
"brier_sum_over_options": 0.37579718896533437,
"source_balanced_nll": 0.6937124217595521,
"source_balanced_brier": 0.30821745985835847,
"hard_label_n": 9124,
"hard_accuracy": 0.6998027181060938,
"ece15_hard": 0.03971737650838311,
"ece15_soft_target_top_label": 0.039635874875866624,
"argmax_changes_vs_T1": 0
}
},
"noul": {
"baseline_T1": {
"n": 415,
"nll": 0.3746133918301991,
"brier_sum_over_options": 0.07878220540428955,
"source_balanced_nll": 0.3561816083069769,
"source_balanced_brier": 0.07147533612369922,
"hard_label_n": 142,
"hard_accuracy": 0.971830985915493,
"ece15_hard": 0.05048296869603257,
"ece15_soft_target_top_label": 0.028886424994996142,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 415,
"nll": 0.38505246607559207,
"brier_sum_over_options": 0.08294675305193852,
"source_balanced_nll": 0.3686868791865422,
"source_balanced_brier": 0.07672669673180794,
"hard_label_n": 142,
"hard_accuracy": 0.971830985915493,
"ece15_hard": 0.08497080667781354,
"ece15_soft_target_top_label": 0.04488882620673573,
"argmax_changes_vs_T1": 0
}
}
},
"option_count": {
"2": {
"baseline_T1": {
"n": 3739,
"nll": 0.3332287517972207,
"brier_sum_over_options": 0.1835811451336453,
"source_balanced_nll": 0.34593070447439217,
"source_balanced_brier": 0.17391155278332907,
"hard_label_n": 3402,
"hard_accuracy": 0.8644914756025868,
"ece15_hard": 0.02245159561941003,
"ece15_soft_target_top_label": 0.023019111682206526,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3739,
"nll": 0.3324121227893266,
"brier_sum_over_options": 0.18317845144837797,
"source_balanced_nll": 0.3509764214190602,
"source_balanced_brier": 0.17633861775459045,
"hard_label_n": 3402,
"hard_accuracy": 0.8644914756025868,
"ece15_hard": 0.018485063230447577,
"ece15_soft_target_top_label": 0.017437953045210633,
"argmax_changes_vs_T1": 0
}
},
"4": {
"baseline_T1": {
"n": 1907,
"nll": 0.9022618813320882,
"brier_sum_over_options": 0.4423083809261321,
"source_balanced_nll": 0.7193985221889149,
"source_balanced_brier": 0.33394772375263204,
"hard_label_n": 1712,
"hard_accuracy": 0.6378504672897196,
"ece15_hard": 0.1047506033581767,
"ece15_soft_target_top_label": 0.11000249335029898,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1907,
"nll": 0.8681695016556669,
"brier_sum_over_options": 0.42566309616876935,
"source_balanced_nll": 0.7044589075652965,
"source_balanced_brier": 0.32323273079786935,
"hard_label_n": 1712,
"hard_accuracy": 0.6378504672897196,
"ece15_hard": 0.06361082805090897,
"ece15_soft_target_top_label": 0.06251297995699959,
"argmax_changes_vs_T1": 0
}
},
"3": {
"baseline_T1": {
"n": 1624,
"nll": 0.7716510297442911,
"brier_sum_over_options": 0.4526422204640664,
"source_balanced_nll": 0.5180689872780773,
"source_balanced_brier": 0.2984113713437683,
"hard_label_n": 1622,
"hard_accuracy": 0.6701602959309494,
"ece15_hard": 0.11125267540216917,
"ece15_soft_target_top_label": 0.11116108052713702,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1624,
"nll": 0.7361560390543282,
"brier_sum_over_options": 0.4351161008978419,
"source_balanced_nll": 0.523276249215232,
"source_balanced_brier": 0.2996238087253387,
"hard_label_n": 1622,
"hard_accuracy": 0.6701602959309494,
"ece15_hard": 0.06738145921212663,
"ece15_soft_target_top_label": 0.06726816249045443,
"argmax_changes_vs_T1": 0
}
},
"5": {
"baseline_T1": {
"n": 950,
"nll": 1.0321579332718551,
"brier_sum_over_options": 0.45636761085648353,
"source_balanced_nll": 0.7373689648085122,
"source_balanced_brier": 0.2650205609566993,
"hard_label_n": 809,
"hard_accuracy": 0.6007416563658838,
"ece15_hard": 0.09083865295497663,
"ece15_soft_target_top_label": 0.09551498865798673,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 950,
"nll": 1.0045923357674995,
"brier_sum_over_options": 0.44522656892983503,
"source_balanced_nll": 0.7313919900168762,
"source_balanced_brier": 0.26329914692243295,
"hard_label_n": 809,
"hard_accuracy": 0.6007416563658838,
"ece15_hard": 0.041113167177559154,
"ece15_soft_target_top_label": 0.039897717676097656,
"argmax_changes_vs_T1": 0
}
},
"7": {
"baseline_T1": {
"n": 251,
"nll": 1.3152865948450587,
"brier_sum_over_options": 0.5748395813866473,
"source_balanced_nll": 1.1292949784944664,
"source_balanced_brier": 0.23963345216270032,
"hard_label_n": 210,
"hard_accuracy": 0.5190476190476191,
"ece15_hard": 0.17706544190724635,
"ece15_soft_target_top_label": 0.1548424924379715,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 251,
"nll": 1.2478735234414642,
"brier_sum_over_options": 0.5498257461063198,
"source_balanced_nll": 1.1229184732025017,
"source_balanced_brier": 0.23985396884350701,
"hard_label_n": 210,
"hard_accuracy": 0.5190476190476191,
"ece15_hard": 0.15229650266605774,
"ece15_soft_target_top_label": 0.12258486410872624,
"argmax_changes_vs_T1": 0
}
},
"10": {
"baseline_T1": {
"n": 177,
"nll": 0.3206380389579705,
"brier_sum_over_options": 0.14555325025959065,
"source_balanced_nll": 0.8944668652808186,
"source_balanced_brier": 0.41634359681708066,
"hard_label_n": 176,
"hard_accuracy": 0.9147727272727273,
"ece15_hard": 0.050593007747918244,
"ece15_soft_target_top_label": 0.04944359431042387,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 177,
"nll": 0.33478579212073806,
"brier_sum_over_options": 0.14833699565831765,
"source_balanced_nll": 0.8508665765404974,
"source_balanced_brier": 0.40010888757583646,
"hard_label_n": 176,
"hard_accuracy": 0.9147727272727273,
"ece15_hard": 0.06139121810069679,
"ece15_soft_target_top_label": 0.060381498793323045,
"argmax_changes_vs_T1": 0
}
},
"9": {
"baseline_T1": {
"n": 9,
"nll": 0.4476933610233881,
"brier_sum_over_options": 0.21395305071412862,
"source_balanced_nll": 0.4029329139445354,
"source_balanced_brier": 0.19255774859625302,
"hard_label_n": 9,
"hard_accuracy": 0.8888888888888888,
"ece15_hard": 0.10807980843377268,
"ece15_soft_target_top_label": 0.10807980843377268,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 9,
"nll": 0.38291973817084257,
"brier_sum_over_options": 0.2054226490983549,
"source_balanced_nll": 0.34470287752893114,
"source_balanced_brier": 0.18488054585265584,
"hard_label_n": 9,
"hard_accuracy": 0.8888888888888888,
"ece15_hard": 0.11729725515014063,
"ece15_soft_target_top_label": 0.11729725515014063,
"argmax_changes_vs_T1": 0
}
},
"17": {
"baseline_T1": {
"n": 205,
"nll": 1.6132380005191462,
"brier_sum_over_options": 0.5970722049808997,
"source_balanced_nll": 0.9233621719373097,
"source_balanced_brier": 0.349952645388343,
"hard_label_n": 205,
"hard_accuracy": 0.5365853658536586,
"ece15_hard": 0.1328836293853757,
"ece15_soft_target_top_label": 0.1328836293853757,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 205,
"nll": 1.5793890122940422,
"brier_sum_over_options": 0.597448295599059,
"source_balanced_nll": 0.9348759607764361,
"source_balanced_brier": 0.36374541657296194,
"hard_label_n": 205,
"hard_accuracy": 0.5365853658536586,
"ece15_hard": 0.10492652707851656,
"ece15_soft_target_top_label": 0.10492652707851656,
"argmax_changes_vs_T1": 0
}
},
"11": {
"baseline_T1": {
"n": 39,
"nll": 1.7329636387541798,
"brier_sum_over_options": 0.667152374977268,
"source_balanced_nll": 0.8892872556929191,
"source_balanced_brier": 0.34235450824438984,
"hard_label_n": 38,
"hard_accuracy": 0.5263157894736842,
"ece15_hard": 0.2929348492210114,
"ece15_soft_target_top_label": 0.2901902065207956,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 39,
"nll": 1.5641598543735449,
"brier_sum_over_options": 0.6349368441311507,
"source_balanced_nll": 0.8027125855206154,
"source_balanced_brier": 0.3258228615687921,
"hard_label_n": 38,
"hard_accuracy": 0.5263157894736842,
"ece15_hard": 0.2295083563890572,
"ece15_soft_target_top_label": 0.22773843639126312,
"argmax_changes_vs_T1": 0
}
},
"16": {
"baseline_T1": {
"n": 66,
"nll": 1.385363536431591,
"brier_sum_over_options": 0.6357574280908284,
"source_balanced_nll": 1.3853635364315908,
"source_balanced_brier": 0.6357574280908282,
"hard_label_n": 66,
"hard_accuracy": 0.4696969696969697,
"ece15_hard": 0.18863946073552296,
"ece15_soft_target_top_label": 0.18863946073552296,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 66,
"nll": 1.3778733417442919,
"brier_sum_over_options": 0.6324273088852371,
"source_balanced_nll": 1.3778733417442919,
"source_balanced_brier": 0.6324273088852372,
"hard_label_n": 66,
"hard_accuracy": 0.4696969696969697,
"ece15_hard": 0.16052618463860757,
"ece15_soft_target_top_label": 0.16052618463860757,
"argmax_changes_vs_T1": 0
}
},
"24": {
"baseline_T1": {
"n": 199,
"nll": 2.52009298708954,
"brier_sum_over_options": 0.8993141646552938,
"source_balanced_nll": 2.52009298708954,
"source_balanced_brier": 0.8993141646552935,
"hard_label_n": 199,
"hard_accuracy": 0.2663316582914573,
"ece15_hard": 0.20258006075459445,
"ece15_soft_target_top_label": 0.20258006075459445,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 199,
"nll": 2.39154090351234,
"brier_sum_over_options": 0.8603586852754271,
"source_balanced_nll": 2.3915409035123396,
"source_balanced_brier": 0.8603586852754269,
"hard_label_n": 199,
"hard_accuracy": 0.2663316582914573,
"ece15_hard": 0.12110174706098106,
"ece15_soft_target_top_label": 0.12110174706098106,
"argmax_changes_vs_T1": 0
}
},
"19": {
"baseline_T1": {
"n": 250,
"nll": 2.000001528751946,
"brier_sum_over_options": 0.764220994427989,
"source_balanced_nll": 1.391390760354478,
"source_balanced_brier": 0.5522001864064683,
"hard_label_n": 250,
"hard_accuracy": 0.448,
"ece15_hard": 0.26716842748351055,
"ece15_soft_target_top_label": 0.26716842748351055,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 250,
"nll": 1.7969496853885047,
"brier_sum_over_options": 0.7189072714462958,
"source_balanced_nll": 1.224825530838087,
"source_balanced_brier": 0.5156647723552193,
"hard_label_n": 250,
"hard_accuracy": 0.448,
"ece15_hard": 0.18910919644120522,
"ece15_soft_target_top_label": 0.18910919644120522,
"argmax_changes_vs_T1": 0
}
},
"6": {
"baseline_T1": {
"n": 42,
"nll": 0.7952857752023642,
"brier_sum_over_options": 0.25679041419443505,
"source_balanced_nll": 1.3649593159260112,
"source_balanced_brier": 0.49742160192872276,
"hard_label_n": 28,
"hard_accuracy": 0.7857142857142857,
"ece15_hard": 0.12202502285315012,
"ece15_soft_target_top_label": 0.10849406411904323,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 42,
"nll": 0.7946724429502305,
"brier_sum_over_options": 0.2508575171249201,
"source_balanced_nll": 1.303336818211357,
"source_balanced_brier": 0.4687985480720983,
"hard_label_n": 28,
"hard_accuracy": 0.7857142857142857,
"ece15_hard": 0.0990461413124916,
"ece15_soft_target_top_label": 0.08061339596167542,
"argmax_changes_vs_T1": 0
}
},
"46": {
"baseline_T1": {
"n": 28,
"nll": 0.26639228936146403,
"brier_sum_over_options": 0.12504826564601207,
"source_balanced_nll": 0.2663922893614641,
"source_balanced_brier": 0.1250482656460121,
"hard_label_n": 28,
"hard_accuracy": 0.9285714285714286,
"ece15_hard": 0.1005493388651708,
"ece15_soft_target_top_label": 0.1005493388651708,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 28,
"nll": 0.2733773807804469,
"brier_sum_over_options": 0.13021424100801882,
"source_balanced_nll": 0.27337738078044704,
"source_balanced_brier": 0.13021424100801882,
"hard_label_n": 28,
"hard_accuracy": 0.9285714285714286,
"ece15_hard": 0.08000988992707549,
"ece15_soft_target_top_label": 0.08000988992707549,
"argmax_changes_vs_T1": 0
}
},
"151": {
"baseline_T1": {
"n": 27,
"nll": 0.01318742037085779,
"brier_sum_over_options": 0.003061869506362786,
"source_balanced_nll": 0.013187420370857789,
"source_balanced_brier": 0.003061869506362787,
"hard_label_n": 27,
"hard_accuracy": 1.0,
"ece15_hard": 0.012221792881074414,
"ece15_soft_target_top_label": 0.012221792881074414,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 27,
"nll": 0.0324108944722723,
"brier_sum_over_options": 0.005900679165396043,
"source_balanced_nll": 0.0324108944722723,
"source_balanced_brier": 0.005900679165396044,
"hard_label_n": 27,
"hard_accuracy": 1.0,
"ece15_hard": 0.030124594022812777,
"ece15_soft_target_top_label": 0.030124594022812777,
"argmax_changes_vs_T1": 0
}
},
"174": {
"baseline_T1": {
"n": 199,
"nll": 6.158517993150494,
"brier_sum_over_options": 1.1277309962228477,
"source_balanced_nll": 6.158517993150494,
"source_balanced_brier": 1.1277309962228474,
"hard_label_n": 199,
"hard_accuracy": 0.04522613065326633,
"ece15_hard": 0.2688069486260517,
"ece15_soft_target_top_label": 0.2688069486260517,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 199,
"nll": 5.580793881646713,
"brier_sum_over_options": 1.0614228245534538,
"source_balanced_nll": 5.580793881646713,
"source_balanced_brier": 1.0614228245534538,
"hard_label_n": 199,
"hard_accuracy": 0.04522613065326633,
"ece15_hard": 0.16975478917023556,
"ece15_soft_target_top_label": 0.16975478917023556,
"argmax_changes_vs_T1": 0
}
},
"8": {
"baseline_T1": {
"n": 128,
"nll": 0.6207579846866409,
"brier_sum_over_options": 0.28413876758825996,
"source_balanced_nll": 1.0954452550702964,
"source_balanced_brier": 0.2678070927481252,
"hard_label_n": 127,
"hard_accuracy": 0.7795275590551181,
"ece15_hard": 0.08215042145998787,
"ece15_soft_target_top_label": 0.08231420366151865,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 128,
"nll": 0.5890299247160777,
"brier_sum_over_options": 0.2733743371441415,
"source_balanced_nll": 1.0525137725327651,
"source_balanced_brier": 0.2585817551466508,
"hard_label_n": 127,
"hard_accuracy": 0.7795275590551181,
"ece15_hard": 0.06598347246885168,
"ece15_soft_target_top_label": 0.06621948240680248,
"argmax_changes_vs_T1": 0
}
},
"26": {
"baseline_T1": {
"n": 10,
"nll": 0.5025231596563897,
"brier_sum_over_options": 0.24677783766682865,
"source_balanced_nll": 0.36044603902995376,
"source_balanced_brier": 0.1528364128764804,
"hard_label_n": 10,
"hard_accuracy": 0.8,
"ece15_hard": 0.11644059096268489,
"ece15_soft_target_top_label": 0.11644059096268489,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10,
"nll": 0.609105311087961,
"brier_sum_over_options": 0.2651768630573715,
"source_balanced_nll": 0.49112756697666576,
"source_balanced_brier": 0.19258974008385663,
"hard_label_n": 10,
"hard_accuracy": 0.8,
"ece15_hard": 0.19986552305602281,
"ece15_soft_target_top_label": 0.19986552305602281,
"argmax_changes_vs_T1": 0
}
},
"60": {
"baseline_T1": {
"n": 30,
"nll": 0.5548174319487342,
"brier_sum_over_options": 0.2479334677808331,
"source_balanced_nll": 0.5548174319487341,
"source_balanced_brier": 0.2479334677808331,
"hard_label_n": 30,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.10795255537426568,
"ece15_soft_target_top_label": 0.10795255537426568,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 30,
"nll": 0.537288122706222,
"brier_sum_over_options": 0.24392409515753188,
"source_balanced_nll": 0.537288122706222,
"source_balanced_brier": 0.2439240951575319,
"hard_label_n": 30,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.13914162287049503,
"ece15_soft_target_top_label": 0.13914162287049503,
"argmax_changes_vs_T1": 0
}
},
"77": {
"baseline_T1": {
"n": 23,
"nll": 0.24889848198509765,
"brier_sum_over_options": 0.10007059041852256,
"source_balanced_nll": 0.24889848198509765,
"source_balanced_brier": 0.10007059041852255,
"hard_label_n": 23,
"hard_accuracy": 0.9565217391304348,
"ece15_hard": 0.10821931280222388,
"ece15_soft_target_top_label": 0.10821931280222388,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 23,
"nll": 0.29383817968107395,
"brier_sum_over_options": 0.10984460726191307,
"source_balanced_nll": 0.293838179681074,
"source_balanced_brier": 0.10984460726191303,
"hard_label_n": 23,
"hard_accuracy": 0.9565217391304348,
"ece15_hard": 0.1510796294711406,
"ece15_soft_target_top_label": 0.1510796294711406,
"argmax_changes_vs_T1": 0
}
},
"14": {
"baseline_T1": {
"n": 41,
"nll": 1.0006219052775647,
"brier_sum_over_options": 0.4079138427183757,
"source_balanced_nll": 0.5926213025283489,
"source_balanced_brier": 0.25326978504067077,
"hard_label_n": 41,
"hard_accuracy": 0.7317073170731707,
"ece15_hard": 0.14411558905984945,
"ece15_soft_target_top_label": 0.14411558905984945,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 41,
"nll": 0.9985517823084674,
"brier_sum_over_options": 0.40942780379684524,
"source_balanced_nll": 0.616780589952582,
"source_balanced_brier": 0.254690053332879,
"hard_label_n": 41,
"hard_accuracy": 0.7317073170731707,
"ece15_hard": 0.1639515813029694,
"ece15_soft_target_top_label": 0.1639515813029694,
"argmax_changes_vs_T1": 0
}
},
"25": {
"baseline_T1": {
"n": 1,
"nll": 3.0424463714125904e-05,
"brier_sum_over_options": 1.1082748128296532e-09,
"source_balanced_nll": 3.0424463714125904e-05,
"source_balanced_brier": 1.1082748128296532e-09,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 3.042400089481312e-05,
"ece15_soft_target_top_label": 3.042400089481312e-05,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1,
"nll": 0.00044176654284283076,
"brier_sum_over_options": 2.1910825266858835e-07,
"source_balanced_nll": 0.00044176654284283076,
"source_balanced_brier": 2.1910825266858835e-07,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.000441668978371057,
"ece15_soft_target_top_label": 0.000441668978371057,
"argmax_changes_vs_T1": 0
}
},
"55": {
"baseline_T1": {
"n": 26,
"nll": 0.5464120909376707,
"brier_sum_over_options": 0.3016258560842057,
"source_balanced_nll": 0.5464120909376707,
"source_balanced_brier": 0.30162585608420583,
"hard_label_n": 26,
"hard_accuracy": 0.8076923076923077,
"ece15_hard": 0.19163817005247774,
"ece15_soft_target_top_label": 0.19163817005247774,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 26,
"nll": 0.5395469799824957,
"brier_sum_over_options": 0.2967736418453159,
"source_balanced_nll": 0.5395469799824957,
"source_balanced_brier": 0.29677364184531596,
"hard_label_n": 26,
"hard_accuracy": 0.8076923076923077,
"ece15_hard": 0.19311131159010958,
"ece15_soft_target_top_label": 0.19311131159010958,
"argmax_changes_vs_T1": 0
}
},
"100": {
"baseline_T1": {
"n": 10,
"nll": 1.2505443561707321,
"brier_sum_over_options": 0.47732782682331704,
"source_balanced_nll": 1.2505443561707321,
"source_balanced_brier": 0.47732782682331715,
"hard_label_n": 10,
"hard_accuracy": 0.7,
"ece15_hard": 0.18308357087203325,
"ece15_soft_target_top_label": 0.18308357087203325,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10,
"nll": 1.1278725658570645,
"brier_sum_over_options": 0.45785612712555296,
"source_balanced_nll": 1.1278725658570645,
"source_balanced_brier": 0.4578561271255529,
"hard_label_n": 10,
"hard_accuracy": 0.7,
"ece15_hard": 0.16359466837116274,
"ece15_soft_target_top_label": 0.16359466837116274,
"argmax_changes_vs_T1": 0
}
},
"12": {
"baseline_T1": {
"n": 4,
"nll": 0.22633905322119835,
"brier_sum_over_options": 0.16548968556165386,
"source_balanced_nll": 0.22633905322119835,
"source_balanced_brier": 0.16548968556165386,
"hard_label_n": 4,
"hard_accuracy": 0.75,
"ece15_hard": 0.15544531288246594,
"ece15_soft_target_top_label": 0.15544531288246594,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 4,
"nll": 0.24186842126102173,
"brier_sum_over_options": 0.1589598280414067,
"source_balanced_nll": 0.24186842126102173,
"source_balanced_brier": 0.1589598280414067,
"hard_label_n": 4,
"hard_accuracy": 0.75,
"ece15_hard": 0.17253393357035618,
"ece15_soft_target_top_label": 0.17253393357035618,
"argmax_changes_vs_T1": 0
}
},
"96": {
"baseline_T1": {
"n": 7,
"nll": 0.623225138950921,
"brier_sum_over_options": 0.16453340208446093,
"source_balanced_nll": 0.6232251389509212,
"source_balanced_brier": 0.16453340208446096,
"hard_label_n": 6,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.055500330162458915,
"ece15_soft_target_top_label": 0.11378535016952016,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 7,
"nll": 0.619194953153693,
"brier_sum_over_options": 0.16303123677100362,
"source_balanced_nll": 0.619194953153693,
"source_balanced_brier": 0.16303123677100367,
"hard_label_n": 6,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.06957412809356438,
"ece15_soft_target_top_label": 0.11649380646134164,
"argmax_changes_vs_T1": 0
}
},
"13": {
"baseline_T1": {
"n": 3,
"nll": 0.00010359158169846062,
"brier_sum_over_options": 2.3205667611402075e-08,
"source_balanced_nll": 0.00010359158169846062,
"source_balanced_brier": 2.3205667611402072e-08,
"hard_label_n": 3,
"hard_accuracy": 1.0,
"ece15_hard": 0.00010358306109203852,
"ece15_soft_target_top_label": 0.00010358306109203852,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3,
"nll": 0.0009203684616632247,
"brier_sum_over_options": 1.555861995072091e-06,
"source_balanced_nll": 0.0009203684616632247,
"source_balanced_brier": 1.555861995072091e-06,
"hard_label_n": 3,
"hard_accuracy": 1.0,
"ece15_hard": 0.0009197509787280955,
"ece15_soft_target_top_label": 0.0009197509787280955,
"argmax_changes_vs_T1": 0
}
},
"18": {
"baseline_T1": {
"n": 2,
"nll": 0.0002237176497633426,
"brier_sum_over_options": 1.0071760210689769e-07,
"source_balanced_nll": 0.0002237176497633426,
"source_balanced_brier": 1.0071760210689769e-07,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.00022368672505856946,
"ece15_soft_target_top_label": 0.00022368672505856946,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2,
"nll": 0.0017783344770914305,
"brier_sum_over_options": 4.616268324223769e-06,
"source_balanced_nll": 0.0017783344770914305,
"source_balanced_brier": 4.616268324223769e-06,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.0017766436394408558,
"ece15_soft_target_top_label": 0.0017766436394408558,
"argmax_changes_vs_T1": 0
}
},
"20": {
"baseline_T1": {
"n": 1,
"nll": 7.0881026342218e-05,
"brier_sum_over_options": 5.737917643872293e-09,
"source_balanced_nll": 7.0881026342218e-05,
"source_balanced_brier": 5.737917643872293e-09,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 7.087851434162129e-05,
"ece15_soft_target_top_label": 7.087851434162129e-05,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1,
"nll": 0.0008676947730364941,
"brier_sum_over_options": 8.306677880213897e-07,
"source_balanced_nll": 0.0008676947730364941,
"source_balanced_brier": 8.306677880213897e-07,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.0008673184347837148,
"ece15_soft_target_top_label": 0.0008673184347837148,
"argmax_changes_vs_T1": 0
}
},
"15": {
"baseline_T1": {
"n": 2,
"nll": 0.21302172843123862,
"brier_sum_over_options": 0.08973998493512275,
"source_balanced_nll": 0.21302172843123862,
"source_balanced_brier": 0.08973998493512275,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.17346277685360906,
"ece15_soft_target_top_label": 0.17346277685360906,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2,
"nll": 0.28318648946835734,
"brier_sum_over_options": 0.1311346167392156,
"source_balanced_nll": 0.28318648946835734,
"source_balanced_brier": 0.1311346167392156,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.21632645505646092,
"ece15_soft_target_top_label": 0.21632645505646092,
"argmax_changes_vs_T1": 0
}
}
},
"length_bucket": {
"256": {
"baseline_T1": {
"n": 2795,
"nll": 0.47037257600293053,
"brier_sum_over_options": 0.22541632435245837,
"source_balanced_nll": 0.43267872716046035,
"source_balanced_brier": 0.19970012750356125,
"hard_label_n": 2438,
"hard_accuracy": 0.8351107465135357,
"ece15_hard": 0.023655126639394466,
"ece15_soft_target_top_label": 0.031499768143108876,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2795,
"nll": 0.4643596142831439,
"brier_sum_over_options": 0.22369417931993363,
"source_balanced_nll": 0.43952051425113703,
"source_balanced_brier": 0.2019426030434196,
"hard_label_n": 2438,
"hard_accuracy": 0.8351107465135357,
"ece15_hard": 0.018615318161063654,
"ece15_soft_target_top_label": 0.011551091968849354,
"argmax_changes_vs_T1": 0
}
},
"512": {
"baseline_T1": {
"n": 3633,
"nll": 0.7961120431431014,
"brier_sum_over_options": 0.3866354321805597,
"source_balanced_nll": 0.6723262697528442,
"source_balanced_brier": 0.3246270647083185,
"hard_label_n": 3381,
"hard_accuracy": 0.6876663708961845,
"ece15_hard": 0.07185174553774616,
"ece15_soft_target_top_label": 0.07249608384810159,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3633,
"nll": 0.7767235169627644,
"brier_sum_over_options": 0.3775918950131493,
"source_balanced_nll": 0.6589270592749751,
"source_balanced_brier": 0.3176360770357968,
"hard_label_n": 3381,
"hard_accuracy": 0.6876663708961845,
"ece15_hard": 0.03326765310622277,
"ece15_soft_target_top_label": 0.03159729681646724,
"argmax_changes_vs_T1": 0
}
},
"1024": {
"baseline_T1": {
"n": 1854,
"nll": 1.0735632875231345,
"brier_sum_over_options": 0.4711795599195009,
"source_balanced_nll": 0.8809112187585068,
"source_balanced_brier": 0.3926998694092402,
"hard_label_n": 1767,
"hard_accuracy": 0.6327108092812677,
"ece15_hard": 0.12052197658515143,
"ece15_soft_target_top_label": 0.11736387380649344,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1854,
"nll": 1.0154387575225823,
"brier_sum_over_options": 0.4524833167245523,
"source_balanced_nll": 0.8361058718242633,
"source_balanced_brier": 0.37985981229200694,
"hard_label_n": 1767,
"hard_accuracy": 0.6327108092812677,
"ece15_hard": 0.06732815180475929,
"ece15_soft_target_top_label": 0.06464806043367109,
"argmax_changes_vs_T1": 0
}
},
"2048": {
"baseline_T1": {
"n": 663,
"nll": 0.6541135388322734,
"brier_sum_over_options": 0.34538001341076485,
"source_balanced_nll": 0.7069701243738188,
"source_balanced_brier": 0.3692422084606182,
"hard_label_n": 628,
"hard_accuracy": 0.7404458598726115,
"ece15_hard": 0.08093880400978243,
"ece15_soft_target_top_label": 0.08004513152560036,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 663,
"nll": 0.6278369365495612,
"brier_sum_over_options": 0.334457831205003,
"source_balanced_nll": 0.6834964588529152,
"source_balanced_brier": 0.3622779376606939,
"hard_label_n": 628,
"hard_accuracy": 0.7404458598726115,
"ece15_hard": 0.048098854116713524,
"ece15_soft_target_top_label": 0.04538127122956523,
"argmax_changes_vs_T1": 0
}
},
"4096": {
"baseline_T1": {
"n": 486,
"nll": 2.9953129558521128,
"brier_sum_over_options": 0.728661071956894,
"source_balanced_nll": 0.9605047895483201,
"source_balanced_brier": 0.35616517994562186,
"hard_label_n": 483,
"hard_accuracy": 0.41200828157349895,
"ece15_hard": 0.19798039561026823,
"ece15_soft_target_top_label": 0.1989211076770336,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 486,
"nll": 2.7227974288105425,
"brier_sum_over_options": 0.6854139842379053,
"source_balanced_nll": 0.8889657813488471,
"source_balanced_brier": 0.3341220032284308,
"hard_label_n": 483,
"hard_accuracy": 0.41200828157349895,
"ece15_hard": 0.1369973550975109,
"ece15_soft_target_top_label": 0.13779306928471627,
"argmax_changes_vs_T1": 0
}
},
"8192": {
"baseline_T1": {
"n": 220,
"nll": 0.8261859145282336,
"brier_sum_over_options": 0.5000690949390265,
"source_balanced_nll": 0.7186466661923275,
"source_balanced_brier": 0.42251676814245676,
"hard_label_n": 220,
"hard_accuracy": 0.6181818181818182,
"ece15_hard": 0.17249925138328506,
"ece15_soft_target_top_label": 0.17249925138328506,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 220,
"nll": 0.7679840071494353,
"brier_sum_over_options": 0.4714047900952329,
"source_balanced_nll": 0.6695459040455484,
"source_balanced_brier": 0.4027365452863051,
"hard_label_n": 220,
"hard_accuracy": 0.6181818181818182,
"ece15_hard": 0.13520540837183176,
"ece15_soft_target_top_label": 0.13520540837183176,
"argmax_changes_vs_T1": 0
}
},
"16384": {
"baseline_T1": {
"n": 349,
"nll": 0.7477118306090438,
"brier_sum_over_options": 0.42469192133840344,
"source_balanced_nll": 0.8343087539394536,
"source_balanced_brier": 0.46741366218669134,
"hard_label_n": 349,
"hard_accuracy": 0.6991404011461319,
"ece15_hard": 0.13091306333007388,
"ece15_soft_target_top_label": 0.13091306333007388,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 349,
"nll": 0.6971136317498758,
"brier_sum_over_options": 0.4067407206604246,
"source_balanced_nll": 0.7792284684133695,
"source_balanced_brier": 0.4517577915031896,
"hard_label_n": 349,
"hard_accuracy": 0.6991404011461319,
"ece15_hard": 0.08009399151033489,
"ece15_soft_target_top_label": 0.08009399151033489,
"argmax_changes_vs_T1": 0
}
}
}
},
"accepted": true
},
"flash": {
"fit_and_selection": {
"temperature": 1.2206820938802239,
"selected_fit_loss": "nll",
"candidates": [
{
"temperature": 1.0,
"fit_loss": "identity",
"at_search_boundary": false,
"selection_nll": 0.737196373394814,
"selection_brier": 0.33375038462744916,
"selection_score": 0.3410571662820997,
"eligible": true
},
{
"temperature": 1.2206820938802239,
"fit_loss": "nll",
"fit_objective": 0.7515176189861205,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.730124808135844,
"selection_brier": 0.3315942599645005,
"selection_score": 0.3392871454965463,
"eligible": true
},
{
"temperature": 1.227575800140271,
"fit_loss": "brier",
"fit_objective": 0.3389004130008895,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.7302632405973527,
"selection_brier": 0.33162998870737326,
"selection_score": 0.3393661199072915,
"eligible": true
}
],
"policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization."
},
"audit": {
"pooled": {
"passed": true,
"baseline_T1": {
"n": 10000,
"nll": 0.8570002323848743,
"brier_sum_over_options": 0.37919432626901206,
"source_balanced_nll": 0.7118519293468761,
"source_balanced_brier": 0.3106978930512085,
"hard_label_n": 9266,
"hard_accuracy": 0.6965249298510684,
"ece15_hard": 0.062451612223343875,
"ece15_soft_target_top_label": 0.06243402250680544,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10000,
"nll": 0.8321960092542425,
"brier_sum_over_options": 0.37160163535588864,
"source_balanced_nll": 0.7027353839673045,
"source_balanced_brier": 0.30851932327767795,
"hard_label_n": 9266,
"hard_accuracy": 0.6965249298510684,
"ece15_hard": 0.030980441201261717,
"ece15_soft_target_top_label": 0.02841899539582912,
"argmax_changes_vs_T1": 0
},
"source_families": 84,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.026075691245677348,
0.0036036286640317217
],
"brier_delta_95": [
-0.006356109913979643,
0.0013150842620000239
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_group": {
"passed": true,
"baseline_T1": {
"n": 8000,
"nll": 0.9032170652755189,
"brier_sum_over_options": 0.39186413206604753,
"source_balanced_nll": 0.7427867356466757,
"source_balanced_brier": 0.32151723440277274,
"hard_label_n": 7499,
"hard_accuracy": 0.6884917989065209,
"ece15_hard": 0.06392807226641213,
"ece15_soft_target_top_label": 0.061682067108722864,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 8000,
"nll": 0.8762545067845626,
"brier_sum_over_options": 0.38424953593396205,
"source_balanced_nll": 0.7327617378039142,
"source_balanced_brier": 0.3194039506091354,
"hard_label_n": 7499,
"hard_accuracy": 0.6884917989065209,
"ece15_hard": 0.028459940989966365,
"ece15_soft_target_top_label": 0.02494309557290096,
"argmax_changes_vs_T1": 0
},
"source_families": 71,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.02793543836392883,
0.004751466455646055
],
"brier_delta_95": [
-0.0066107228100673,
0.0020246791014271407
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_family": {
"passed": true,
"baseline_T1": {
"n": 2000,
"nll": 0.672132900822296,
"brier_sum_over_options": 0.3285151030808702,
"source_balanced_nll": 0.5429002949402779,
"source_balanced_brier": 0.25160764413112674,
"hard_label_n": 1767,
"hard_accuracy": 0.7306168647425014,
"ece15_hard": 0.05762287331091378,
"ece15_soft_target_top_label": 0.06613371317283424,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2000,
"nll": 0.6559620191329626,
"brier_sum_over_options": 0.32101003304359504,
"source_balanced_nll": 0.538745297628898,
"source_balanced_brier": 0.2490725124674108,
"hard_label_n": 1767,
"hard_accuracy": 0.7306168647425014,
"ece15_hard": 0.053041698299378603,
"ece15_soft_target_top_label": 0.0460495879467492,
"argmax_changes_vs_T1": 0
},
"source_families": 13,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.0224885374355191,
0.013856519161634075
],
"brier_delta_95": [
-0.009731275705932721,
0.00442424973831137
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
}
},
"diagnostics": {
"type": {
"choice": {
"baseline_T1": {
"n": 9585,
"nll": 0.8772217338998575,
"brier_sum_over_options": 0.3917876319294491,
"source_balanced_nll": 0.723263364856754,
"source_balanced_brier": 0.31935528714685957,
"hard_label_n": 9124,
"hard_accuracy": 0.6924594476106971,
"ece15_hard": 0.06400940996750015,
"ece15_soft_target_top_label": 0.06551855092083372,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 9585,
"nll": 0.8509885943159481,
"brier_sum_over_options": 0.38371255767134194,
"source_balanced_nll": 0.7133445651926446,
"source_balanced_brier": 0.31689551555221435,
"hard_label_n": 9124,
"hard_accuracy": 0.6924594476106971,
"ece15_hard": 0.03143555594499423,
"ece15_soft_target_top_label": 0.030205136993845923,
"argmax_changes_vs_T1": 0
}
},
"noul": {
"baseline_T1": {
"n": 415,
"nll": 0.38995663715327494,
"brier_sum_over_options": 0.08833448348518269,
"source_balanced_nll": 0.37363381275371293,
"source_balanced_brier": 0.0829826996918807,
"hard_label_n": 142,
"hard_accuracy": 0.9577464788732394,
"ece15_hard": 0.06288195970798233,
"ece15_soft_target_top_label": 0.030598094376512987,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 415,
"nll": 0.39815521933509257,
"brier_sum_over_options": 0.09188310428692517,
"source_balanced_nll": 0.383695176077421,
"source_balanced_brier": 0.08754055668370506,
"hard_label_n": 142,
"hard_accuracy": 0.9577464788732394,
"ece15_hard": 0.07643712522321415,
"ece15_soft_target_top_label": 0.03813514826276408,
"argmax_changes_vs_T1": 0
}
}
},
"option_count": {
"2": {
"baseline_T1": {
"n": 3739,
"nll": 0.34456045849965816,
"brier_sum_over_options": 0.19168513862671624,
"source_balanced_nll": 0.364383448081177,
"source_balanced_brier": 0.1870597652761825,
"hard_label_n": 3402,
"hard_accuracy": 0.8641975308641975,
"ece15_hard": 0.014521320701320558,
"ece15_soft_target_top_label": 0.014485028706927148,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3739,
"nll": 0.347994903587601,
"brier_sum_over_options": 0.19347148230754105,
"source_balanced_nll": 0.3708302552762176,
"source_balanced_brier": 0.19057903058028713,
"hard_label_n": 3402,
"hard_accuracy": 0.8641975308641975,
"ece15_hard": 0.03370223301330605,
"ece15_soft_target_top_label": 0.03081853496373301,
"argmax_changes_vs_T1": 0
}
},
"4": {
"baseline_T1": {
"n": 1907,
"nll": 0.8780784289258254,
"brier_sum_over_options": 0.42942297750497044,
"source_balanced_nll": 0.717786973346652,
"source_balanced_brier": 0.3305826004928006,
"hard_label_n": 1712,
"hard_accuracy": 0.634929906542056,
"ece15_hard": 0.07560011676712683,
"ece15_soft_target_top_label": 0.08055535862735413,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1907,
"nll": 0.8607848007212155,
"brier_sum_over_options": 0.42013227775435097,
"source_balanced_nll": 0.7131964519501658,
"source_balanced_brier": 0.3249144022396803,
"hard_label_n": 1712,
"hard_accuracy": 0.634929906542056,
"ece15_hard": 0.05901139914756741,
"ece15_soft_target_top_label": 0.0535352508262639,
"argmax_changes_vs_T1": 0
}
},
"3": {
"baseline_T1": {
"n": 1624,
"nll": 0.785835337758448,
"brier_sum_over_options": 0.47004015333173127,
"source_balanced_nll": 0.5631316810018027,
"source_balanced_brier": 0.33191132836767095,
"hard_label_n": 1622,
"hard_accuracy": 0.6368680641183724,
"ece15_hard": 0.12745012975209907,
"ece15_soft_target_top_label": 0.12733742597917397,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1624,
"nll": 0.7579783200548996,
"brier_sum_over_options": 0.45355116090254954,
"source_balanced_nll": 0.5641112187421008,
"source_balanced_brier": 0.328408604630681,
"hard_label_n": 1622,
"hard_accuracy": 0.6368680641183724,
"ece15_hard": 0.09248554910019118,
"ece15_soft_target_top_label": 0.09234089293515488,
"argmax_changes_vs_T1": 0
}
},
"5": {
"baseline_T1": {
"n": 950,
"nll": 1.0198279459718724,
"brier_sum_over_options": 0.4537877484898534,
"source_balanced_nll": 0.7411751486322502,
"source_balanced_brier": 0.2627755636080211,
"hard_label_n": 809,
"hard_accuracy": 0.6093943139678616,
"ece15_hard": 0.0739815683708236,
"ece15_soft_target_top_label": 0.0752115324492788,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 950,
"nll": 1.0058757211412626,
"brier_sum_over_options": 0.4454651778503967,
"source_balanced_nll": 0.7446224058860016,
"source_balanced_brier": 0.2646854264518146,
"hard_label_n": 809,
"hard_accuracy": 0.6093943139678616,
"ece15_hard": 0.043150569558027416,
"ece15_soft_target_top_label": 0.04000418776261011,
"argmax_changes_vs_T1": 0
}
},
"7": {
"baseline_T1": {
"n": 251,
"nll": 1.2660457526018802,
"brier_sum_over_options": 0.5568022902060177,
"source_balanced_nll": 1.114109267653641,
"source_balanced_brier": 0.2303607680244388,
"hard_label_n": 210,
"hard_accuracy": 0.5476190476190477,
"ece15_hard": 0.15114649965418456,
"ece15_soft_target_top_label": 0.11907381211915682,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 251,
"nll": 1.2447503100682535,
"brier_sum_over_options": 0.5440920981041713,
"source_balanced_nll": 1.1284429223392554,
"source_balanced_brier": 0.23633839305922688,
"hard_label_n": 210,
"hard_accuracy": 0.5476190476190477,
"ece15_hard": 0.13768108972995582,
"ece15_soft_target_top_label": 0.11032470220575231,
"argmax_changes_vs_T1": 0
}
},
"10": {
"baseline_T1": {
"n": 177,
"nll": 0.3876108425278409,
"brier_sum_over_options": 0.17220266211499055,
"source_balanced_nll": 1.0141946727194626,
"source_balanced_brier": 0.46444338337530816,
"hard_label_n": 176,
"hard_accuracy": 0.8806818181818182,
"ece15_hard": 0.06343742806851521,
"ece15_soft_target_top_label": 0.06477182598055643,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 177,
"nll": 0.4061828262405223,
"brier_sum_over_options": 0.17601555173169234,
"source_balanced_nll": 0.9728960016273719,
"source_balanced_brier": 0.444851552314224,
"hard_label_n": 176,
"hard_accuracy": 0.8806818181818182,
"ece15_hard": 0.0840859170202171,
"ece15_soft_target_top_label": 0.08512382515535123,
"argmax_changes_vs_T1": 0
}
},
"9": {
"baseline_T1": {
"n": 9,
"nll": 0.5181241857958109,
"brier_sum_over_options": 0.22723160082465763,
"source_balanced_nll": 0.46633998503336405,
"source_balanced_brier": 0.2045084586306961,
"hard_label_n": 9,
"hard_accuracy": 0.8888888888888888,
"ece15_hard": 0.12437858887005425,
"ece15_soft_target_top_label": 0.12437858887005425,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 9,
"nll": 0.45631116645501074,
"brier_sum_over_options": 0.22506445202827713,
"source_balanced_nll": 0.41083668871677403,
"source_balanced_brier": 0.20255847018619647,
"hard_label_n": 9,
"hard_accuracy": 0.8888888888888888,
"ece15_hard": 0.151315777849797,
"ece15_soft_target_top_label": 0.151315777849797,
"argmax_changes_vs_T1": 0
}
},
"17": {
"baseline_T1": {
"n": 205,
"nll": 1.6704944121216594,
"brier_sum_over_options": 0.6114686696715282,
"source_balanced_nll": 0.9534717316424565,
"source_balanced_brier": 0.3486444747645515,
"hard_label_n": 205,
"hard_accuracy": 0.5219512195121951,
"ece15_hard": 0.1436878908421718,
"ece15_soft_target_top_label": 0.1436878908421718,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 205,
"nll": 1.6679925852627786,
"brier_sum_over_options": 0.619127888892377,
"source_balanced_nll": 0.9860940081523526,
"source_balanced_brier": 0.36896947799357965,
"hard_label_n": 205,
"hard_accuracy": 0.5219512195121951,
"ece15_hard": 0.12894796792450236,
"ece15_soft_target_top_label": 0.12894796792450236,
"argmax_changes_vs_T1": 0
}
},
"11": {
"baseline_T1": {
"n": 39,
"nll": 1.7691923654930493,
"brier_sum_over_options": 0.6467617561982862,
"source_balanced_nll": 0.9078878985181886,
"source_balanced_brier": 0.3318909017035189,
"hard_label_n": 38,
"hard_accuracy": 0.5,
"ece15_hard": 0.179558657341575,
"ece15_soft_target_top_label": 0.1866719201381481,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 39,
"nll": 1.644164531310609,
"brier_sum_over_options": 0.6318981820120347,
"source_balanced_nll": 0.8438272165868839,
"source_balanced_brier": 0.32426357491109076,
"hard_label_n": 38,
"hard_accuracy": 0.5,
"ece15_hard": 0.2102966208414764,
"ece15_soft_target_top_label": 0.21525189865464361,
"argmax_changes_vs_T1": 0
}
},
"16": {
"baseline_T1": {
"n": 66,
"nll": 1.5103265510876325,
"brier_sum_over_options": 0.6909272488355744,
"source_balanced_nll": 1.5103265510876318,
"source_balanced_brier": 0.6909272488355747,
"hard_label_n": 66,
"hard_accuracy": 0.4393939393939394,
"ece15_hard": 0.16550696094702416,
"ece15_soft_target_top_label": 0.16550696094702416,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 66,
"nll": 1.4945455778603822,
"brier_sum_over_options": 0.675276809393677,
"source_balanced_nll": 1.4945455778603824,
"source_balanced_brier": 0.675276809393677,
"hard_label_n": 66,
"hard_accuracy": 0.4393939393939394,
"ece15_hard": 0.17113553935093878,
"ece15_soft_target_top_label": 0.17113553935093878,
"argmax_changes_vs_T1": 0
}
},
"24": {
"baseline_T1": {
"n": 199,
"nll": 2.4924340025418417,
"brier_sum_over_options": 0.8903662726669508,
"source_balanced_nll": 2.4924340025418408,
"source_balanced_brier": 0.8903662726669506,
"hard_label_n": 199,
"hard_accuracy": 0.23618090452261306,
"ece15_hard": 0.19438784148009122,
"ece15_soft_target_top_label": 0.19438784148009122,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 199,
"nll": 2.4120819565089704,
"brier_sum_over_options": 0.8612717774028041,
"source_balanced_nll": 2.4120819565089704,
"source_balanced_brier": 0.8612717774028039,
"hard_label_n": 199,
"hard_accuracy": 0.23618090452261306,
"ece15_hard": 0.11830923555486911,
"ece15_soft_target_top_label": 0.11830923555486911,
"argmax_changes_vs_T1": 0
}
},
"19": {
"baseline_T1": {
"n": 250,
"nll": 2.0952916773884236,
"brier_sum_over_options": 0.7711542523586539,
"source_balanced_nll": 1.4374930375559447,
"source_balanced_brier": 0.5555214202693165,
"hard_label_n": 250,
"hard_accuracy": 0.428,
"ece15_hard": 0.2610132899083695,
"ece15_soft_target_top_label": 0.2610132899083695,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 250,
"nll": 1.91575608646441,
"brier_sum_over_options": 0.7345501633191516,
"source_balanced_nll": 1.2983558513387234,
"source_balanced_brier": 0.5274825565155761,
"hard_label_n": 250,
"hard_accuracy": 0.428,
"ece15_hard": 0.1939003334998453,
"ece15_soft_target_top_label": 0.1939003334998453,
"argmax_changes_vs_T1": 0
}
},
"6": {
"baseline_T1": {
"n": 42,
"nll": 0.8288988397766355,
"brier_sum_over_options": 0.27777704347580634,
"source_balanced_nll": 1.4039889742757714,
"source_balanced_brier": 0.47600245332885854,
"hard_label_n": 28,
"hard_accuracy": 0.7857142857142857,
"ece15_hard": 0.19954755091637338,
"ece15_soft_target_top_label": 0.18117467075035296,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 42,
"nll": 0.8314159941361794,
"brier_sum_over_options": 0.2756424929725876,
"source_balanced_nll": 1.3491880345668303,
"source_balanced_brier": 0.4640535072101043,
"hard_label_n": 28,
"hard_accuracy": 0.7857142857142857,
"ece15_hard": 0.11473882152610511,
"ece15_soft_target_top_label": 0.09996606237372609,
"argmax_changes_vs_T1": 0
}
},
"46": {
"baseline_T1": {
"n": 28,
"nll": 0.22063032128614127,
"brier_sum_over_options": 0.1149215211280737,
"source_balanced_nll": 0.22063032128614127,
"source_balanced_brier": 0.1149215211280737,
"hard_label_n": 28,
"hard_accuracy": 0.9285714285714286,
"ece15_hard": 0.03850126263478336,
"ece15_soft_target_top_label": 0.03850126263478336,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 28,
"nll": 0.2321652667445951,
"brier_sum_over_options": 0.11680010408691169,
"source_balanced_nll": 0.2321652667445952,
"source_balanced_brier": 0.11680010408691172,
"hard_label_n": 28,
"hard_accuracy": 0.9285714285714286,
"ece15_hard": 0.07196001785616364,
"ece15_soft_target_top_label": 0.07196001785616364,
"argmax_changes_vs_T1": 0
}
},
"151": {
"baseline_T1": {
"n": 27,
"nll": 0.004604715853748762,
"brier_sum_over_options": 0.00012750703767116354,
"source_balanced_nll": 0.004604715853748763,
"source_balanced_brier": 0.0001275070376711636,
"hard_label_n": 27,
"hard_accuracy": 1.0,
"ece15_hard": 0.004568269164576533,
"ece15_soft_target_top_label": 0.004568269164576533,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 27,
"nll": 0.014876589347728297,
"brier_sum_over_options": 0.0006647270336140146,
"source_balanced_nll": 0.014876589347728297,
"source_balanced_brier": 0.0006647270336140146,
"hard_label_n": 27,
"hard_accuracy": 1.0,
"ece15_hard": 0.014646935063854838,
"ece15_soft_target_top_label": 0.014646935063854838,
"argmax_changes_vs_T1": 0
}
},
"174": {
"baseline_T1": {
"n": 199,
"nll": 6.089131697100824,
"brier_sum_over_options": 1.1393897207933026,
"source_balanced_nll": 6.089131697100823,
"source_balanced_brier": 1.1393897207933021,
"hard_label_n": 199,
"hard_accuracy": 0.05527638190954774,
"ece15_hard": 0.2599587761255519,
"ece15_soft_target_top_label": 0.2599587761255519,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 199,
"nll": 5.5972312315964174,
"brier_sum_over_options": 1.079531224764791,
"source_balanced_nll": 5.5972312315964174,
"source_balanced_brier": 1.0795312247647908,
"hard_label_n": 199,
"hard_accuracy": 0.05527638190954774,
"ece15_hard": 0.17813420139289587,
"ece15_soft_target_top_label": 0.17813420139289587,
"argmax_changes_vs_T1": 0
}
},
"8": {
"baseline_T1": {
"n": 128,
"nll": 0.6217994993057765,
"brier_sum_over_options": 0.28379704746901147,
"source_balanced_nll": 1.13244544662642,
"source_balanced_brier": 0.28692143547730253,
"hard_label_n": 127,
"hard_accuracy": 0.7795275590551181,
"ece15_hard": 0.07812389647018171,
"ece15_soft_target_top_label": 0.0783288755985589,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 128,
"nll": 0.6027463276612296,
"brier_sum_over_options": 0.2772966153890374,
"source_balanced_nll": 1.0967937460023398,
"source_balanced_brier": 0.27956312011942985,
"hard_label_n": 127,
"hard_accuracy": 0.7795275590551181,
"ece15_hard": 0.06140453183929856,
"ece15_soft_target_top_label": 0.061690747026549635,
"argmax_changes_vs_T1": 0
}
},
"26": {
"baseline_T1": {
"n": 10,
"nll": 0.6782343122294427,
"brier_sum_over_options": 0.31888276696776036,
"source_balanced_nll": 0.6839362107385382,
"source_balanced_brier": 0.3092345143753137,
"hard_label_n": 10,
"hard_accuracy": 0.8,
"ece15_hard": 0.15755902064415173,
"ece15_soft_target_top_label": 0.15755902064415173,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10,
"nll": 0.7667196391891683,
"brier_sum_over_options": 0.3324455276288491,
"source_balanced_nll": 0.821816955421676,
"source_balanced_brier": 0.36423887110224457,
"hard_label_n": 10,
"hard_accuracy": 0.8,
"ece15_hard": 0.16071276209491955,
"ece15_soft_target_top_label": 0.16071276209491955,
"argmax_changes_vs_T1": 0
}
},
"60": {
"baseline_T1": {
"n": 30,
"nll": 0.6068868724152872,
"brier_sum_over_options": 0.23492470460340126,
"source_balanced_nll": 0.6068868724152869,
"source_balanced_brier": 0.23492470460340123,
"hard_label_n": 30,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.1213210928743784,
"ece15_soft_target_top_label": 0.1213210928743784,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 30,
"nll": 0.574208985443045,
"brier_sum_over_options": 0.22809119128151892,
"source_balanced_nll": 0.574208985443045,
"source_balanced_brier": 0.2280911912815189,
"hard_label_n": 30,
"hard_accuracy": 0.8333333333333334,
"ece15_hard": 0.12498927840457866,
"ece15_soft_target_top_label": 0.12498927840457866,
"argmax_changes_vs_T1": 0
}
},
"77": {
"baseline_T1": {
"n": 23,
"nll": 0.17979156775813146,
"brier_sum_over_options": 0.07855611335844186,
"source_balanced_nll": 0.1797915677581315,
"source_balanced_brier": 0.07855611335844186,
"hard_label_n": 23,
"hard_accuracy": 0.9565217391304348,
"ece15_hard": 0.08498243179602454,
"ece15_soft_target_top_label": 0.08498243179602454,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 23,
"nll": 0.2166132535367895,
"brier_sum_over_options": 0.08438267023213168,
"source_balanced_nll": 0.2166132535367895,
"source_balanced_brier": 0.08438267023213171,
"hard_label_n": 23,
"hard_accuracy": 0.9565217391304348,
"ece15_hard": 0.1159196567852966,
"ece15_soft_target_top_label": 0.1159196567852966,
"argmax_changes_vs_T1": 0
}
},
"14": {
"baseline_T1": {
"n": 41,
"nll": 1.0668751741666684,
"brier_sum_over_options": 0.4378387318531714,
"source_balanced_nll": 0.6278262886817039,
"source_balanced_brier": 0.2651493353026454,
"hard_label_n": 41,
"hard_accuracy": 0.6585365853658537,
"ece15_hard": 0.14833965614136363,
"ece15_soft_target_top_label": 0.14833965614136363,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 41,
"nll": 1.0805650109548282,
"brier_sum_over_options": 0.440672101920061,
"source_balanced_nll": 0.6655367164535373,
"source_balanced_brier": 0.2739978949113573,
"hard_label_n": 41,
"hard_accuracy": 0.6585365853658537,
"ece15_hard": 0.19374436248848903,
"ece15_soft_target_top_label": 0.19374436248848903,
"argmax_changes_vs_T1": 0
}
},
"25": {
"baseline_T1": {
"n": 1,
"nll": 0.00021037501346573512,
"brier_sum_over_options": 6.988059189787206e-08,
"source_balanced_nll": 0.00021037501346573512,
"source_balanced_brier": 6.988059189787206e-08,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.00021035288619430936,
"ece15_soft_target_top_label": 0.00021035288619430936,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1,
"nll": 0.0012814849995642606,
"brier_sum_over_options": 2.2566101897773096e-06,
"source_balanced_nll": 0.0012814849995642606,
"source_balanced_brier": 2.2566101897773096e-06,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.0012806642482930952,
"ece15_soft_target_top_label": 0.0012806642482930952,
"argmax_changes_vs_T1": 0
}
},
"55": {
"baseline_T1": {
"n": 26,
"nll": 0.5595560475712149,
"brier_sum_over_options": 0.31853276986126855,
"source_balanced_nll": 0.559556047571215,
"source_balanced_brier": 0.3185327698612685,
"hard_label_n": 26,
"hard_accuracy": 0.8076923076923077,
"ece15_hard": 0.13763858319692607,
"ece15_soft_target_top_label": 0.13763858319692607,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 26,
"nll": 0.5612721713452827,
"brier_sum_over_options": 0.3149761008589313,
"source_balanced_nll": 0.5612721713452827,
"source_balanced_brier": 0.3149761008589313,
"hard_label_n": 26,
"hard_accuracy": 0.8076923076923077,
"ece15_hard": 0.19223441714332415,
"ece15_soft_target_top_label": 0.19223441714332415,
"argmax_changes_vs_T1": 0
}
},
"100": {
"baseline_T1": {
"n": 10,
"nll": 1.4187601674943622,
"brier_sum_over_options": 0.5283951877402946,
"source_balanced_nll": 1.4187601674943622,
"source_balanced_brier": 0.5283951877402946,
"hard_label_n": 10,
"hard_accuracy": 0.7,
"ece15_hard": 0.2810215214838676,
"ece15_soft_target_top_label": 0.2810215214838676,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10,
"nll": 1.2693268402366198,
"brier_sum_over_options": 0.5146513335916041,
"source_balanced_nll": 1.2693268402366196,
"source_balanced_brier": 0.514651333591604,
"hard_label_n": 10,
"hard_accuracy": 0.7,
"ece15_hard": 0.2791317548095704,
"ece15_soft_target_top_label": 0.2791317548095704,
"argmax_changes_vs_T1": 0
}
},
"12": {
"baseline_T1": {
"n": 4,
"nll": 0.19016785862226854,
"brier_sum_over_options": 0.11915582199701051,
"source_balanced_nll": 0.19016785862226854,
"source_balanced_brier": 0.11915582199701051,
"hard_label_n": 4,
"hard_accuracy": 1.0,
"ece15_hard": 0.14435070393884972,
"ece15_soft_target_top_label": 0.14435070393884972,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 4,
"nll": 0.22933654270800785,
"brier_sum_over_options": 0.12460462824330547,
"source_balanced_nll": 0.22933654270800785,
"source_balanced_brier": 0.12460462824330547,
"hard_label_n": 4,
"hard_accuracy": 1.0,
"ece15_hard": 0.17996493569310962,
"ece15_soft_target_top_label": 0.17996493569310962,
"argmax_changes_vs_T1": 0
}
},
"96": {
"baseline_T1": {
"n": 7,
"nll": 0.6344054687477539,
"brier_sum_over_options": 0.14243373382021554,
"source_balanced_nll": 0.634405468747754,
"source_balanced_brier": 0.14243373382021557,
"hard_label_n": 6,
"hard_accuracy": 1.0,
"ece15_hard": 0.12852053366837077,
"ece15_soft_target_top_label": 0.11544498217299713,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 7,
"nll": 0.6260825930519244,
"brier_sum_over_options": 0.14654161450034958,
"source_balanced_nll": 0.6260825930519244,
"source_balanced_brier": 0.1465416145003496,
"hard_label_n": 6,
"hard_accuracy": 1.0,
"ece15_hard": 0.1503075372779968,
"ece15_soft_target_top_label": 0.13526183634693043,
"argmax_changes_vs_T1": 0
}
},
"13": {
"baseline_T1": {
"n": 3,
"nll": 0.055850372198111974,
"brier_sum_over_options": 0.014579928048139838,
"source_balanced_nll": 0.05585037219811197,
"source_balanced_brier": 0.014579928048139838,
"hard_label_n": 3,
"hard_accuracy": 1.0,
"ece15_hard": 0.051428429669643005,
"ece15_soft_target_top_label": 0.051428429669643005,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3,
"nll": 0.07925563588489214,
"brier_sum_over_options": 0.026024010571508705,
"source_balanced_nll": 0.07925563588489212,
"source_balanced_brier": 0.026024010571508705,
"hard_label_n": 3,
"hard_accuracy": 1.0,
"ece15_hard": 0.0706061979456305,
"ece15_soft_target_top_label": 0.0706061979456305,
"argmax_changes_vs_T1": 0
}
},
"18": {
"baseline_T1": {
"n": 2,
"nll": 0.00015321243970713315,
"brier_sum_over_options": 5.812257036566845e-08,
"source_balanced_nll": 0.00015321243970713315,
"source_balanced_brier": 5.812257036566845e-08,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.00015319539995872766,
"ece15_soft_target_top_label": 0.00015319539995872766,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2,
"nll": 0.0009299787065514727,
"brier_sum_over_options": 1.582427728624678e-06,
"source_balanced_nll": 0.0009299787065514727,
"source_balanced_brier": 1.582427728624678e-06,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.0009294456581273591,
"ece15_soft_target_top_label": 0.0009294456581273591,
"argmax_changes_vs_T1": 0
}
},
"20": {
"baseline_T1": {
"n": 1,
"nll": 0.0003486236345549233,
"brier_sum_over_options": 1.3883399864364483e-07,
"source_balanced_nll": 0.0003486236345549233,
"source_balanced_brier": 1.3883399864364483e-07,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.0003485628723969292,
"ece15_soft_target_top_label": 0.0003485628723969292,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1,
"nll": 0.002337668970627584,
"brier_sum_over_options": 6.0080194953139875e-06,
"source_balanced_nll": 0.002337668970627584,
"source_balanced_brier": 6.0080194953139875e-06,
"hard_label_n": 1,
"hard_accuracy": 1.0,
"ece15_hard": 0.0023349387503842634,
"ece15_soft_target_top_label": 0.0023349387503842634,
"argmax_changes_vs_T1": 0
}
},
"15": {
"baseline_T1": {
"n": 2,
"nll": 0.3751794034386599,
"brier_sum_over_options": 0.20958336260990054,
"source_balanced_nll": 0.3751794034386599,
"source_balanced_brier": 0.20958336260990054,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.26391215423193704,
"ece15_soft_target_top_label": 0.26391215423193704,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2,
"nll": 0.421759116090247,
"brier_sum_over_options": 0.2319420283401216,
"source_balanced_nll": 0.421759116090247,
"source_balanced_brier": 0.2319420283401216,
"hard_label_n": 2,
"hard_accuracy": 1.0,
"ece15_hard": 0.28501566512025645,
"ece15_soft_target_top_label": 0.28501566512025645,
"argmax_changes_vs_T1": 0
}
}
},
"length_bucket": {
"128": {
"baseline_T1": {
"n": 2795,
"nll": 0.47894481602307243,
"brier_sum_over_options": 0.2329793196852082,
"source_balanced_nll": 0.43759908640966344,
"source_balanced_brier": 0.20097543908466445,
"hard_label_n": 2438,
"hard_accuracy": 0.8264971287940935,
"ece15_hard": 0.013381028589857631,
"ece15_soft_target_top_label": 0.018470153965915073,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 2795,
"nll": 0.47843872330270093,
"brier_sum_over_options": 0.23320563196516056,
"source_balanced_nll": 0.4478829139106674,
"source_balanced_brier": 0.2049984771990012,
"hard_label_n": 2438,
"hard_accuracy": 0.8264971287940935,
"ece15_hard": 0.03341391723188097,
"ece15_soft_target_top_label": 0.024613462816985565,
"argmax_changes_vs_T1": 0
}
},
"256": {
"baseline_T1": {
"n": 3633,
"nll": 0.8082300419985322,
"brier_sum_over_options": 0.3924719370871102,
"source_balanced_nll": 0.7027756248316769,
"source_balanced_brier": 0.33934571675261077,
"hard_label_n": 3381,
"hard_accuracy": 0.6805678793256433,
"ece15_hard": 0.053401579972982206,
"ece15_soft_target_top_label": 0.05343383213420854,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 3633,
"nll": 0.7997880763542585,
"brier_sum_over_options": 0.38734689731454064,
"source_balanced_nll": 0.6938142892251751,
"source_balanced_brier": 0.3351683380384077,
"hard_label_n": 3381,
"hard_accuracy": 0.6805678793256433,
"ece15_hard": 0.03154354282947759,
"ece15_soft_target_top_label": 0.02792805372156109,
"argmax_changes_vs_T1": 0
}
},
"512": {
"baseline_T1": {
"n": 1854,
"nll": 1.0837142432662947,
"brier_sum_over_options": 0.47187547077627484,
"source_balanced_nll": 0.873295200388099,
"source_balanced_brier": 0.38996230717858815,
"hard_label_n": 1767,
"hard_accuracy": 0.619128466327108,
"ece15_hard": 0.10746793409973858,
"ece15_soft_target_top_label": 0.10391363109551448,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 1854,
"nll": 1.0427599018966052,
"brier_sum_over_options": 0.4583898241454803,
"source_balanced_nll": 0.8466079492872749,
"source_balanced_brier": 0.38203669707621724,
"hard_label_n": 1767,
"hard_accuracy": 0.619128466327108,
"ece15_hard": 0.06000675148381363,
"ece15_soft_target_top_label": 0.05644598288132306,
"argmax_changes_vs_T1": 0
}
},
"1024": {
"baseline_T1": {
"n": 663,
"nll": 0.6370531661532262,
"brier_sum_over_options": 0.34126908148587454,
"source_balanced_nll": 0.7172978633192849,
"source_balanced_brier": 0.38140375127676807,
"hard_label_n": 628,
"hard_accuracy": 0.75,
"ece15_hard": 0.06338353188088795,
"ece15_soft_target_top_label": 0.05893613871948881,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 663,
"nll": 0.6251593312320077,
"brier_sum_over_options": 0.3354781540029996,
"source_balanced_nll": 0.6996748713870239,
"source_balanced_brier": 0.37360134750248913,
"hard_label_n": 628,
"hard_accuracy": 0.75,
"ece15_hard": 0.03617791572725478,
"ece15_soft_target_top_label": 0.037583535790362256,
"argmax_changes_vs_T1": 0
}
},
"2048": {
"baseline_T1": {
"n": 486,
"nll": 2.934371753220357,
"brier_sum_over_options": 0.7212951137452513,
"source_balanced_nll": 0.9372840096614883,
"source_balanced_brier": 0.34372131190130417,
"hard_label_n": 483,
"hard_accuracy": 0.4057971014492754,
"ece15_hard": 0.18174196601739398,
"ece15_soft_target_top_label": 0.18173537988281505,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 486,
"nll": 2.7125354846028364,
"brier_sum_over_options": 0.6861812323728023,
"source_balanced_nll": 0.8875682417282856,
"source_balanced_brier": 0.3308451224553599,
"hard_label_n": 483,
"hard_accuracy": 0.4057971014492754,
"ece15_hard": 0.13102365024567325,
"ece15_soft_target_top_label": 0.13083062698046766,
"argmax_changes_vs_T1": 0
}
},
"4096": {
"baseline_T1": {
"n": 220,
"nll": 0.8239227941876504,
"brier_sum_over_options": 0.50357074573417,
"source_balanced_nll": 0.6915513187665021,
"source_balanced_brier": 0.4315091788503526,
"hard_label_n": 220,
"hard_accuracy": 0.6136363636363636,
"ece15_hard": 0.17151284595484564,
"ece15_soft_target_top_label": 0.17151284595484564,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 220,
"nll": 0.7783430711541587,
"brier_sum_over_options": 0.47929985097472433,
"source_balanced_nll": 0.6679516357179132,
"source_balanced_brier": 0.41560952370795534,
"hard_label_n": 220,
"hard_accuracy": 0.6136363636363636,
"ece15_hard": 0.14466059920242538,
"ece15_soft_target_top_label": 0.14466059920242538,
"argmax_changes_vs_T1": 0
}
},
"8192": {
"baseline_T1": {
"n": 349,
"nll": 0.7338443482003226,
"brier_sum_over_options": 0.4368538783565455,
"source_balanced_nll": 0.8297455107236107,
"source_balanced_brier": 0.48946470854565366,
"hard_label_n": 349,
"hard_accuracy": 0.6934097421203438,
"ece15_hard": 0.14190864490761737,
"ece15_soft_target_top_label": 0.14190864490761737,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 349,
"nll": 0.6928612138175814,
"brier_sum_over_options": 0.41767489414099057,
"source_balanced_nll": 0.7825939870960537,
"source_balanced_brier": 0.47023329484963994,
"hard_label_n": 349,
"hard_accuracy": 0.6934097421203438,
"ece15_hard": 0.11256430198077799,
"ece15_soft_target_top_label": 0.11256430198077799,
"argmax_changes_vs_T1": 0
}
}
}
},
"accepted": true
}
},
"accepted": true,
"policy": "Select on selection only, then audit once; pooled and each audit panel must not worsen pooled or equal-family NLL/Brier versus T=1.0. No audit refitting or public/sealed benchmark fitting."
}
{
"source_release_id": "2818c9c47a86170ce9192c4cabd0715cb26f4c89b0ad557f8c3f15358b7f012a",
"data_counts": {
"fit": 7632,
"selection": 2368,
"audit_family": 2000,
"audit_group": 8000
},
"data_identity_sha256": "676e3c06bb9d3858875515c3c2da5f25b2b478fa2b9253ad11fa8f6b196b69ed",
"data_scope": "Reused historical holdout, newly screened and rescored. Not a fresh sealed test.",
"selection_and_audit_scoring": "Singleton, exact native serving numerical path; fit batching screened separately",
"baseline_temperature": 1.0,
"modes": {
"default": {
"fit_and_selection": {
"temperature": 1.2626014578759173,
"selected_fit_loss": "nll",
"candidates": [
{
"temperature": 1.0,
"fit_loss": "identity",
"at_search_boundary": false,
"selection_nll": 0.7275512028121913,
"selection_brier": 0.327622453770135,
"selection_score": 0.33580234422981187,
"eligible": true
},
{
"temperature": 1.2626014578759173,
"fit_loss": "nll",
"fit_objective": 0.7168196817779551,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712893716489562,
"selection_brier": 0.32333401069095247,
"selection_score": 0.3308736186536595,
"eligible": true
},
{
"temperature": 1.27003745705363,
"fit_loss": "brier",
"fit_objective": 0.31996540663853446,
"at_search_boundary": false,
"bounds": [
0.05,
20.0
],
"selection_nll": 0.712955260655177,
"selection_brier": 0.32334033241842747,
"selection_score": 0.3309141805567075,
"eligible": true
}
],
"policy": "Fit NLL and Brier temperatures on fit only; select once on separate selection rows. Identity T=1 is a candidate. No audit optimization."
},
"audit": {
"pooled": {
"passed": true,
"baseline_T1": {
"n": 10000,
"nll": 0.852946444060366,
"brier_sum_over_options": 0.3749600967063925,
"source_balanced_nll": 0.6999376751969619,
"source_balanced_brier": 0.30462924474543973,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.07901558557138746,
"ece15_soft_target_top_label": 0.07927800766590869,
"argmax_changes_vs_T1": 0
},
"candidate": {
"n": 10000,
"nll": 0.8154129693881843,
"brier_sum_over_options": 0.3636438958749284,
"source_balanced_nll": 0.6835431229745554,
"source_balanced_brier": 0.2998612344780709,
"hard_label_n": 9266,
"hard_accuracy": 0.703971508741636,
"ece15_hard": 0.038995968182245856,
"ece15_soft_target_top_label": 0.036747079044559566,
"argmax_changes_vs_T1": 0
},
"source_families": 84,
"family_cluster_bootstrap": {
"nll_delta_95": [
-0.03605991235662911,
-0.0013140520669599177
],
"brier_delta_95": [
-0.009391151837829768,
-0.0006976623009813496
]
},
"interval_scope": "New minus T=1; equal family weighting on this fixed historical holdout, not a guarantee under deployment shift."
},
"audit_group": {
"passed": true,