| { |
| "what": "End-to-end fidelity of this bundle (as shipped) against the source model MiniMaxAI/MiniMax-M3.1-Flash, teacher-forced on held-out conversations never used for calibration (290 rows, 456,954 scored positions).", |
| "source": "MiniMaxAI/MiniMax-M3.1-Flash", |
| "source_revision": "262ae6830934f3512211081b8a3f16425b93a239", |
| "reference": "the source model at its release precision (NVFP4 experts, MXFP8 trunk, BF16 rest), run layer by layer; its top-128 log-probs saved per position", |
| "kl": "KL(source || bundle) over the source's top-128 tokens + one tail bucket (a lower bound of the full-vocabulary KL; tail_mass = the source's mean probability outside its top-128). top-k: the source's argmax is in the bundle's top k.", |
| "subsets": { |
| "asst": "assistant-written tokens (what the model generates)", |
| "all": "every scored position, incl. system prompts, tool schemas, tool results, user text and raw documents" |
| }, |
| "by_ref_top1_note": "bucketed by the source's top-1 probability, over ALL positions (not assistant-only)", |
| "nll_note": "mean per-row negative log-likelihood of the held-out text, nats / token, bundle vs source", |
| "domains": { |
| "agentic": { |
| "all": { |
| "n": 86482, |
| "kl_mean": 1.508469820022583, |
| "kl_median": 0.0983579158782959, |
| "kl_p90": 5.2097625732421875, |
| "kl_p99": 16.28428161621094, |
| "kl_p999": 23.14310354423523, |
| "kl_max": 38.00580596923828, |
| "top1": 0.6668324044309799, |
| "top5": 0.8107814342869036, |
| "top10": 0.835063943942092, |
| "tail_mass": 0.15552498400211334 |
| }, |
| "asst": { |
| "n": 34066, |
| "kl_mean": 0.12770643830299377, |
| "kl_median": 0.02191738970577717, |
| "kl_p90": 0.3034159988164902, |
| "kl_p99": 1.5784737825393673, |
| "kl_p999": 4.44348054647445, |
| "kl_max": 15.999199867248535, |
| "top1": 0.8881289262020783, |
| "top5": 0.993042916691129, |
| "top10": 0.9971525861562849, |
| "tail_mass": 0.0009966259822249413 |
| } |
| }, |
| "coding": { |
| "all": { |
| "n": 62399, |
| "kl_mean": 0.2324461042881012, |
| "kl_median": 0.0038485638797283173, |
| "kl_p90": 0.4210358142852787, |
| "kl_p99": 4.388715581893899, |
| "kl_p999": 13.684274654388423, |
| "kl_max": 21.77830696105957, |
| "top1": 0.8752544111283834, |
| "top5": 0.9731085434061443, |
| "top10": 0.9817304764499432, |
| "tail_mass": 0.10626104474067688 |
| }, |
| "asst": { |
| "n": 48223, |
| "kl_mean": 0.1396009474992752, |
| "kl_median": 0.0007559706573374569, |
| "kl_p90": 0.31315933465957696, |
| "kl_p99": 2.407493257522581, |
| "kl_p999": 6.208026328086806, |
| "kl_max": 21.207860946655273, |
| "top1": 0.9131949484685731, |
| "top5": 0.9908342492171786, |
| "top10": 0.9958940754411795, |
| "tail_mass": 0.001385890762321651 |
| } |
| }, |
| "encyclopedic": { |
| "all": { |
| "n": 41166, |
| "kl_mean": 1.5296459197998047, |
| "kl_median": 0.486764132976532, |
| "kl_p90": 4.187471628189087, |
| "kl_p99": 14.101272535324075, |
| "kl_p999": 20.763636550903083, |
| "kl_max": 31.98396873474121, |
| "top1": 0.5733129281445853, |
| "top5": 0.7892435505028421, |
| "top10": 0.8318758198513336, |
| "tail_mass": 0.1176367849111557 |
| }, |
| "asst": { |
| "n": 0 |
| } |
| }, |
| "long": { |
| "all": { |
| "n": 46820, |
| "kl_mean": 3.208937883377075, |
| "kl_median": 0.8069904148578644, |
| "kl_p90": 11.4564621925354, |
| "kl_p99": 19.34959836959837, |
| "kl_p999": 25.232344348907393, |
| "kl_max": 37.460914611816406, |
| "top1": 0.47689021785561725, |
| "top5": 0.6649935924818454, |
| "top10": 0.7075822298163178, |
| "tail_mass": 0.10943077504634857 |
| }, |
| "asst": { |
| "n": 782, |
| "kl_mean": 0.14281269907951355, |
| "kl_median": 0.007189885014668107, |
| "kl_p90": 0.3860674440860748, |
| "kl_p99": 2.17156644821166, |
| "kl_p999": 3.2352663674354565, |
| "kl_max": 3.256012201309204, |
| "top1": 0.8951406649616368, |
| "top5": 0.9987212276214834, |
| "top10": 1.0, |
| "tail_mass": 0.0006181662320159376 |
| } |
| }, |
| "multilingual": { |
| "all": { |
| "n": 33975, |
| "kl_mean": 0.9165863990783691, |
| "kl_median": 0.1339367777109146, |
| "kl_p90": 2.781315851211549, |
| "kl_p99": 9.240150108337406, |
| "kl_p999": 14.694798120498657, |
| "kl_max": 25.48598289489746, |
| "top1": 0.6667844002943341, |
| "top5": 0.8611920529801325, |
| "top10": 0.8986607799852833, |
| "tail_mass": 0.1891670674085617 |
| }, |
| "asst": { |
| "n": 18393, |
| "kl_mean": 0.7317777276039124, |
| "kl_median": 0.17410515248775482, |
| "kl_p90": 2.12237434387207, |
| "kl_p99": 6.620574245452857, |
| "kl_p999": 11.843588722229017, |
| "kl_max": 21.35243034362793, |
| "top1": 0.7141303756864025, |
| "top5": 0.9152938617952482, |
| "top10": 0.9489479693361605, |
| "tail_mass": 0.02085803635418415 |
| } |
| }, |
| "reasoning": { |
| "all": { |
| "n": 41707, |
| "kl_mean": 0.2212357223033905, |
| "kl_median": 0.006241295486688614, |
| "kl_p90": 0.39873183965682996, |
| "kl_p99": 4.193505477905284, |
| "kl_p999": 13.219146331787151, |
| "kl_max": 22.650331497192383, |
| "top1": 0.8675282326707747, |
| "top5": 0.9721629462680126, |
| "top10": 0.9811062891121395, |
| "tail_mass": 0.14284728467464447 |
| }, |
| "asst": { |
| "n": 29068, |
| "kl_mean": 0.09076923877000809, |
| "kl_median": 0.0013791727251373231, |
| "kl_p90": 0.2243695944547653, |
| "kl_p99": 1.3067078232765155, |
| "kl_p999": 4.1991275463105655, |
| "kl_max": 10.24682331085205, |
| "top1": 0.9262763176001101, |
| "top5": 0.997419843126462, |
| "top10": 0.9994495665336453, |
| "tail_mass": 0.00013747712364420295 |
| } |
| }, |
| "science": { |
| "all": { |
| "n": 26734, |
| "kl_mean": 0.4227638244628906, |
| "kl_median": 0.03396650031208992, |
| "kl_p90": 0.9841008961200715, |
| "kl_p99": 6.505592489242525, |
| "kl_p999": 13.584025408744793, |
| "kl_max": 21.618947982788086, |
| "top1": 0.7896685868182838, |
| "top5": 0.9468467120520685, |
| "top10": 0.9645021321164061, |
| "tail_mass": 0.16895170509815216 |
| }, |
| "asst": { |
| "n": 16236, |
| "kl_mean": 0.25138387084007263, |
| "kl_median": 0.04278343543410301, |
| "kl_p90": 0.6938526332378387, |
| "kl_p99": 2.756066703796385, |
| "kl_p999": 5.926196625232687, |
| "kl_max": 9.702275276184082, |
| "top1": 0.8451589061345159, |
| "top5": 0.9831855136733185, |
| "top10": 0.9946415373244641, |
| "tail_mass": 0.0010448005050420761 |
| } |
| }, |
| "story": { |
| "all": { |
| "n": 42457, |
| "kl_mean": 0.5738349556922913, |
| "kl_median": 0.1654246747493744, |
| "kl_p90": 1.5167509317398074, |
| "kl_p99": 5.57957151412967, |
| "kl_p999": 11.136457015991317, |
| "kl_max": 22.17845344543457, |
| "top1": 0.7218362107544103, |
| "top5": 0.9280919518571732, |
| "top10": 0.9571566526132322, |
| "tail_mass": 0.09046603739261627 |
| }, |
| "asst": { |
| "n": 36199, |
| "kl_mean": 0.6113303303718567, |
| "kl_median": 0.21570146083831787, |
| "kl_p90": 1.5764646768569948, |
| "kl_p99": 5.6232935142516896, |
| "kl_p999": 11.357586973190424, |
| "kl_max": 22.17845344543457, |
| "top1": 0.7192740130942844, |
| "top5": 0.9297218155197657, |
| "top10": 0.9598331445620045, |
| "tail_mass": 0.011700180359184742 |
| } |
| }, |
| "terminal": { |
| "all": { |
| "n": 75214, |
| "kl_mean": 1.7349458932876587, |
| "kl_median": 0.07192432135343552, |
| "kl_p90": 7.447272348403929, |
| "kl_p99": 16.31735948562621, |
| "kl_p999": 20.731649494171098, |
| "kl_max": 27.6101016998291, |
| "top1": 0.6846331799930864, |
| "top5": 0.8123487648576063, |
| "top10": 0.8342595793336347, |
| "tail_mass": 0.09347301721572876 |
| }, |
| "asst": { |
| "n": 29650, |
| "kl_mean": 0.04812491685152054, |
| "kl_median": 0.002247281721793115, |
| "kl_p90": 0.13092573434114468, |
| "kl_p99": 0.5531926125288009, |
| "kl_p999": 1.7451204873322859, |
| "kl_max": 6.852688312530518, |
| "top1": 0.9363237774030354, |
| "top5": 0.9993591905564924, |
| "top10": 0.9999662731871838, |
| "tail_mass": 0.00034418507129885256 |
| } |
| } |
| }, |
| "asst": { |
| "n": 212617, |
| "kl_mean": 0.2583523094654083, |
| "kl_median": 0.017217256128787994, |
| "kl_p90": 0.6730558037757871, |
| "kl_p99": 3.6089632701873704, |
| "kl_p999": 8.676778823852068, |
| "kl_max": 22.17845344543457, |
| "top1": 0.858694271859729, |
| "top5": 0.9757827455001247, |
| "top10": 0.9868684065714407, |
| "tail_mass": 0.004419249948114157 |
| }, |
| "all": { |
| "n": 456954, |
| "kl_mean": 1.2357864379882812, |
| "kl_median": 0.08520343899726868, |
| "kl_p90": 3.455284643173218, |
| "kl_p99": 15.398652334213246, |
| "kl_p999": 21.3631480636597, |
| "kl_max": 38.00580596923828, |
| "top1": 0.7009480166493782, |
| "top5": 0.8536657956818411, |
| "top10": 0.8785851529913296, |
| "tail_mass": 0.1265328973531723 |
| }, |
| "by_ref_top1": { |
| "0.00-0.30": { |
| "n": 96304, |
| "kl_mean": 1.116187334060669, |
| "kl_median": 0.1648135483264923, |
| "kl_p90": 3.305663824081419, |
| "kl_p99": 11.335794973373414, |
| "kl_p999": 16.780433691024783, |
| "kl_max": 29.367900848388672, |
| "top1": 0.4922225452733012, |
| "top5": 0.7186617378302044, |
| "top10": 0.766894417677355, |
| "tail_mass": 0.5368886590003967 |
| }, |
| "0.30-0.50": { |
| "n": 60456, |
| "kl_mean": 2.1936140060424805, |
| "kl_median": 0.6241882145404816, |
| "kl_p90": 7.443971395492554, |
| "kl_p99": 15.682296943664518, |
| "kl_p999": 21.46995011329628, |
| "kl_max": 30.840351104736328, |
| "top1": 0.4457952891359005, |
| "top5": 0.7625876670636496, |
| "top10": 0.8127067619425699, |
| "tail_mass": 0.05975927785038948 |
| }, |
| "0.50-0.70": { |
| "n": 60296, |
| "kl_mean": 1.9583995342254639, |
| "kl_median": 0.40552155673503876, |
| "kl_p90": 6.8943188190460205, |
| "kl_p99": 16.313577461242698, |
| "kl_p999": 21.978500204086306, |
| "kl_max": 28.978477478027344, |
| "top1": 0.5815476980230861, |
| "top5": 0.8304033435053735, |
| "top10": 0.8594599973464243, |
| "tail_mass": 0.025523990392684937 |
| }, |
| "0.70-0.90": { |
| "n": 63641, |
| "kl_mean": 1.7262171506881714, |
| "kl_median": 0.21256202459335327, |
| "kl_p90": 6.057215213775635, |
| "kl_p99": 17.045007705688473, |
| "kl_p999": 23.031990661621116, |
| "kl_max": 35.65987014770508, |
| "top1": 0.7264813563583224, |
| "top5": 0.867161106833645, |
| "top10": 0.8862839993086218, |
| "tail_mass": 0.012523962184786797 |
| }, |
| "0.90-0.99": { |
| "n": 56183, |
| "kl_mean": 1.2235275506973267, |
| "kl_median": 0.04953225702047348, |
| "kl_p90": 3.1279405593872087, |
| "kl_p99": 17.31826389312744, |
| "kl_p999": 23.185550689697266, |
| "kl_max": 38.00580596923828, |
| "top1": 0.8456294608689461, |
| "top5": 0.9135503622092092, |
| "top10": 0.9236779808838973, |
| "tail_mass": 0.002803773619234562 |
| }, |
| "0.99-1.00": { |
| "n": 120074, |
| "kl_mean": 0.23238903284072876, |
| "kl_median": 5.802925807074644e-05, |
| "kl_p90": 0.014083804842084643, |
| "kl_p99": 10.446070938110468, |
| "kl_p999": 20.292349637985204, |
| "kl_max": 33.77354049682617, |
| "top1": 0.9755484118127155, |
| "top5": 0.984309675699985, |
| "top10": 0.9857587820843813, |
| "tail_mass": 7.347401697188616e-05 |
| } |
| }, |
| "nll": { |
| "bundle": 5.6645501831482195, |
| "ref": 5.437649767152195, |
| "per_domain": { |
| "agentic": { |
| "bundle": 7.0839002597622756, |
| "ref": 6.959710650327729 |
| }, |
| "coding": { |
| "bundle": 4.480246908810674, |
| "ref": 4.402180228914533 |
| }, |
| "encyclopedic": { |
| "bundle": 5.958136944543748, |
| "ref": 5.174985454196022 |
| }, |
| "long": { |
| "bundle": 7.821170926094055, |
| "ref": 7.309172034263611 |
| }, |
| "multilingual": { |
| "bundle": 6.909620495729668, |
| "ref": 6.60997364964596 |
| }, |
| "reasoning": { |
| "bundle": 4.902472792731391, |
| "ref": 4.809402494960361 |
| }, |
| "science": { |
| "bundle": 5.188926813395127, |
| "ref": 4.909698082053143 |
| }, |
| "story": { |
| "bundle": 5.00568154335022, |
| "ref": 4.70310302734375 |
| }, |
| "terminal": { |
| "bundle": 5.92402221262455, |
| "ref": 5.867275953292847 |
| } |
| } |
| } |
| } |
