{
 "what": "End-to-end fidelity of this bundle (as shipped) against the source model MiniMaxAI/MiniMax-M3.1-Flash, teacher-forced on held-out conversations never used for calibration (290 rows, 456,954 scored positions).",
 "source": "MiniMaxAI/MiniMax-M3.1-Flash",
 "source_revision": "262ae6830934f3512211081b8a3f16425b93a239",
 "reference": "the source model at its release precision (NVFP4 experts, MXFP8 trunk, BF16 rest), run layer by layer; its top-128 log-probs saved per position",
 "kl": "KL(source || bundle) over the source's top-128 tokens + one tail bucket (a lower bound of the full-vocabulary KL; tail_mass = the source's mean probability outside its top-128). top-k: the source's argmax is in the bundle's top k.",
 "subsets": {
  "asst": "assistant-written tokens (what the model generates)",
  "all": "every scored position, incl. system prompts, tool schemas, tool results, user text and raw documents"
 },
 "by_ref_top1_note": "bucketed by the source's top-1 probability, over ALL positions (not assistant-only)",
 "nll_note": "mean per-row negative log-likelihood of the held-out text, nats / token, bundle vs source",
 "domains": {
  "agentic": {
   "all": {
    "n": 86482,
    "kl_mean": 1.508469820022583,
    "kl_median": 0.0983579158782959,
    "kl_p90": 5.2097625732421875,
    "kl_p99": 16.28428161621094,
    "kl_p999": 23.14310354423523,
    "kl_max": 38.00580596923828,
    "top1": 0.6668324044309799,
    "top5": 0.8107814342869036,
    "top10": 0.835063943942092,
    "tail_mass": 0.15552498400211334
   },
   "asst": {
    "n": 34066,
    "kl_mean": 0.12770643830299377,
    "kl_median": 0.02191738970577717,
    "kl_p90": 0.3034159988164902,
    "kl_p99": 1.5784737825393673,
    "kl_p999": 4.44348054647445,
    "kl_max": 15.999199867248535,
    "top1": 0.8881289262020783,
    "top5": 0.993042916691129,
    "top10": 0.9971525861562849,
    "tail_mass": 0.0009966259822249413
   }
  },
  "coding": {
   "all": {
    "n": 62399,
    "kl_mean": 0.2324461042881012,
    "kl_median": 0.0038485638797283173,
    "kl_p90": 0.4210358142852787,
    "kl_p99": 4.388715581893899,
    "kl_p999": 13.684274654388423,
    "kl_max": 21.77830696105957,
    "top1": 0.8752544111283834,
    "top5": 0.9731085434061443,
    "top10": 0.9817304764499432,
    "tail_mass": 0.10626104474067688
   },
   "asst": {
    "n": 48223,
    "kl_mean": 0.1396009474992752,
    "kl_median": 0.0007559706573374569,
    "kl_p90": 0.31315933465957696,
    "kl_p99": 2.407493257522581,
    "kl_p999": 6.208026328086806,
    "kl_max": 21.207860946655273,
    "top1": 0.9131949484685731,
    "top5": 0.9908342492171786,
    "top10": 0.9958940754411795,
    "tail_mass": 0.001385890762321651
   }
  },
  "encyclopedic": {
   "all": {
    "n": 41166,
    "kl_mean": 1.5296459197998047,
    "kl_median": 0.486764132976532,
    "kl_p90": 4.187471628189087,
    "kl_p99": 14.101272535324075,
    "kl_p999": 20.763636550903083,
    "kl_max": 31.98396873474121,
    "top1": 0.5733129281445853,
    "top5": 0.7892435505028421,
    "top10": 0.8318758198513336,
    "tail_mass": 0.1176367849111557
   },
   "asst": {
    "n": 0
   }
  },
  "long": {
   "all": {
    "n": 46820,
    "kl_mean": 3.208937883377075,
    "kl_median": 0.8069904148578644,
    "kl_p90": 11.4564621925354,
    "kl_p99": 19.34959836959837,
    "kl_p999": 25.232344348907393,
    "kl_max": 37.460914611816406,
    "top1": 0.47689021785561725,
    "top5": 0.6649935924818454,
    "top10": 0.7075822298163178,
    "tail_mass": 0.10943077504634857
   },
   "asst": {
    "n": 782,
    "kl_mean": 0.14281269907951355,
    "kl_median": 0.007189885014668107,
    "kl_p90": 0.3860674440860748,
    "kl_p99": 2.17156644821166,
    "kl_p999": 3.2352663674354565,
    "kl_max": 3.256012201309204,
    "top1": 0.8951406649616368,
    "top5": 0.9987212276214834,
    "top10": 1.0,
    "tail_mass": 0.0006181662320159376
   }
  },
  "multilingual": {
   "all": {
    "n": 33975,
    "kl_mean": 0.9165863990783691,
    "kl_median": 0.1339367777109146,
    "kl_p90": 2.781315851211549,
    "kl_p99": 9.240150108337406,
    "kl_p999": 14.694798120498657,
    "kl_max": 25.48598289489746,
    "top1": 0.6667844002943341,
    "top5": 0.8611920529801325,
    "top10": 0.8986607799852833,
    "tail_mass": 0.1891670674085617
   },
   "asst": {
    "n": 18393,
    "kl_mean": 0.7317777276039124,
    "kl_median": 0.17410515248775482,
    "kl_p90": 2.12237434387207,
    "kl_p99": 6.620574245452857,
    "kl_p999": 11.843588722229017,
    "kl_max": 21.35243034362793,
    "top1": 0.7141303756864025,
    "top5": 0.9152938617952482,
    "top10": 0.9489479693361605,
    "tail_mass": 0.02085803635418415
   }
  },
  "reasoning": {
   "all": {
    "n": 41707,
    "kl_mean": 0.2212357223033905,
    "kl_median": 0.006241295486688614,
    "kl_p90": 0.39873183965682996,
    "kl_p99": 4.193505477905284,
    "kl_p999": 13.219146331787151,
    "kl_max": 22.650331497192383,
    "top1": 0.8675282326707747,
    "top5": 0.9721629462680126,
    "top10": 0.9811062891121395,
    "tail_mass": 0.14284728467464447
   },
   "asst": {
    "n": 29068,
    "kl_mean": 0.09076923877000809,
    "kl_median": 0.0013791727251373231,
    "kl_p90": 0.2243695944547653,
    "kl_p99": 1.3067078232765155,
    "kl_p999": 4.1991275463105655,
    "kl_max": 10.24682331085205,
    "top1": 0.9262763176001101,
    "top5": 0.997419843126462,
    "top10": 0.9994495665336453,
    "tail_mass": 0.00013747712364420295
   }
  },
  "science": {
   "all": {
    "n": 26734,
    "kl_mean": 0.4227638244628906,
    "kl_median": 0.03396650031208992,
    "kl_p90": 0.9841008961200715,
    "kl_p99": 6.505592489242525,
    "kl_p999": 13.584025408744793,
    "kl_max": 21.618947982788086,
    "top1": 0.7896685868182838,
    "top5": 0.9468467120520685,
    "top10": 0.9645021321164061,
    "tail_mass": 0.16895170509815216
   },
   "asst": {
    "n": 16236,
    "kl_mean": 0.25138387084007263,
    "kl_median": 0.04278343543410301,
    "kl_p90": 0.6938526332378387,
    "kl_p99": 2.756066703796385,
    "kl_p999": 5.926196625232687,
    "kl_max": 9.702275276184082,
    "top1": 0.8451589061345159,
    "top5": 0.9831855136733185,
    "top10": 0.9946415373244641,
    "tail_mass": 0.0010448005050420761
   }
  },
  "story": {
   "all": {
    "n": 42457,
    "kl_mean": 0.5738349556922913,
    "kl_median": 0.1654246747493744,
    "kl_p90": 1.5167509317398074,
    "kl_p99": 5.57957151412967,
    "kl_p999": 11.136457015991317,
    "kl_max": 22.17845344543457,
    "top1": 0.7218362107544103,
    "top5": 0.9280919518571732,
    "top10": 0.9571566526132322,
    "tail_mass": 0.09046603739261627
   },
   "asst": {
    "n": 36199,
    "kl_mean": 0.6113303303718567,
    "kl_median": 0.21570146083831787,
    "kl_p90": 1.5764646768569948,
    "kl_p99": 5.6232935142516896,
    "kl_p999": 11.357586973190424,
    "kl_max": 22.17845344543457,
    "top1": 0.7192740130942844,
    "top5": 0.9297218155197657,
    "top10": 0.9598331445620045,
    "tail_mass": 0.011700180359184742
   }
  },
  "terminal": {
   "all": {
    "n": 75214,
    "kl_mean": 1.7349458932876587,
    "kl_median": 0.07192432135343552,
    "kl_p90": 7.447272348403929,
    "kl_p99": 16.31735948562621,
    "kl_p999": 20.731649494171098,
    "kl_max": 27.6101016998291,
    "top1": 0.6846331799930864,
    "top5": 0.8123487648576063,
    "top10": 0.8342595793336347,
    "tail_mass": 0.09347301721572876
   },
   "asst": {
    "n": 29650,
    "kl_mean": 0.04812491685152054,
    "kl_median": 0.002247281721793115,
    "kl_p90": 0.13092573434114468,
    "kl_p99": 0.5531926125288009,
    "kl_p999": 1.7451204873322859,
    "kl_max": 6.852688312530518,
    "top1": 0.9363237774030354,
    "top5": 0.9993591905564924,
    "top10": 0.9999662731871838,
    "tail_mass": 0.00034418507129885256
   }
  }
 },
 "asst": {
  "n": 212617,
  "kl_mean": 0.2583523094654083,
  "kl_median": 0.017217256128787994,
  "kl_p90": 0.6730558037757871,
  "kl_p99": 3.6089632701873704,
  "kl_p999": 8.676778823852068,
  "kl_max": 22.17845344543457,
  "top1": 0.858694271859729,
  "top5": 0.9757827455001247,
  "top10": 0.9868684065714407,
  "tail_mass": 0.004419249948114157
 },
 "all": {
  "n": 456954,
  "kl_mean": 1.2357864379882812,
  "kl_median": 0.08520343899726868,
  "kl_p90": 3.455284643173218,
  "kl_p99": 15.398652334213246,
  "kl_p999": 21.3631480636597,
  "kl_max": 38.00580596923828,
  "top1": 0.7009480166493782,
  "top5": 0.8536657956818411,
  "top10": 0.8785851529913296,
  "tail_mass": 0.1265328973531723
 },
 "by_ref_top1": {
  "0.00-0.30": {
   "n": 96304,
   "kl_mean": 1.116187334060669,
   "kl_median": 0.1648135483264923,
   "kl_p90": 3.305663824081419,
   "kl_p99": 11.335794973373414,
   "kl_p999": 16.780433691024783,
   "kl_max": 29.367900848388672,
   "top1": 0.4922225452733012,
   "top5": 0.7186617378302044,
   "top10": 0.766894417677355,
   "tail_mass": 0.5368886590003967
  },
  "0.30-0.50": {
   "n": 60456,
   "kl_mean": 2.1936140060424805,
   "kl_median": 0.6241882145404816,
   "kl_p90": 7.443971395492554,
   "kl_p99": 15.682296943664518,
   "kl_p999": 21.46995011329628,
   "kl_max": 30.840351104736328,
   "top1": 0.4457952891359005,
   "top5": 0.7625876670636496,
   "top10": 0.8127067619425699,
   "tail_mass": 0.05975927785038948
  },
  "0.50-0.70": {
   "n": 60296,
   "kl_mean": 1.9583995342254639,
   "kl_median": 0.40552155673503876,
   "kl_p90": 6.8943188190460205,
   "kl_p99": 16.313577461242698,
   "kl_p999": 21.978500204086306,
   "kl_max": 28.978477478027344,
   "top1": 0.5815476980230861,
   "top5": 0.8304033435053735,
   "top10": 0.8594599973464243,
   "tail_mass": 0.025523990392684937
  },
  "0.70-0.90": {
   "n": 63641,
   "kl_mean": 1.7262171506881714,
   "kl_median": 0.21256202459335327,
   "kl_p90": 6.057215213775635,
   "kl_p99": 17.045007705688473,
   "kl_p999": 23.031990661621116,
   "kl_max": 35.65987014770508,
   "top1": 0.7264813563583224,
   "top5": 0.867161106833645,
   "top10": 0.8862839993086218,
   "tail_mass": 0.012523962184786797
  },
  "0.90-0.99": {
   "n": 56183,
   "kl_mean": 1.2235275506973267,
   "kl_median": 0.04953225702047348,
   "kl_p90": 3.1279405593872087,
   "kl_p99": 17.31826389312744,
   "kl_p999": 23.185550689697266,
   "kl_max": 38.00580596923828,
   "top1": 0.8456294608689461,
   "top5": 0.9135503622092092,
   "top10": 0.9236779808838973,
   "tail_mass": 0.002803773619234562
  },
  "0.99-1.00": {
   "n": 120074,
   "kl_mean": 0.23238903284072876,
   "kl_median": 5.802925807074644e-05,
   "kl_p90": 0.014083804842084643,
   "kl_p99": 10.446070938110468,
   "kl_p999": 20.292349637985204,
   "kl_max": 33.77354049682617,
   "top1": 0.9755484118127155,
   "top5": 0.984309675699985,
   "top10": 0.9857587820843813,
   "tail_mass": 7.347401697188616e-05
  }
 },
 "nll": {
  "bundle": 5.6645501831482195,
  "ref": 5.437649767152195,
  "per_domain": {
   "agentic": {
    "bundle": 7.0839002597622756,
    "ref": 6.959710650327729
   },
   "coding": {
    "bundle": 4.480246908810674,
    "ref": 4.402180228914533
   },
   "encyclopedic": {
    "bundle": 5.958136944543748,
    "ref": 5.174985454196022
   },
   "long": {
    "bundle": 7.821170926094055,
    "ref": 7.309172034263611
   },
   "multilingual": {
    "bundle": 6.909620495729668,
    "ref": 6.60997364964596
   },
   "reasoning": {
    "bundle": 4.902472792731391,
    "ref": 4.809402494960361
   },
   "science": {
    "bundle": 5.188926813395127,
    "ref": 4.909698082053143
   },
   "story": {
    "bundle": 5.00568154335022,
    "ref": 4.70310302734375
   },
   "terminal": {
    "bundle": 5.92402221262455,
    "ref": 5.867275953292847
   }
  }
 }
}