| { |
| "what": "Adaptive reasoning decision gate: at every assistant-turn start in the held-out set (no prefix forced), the first token is the model's own choice between <mm:think> (think) and </mm:think> (answer directly). The bundle's choice vs the source's, teacher-forced.", |
| "source": "MiniMaxAI/MiniMax-M3.1-Flash", |
| "source_revision": "262ae6830934f3512211081b8a3f16425b93a239", |
| "fields": { |
| "n": "assistant-turn starts", |
| "agree": "fraction where the bundle and the source make the same think / skip choice (argmax of the two tokens)", |
| "ref_think": "fraction where the source thinks", |
| "q_think": "fraction where the bundle thinks", |
| "mean_abs_dp_think": "mean |P_bundle(<mm:think>) - P_source(<mm:think>)|" |
| }, |
| "by_effort_note": "effort = the reasoning_effort chat-template kwarg of the row; None = no effort tag", |
| "all": { |
| "n": 327, |
| "agree": 0.9755351681957186, |
| "ref_think": 0.8960244648318043, |
| "q_think": 0.8899082568807339, |
| "mean_abs_dp_think": 0.024736272980313813 |
| }, |
| "by_domain": { |
| "agentic": { |
| "n": 86, |
| "agree": 0.9651162790697675, |
| "ref_think": 0.627906976744186, |
| "q_think": 0.6162790697674418, |
| "mean_abs_dp_think": 0.06704882904586505 |
| }, |
| "coding": { |
| "n": 49, |
| "agree": 0.9183673469387755, |
| "ref_think": 0.9591836734693877, |
| "q_think": 0.9591836734693877, |
| "mean_abs_dp_think": 0.021642896144900926 |
| }, |
| "long": { |
| "n": 4, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 3.337360472549511e-05 |
| }, |
| "multilingual": { |
| "n": 40, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 0.00015628930211283032 |
| }, |
| "reasoning": { |
| "n": 45, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 0.005851560307713704 |
| }, |
| "science": { |
| "n": 33, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 1.8494246537941788e-05 |
| }, |
| "story": { |
| "n": 25, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 0.0014211676723039268 |
| }, |
| "terminal": { |
| "n": 45, |
| "agree": 0.9777777777777777, |
| "ref_think": 1.0, |
| "q_think": 0.9777777777777777, |
| "mean_abs_dp_think": 0.021249228293871006 |
| } |
| }, |
| "by_effort": { |
| "None": { |
| "n": 194, |
| "agree": 0.9742268041237113, |
| "ref_think": 0.8917525773195877, |
| "q_think": 0.8865979381443299, |
| "mean_abs_dp_think": 0.024562513000013695 |
| }, |
| "high": { |
| "n": 34, |
| "agree": 0.9705882352941176, |
| "ref_think": 0.8529411764705882, |
| "q_think": 0.8235294117647058, |
| "mean_abs_dp_think": 0.030750843035456197 |
| }, |
| "low": { |
| "n": 24, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 0.007379506022084735 |
| }, |
| "max": { |
| "n": 38, |
| "agree": 0.9736842105263158, |
| "ref_think": 0.868421052631579, |
| "q_think": 0.8421052631578947, |
| "mean_abs_dp_think": 0.02558943064046118 |
| }, |
| "medium": { |
| "n": 26, |
| "agree": 0.9615384615384616, |
| "ref_think": 0.8846153846153846, |
| "q_think": 0.9230769230769231, |
| "mean_abs_dp_think": 0.042194932052480864 |
| }, |
| "xhigh": { |
| "n": 11, |
| "agree": 1.0, |
| "ref_think": 1.0, |
| "q_think": 1.0, |
| "mean_abs_dp_think": 0.0028663942838535613 |
| } |
| } |
| } |
