Buckets:

Mercity/Desicion-Model-Custom / report /figs /manifest_eval.json
Pranav2748's picture
download
raw
13.2 kB
[
{
"slug": "headline",
"title": "Headline accuracy by mode",
"caption": "Fast readout on all 5,442 eval questions; thinking and routed 30% (lowest-margin 30% answered by thinking; labels: mean thinking tokens per question) on the 1,498-question subset, where the fast readout scores 61.8 / 70.3 / 76.0 / 75.9%. Source: saved predictions, loaded and scored as in analysis/final_stats.py.",
"key_numbers": {
"fast": {
"base": 62.7,
"v1": 69.5,
"v2": 72.7,
"rl60": 72.5
},
"thinking": {
"base": 84.0,
"v1": 84.4,
"v2": 81.2,
"rl60": 83.1
},
"routed30": {
"base": 73.0,
"v1": 78.0,
"v2": 81.8,
"rl60": 81.8
},
"tokens30": {
"base": 1069.7,
"v1": 1166.3,
"v2": 1749.4,
"rl60": 729.0
}
}
},
{
"slug": "routing_curve",
"title": "Accuracy vs thinking tokens under margin routing",
"caption": "Accuracy vs mean thinking tokens per question as the lowest-margin share sent to thinking goes from 0% (left) to 100% (right) on the 1,498-question subset; a chain with no answer falls back to the readout (metrics_v2.py rule, hence V2's 84.2% at 100% vs 81.2% thinking-only), large dots mark 30%. Under this rule the 2B hybrid (readout-only 2B fast + untrained 2B thinking) gives 75.8% at 245 tokens at 30% vs 75.6% / 253 in the report's 2B table (probability-gap rule, no fallback); source: saved predictions, CI from results/final_stats.json.",
"key_numbers": {
"base": {
"acc30": 73.03,
"tok30": 1069.7,
"acc100": 84.45,
"tok100": 2772.7
},
"v1": {
"acc30": 77.97,
"tok30": 1166.3,
"acc100": 85.11,
"tok100": 2664.4
},
"v2": {
"acc30": 81.78,
"tok30": 1749.4,
"acc100": 84.25,
"tok100": 3748.2
},
"rl60": {
"acc30": 81.84,
"tok30": 729.0,
"acc100": 83.38,
"tok100": 1802.0
},
"hybrid2b": {
"acc30": 75.83,
"tok30": 245.1,
"acc100": 79.51,
"tok100": 763.2
},
"tokens30_rl60_minus_v2": [
-1020,
[
-1262,
-797
]
]
}
},
{
"slug": "per_set",
"title": "Fast-readout accuracy by eval set",
"caption": "Fast-readout accuracy on each eval set (5,442 questions in all), base -> v1 -> V2 -> V2 + RL; the two sets least like the training data (OpenDecisions, LocalLLaMA) gained least. Source: saved predictions (equal to runs/v2/results/metrics_rl.json).",
"key_numbers": {
"eval_proc": {
"base": 46.8,
"v1": 71.9,
"v2": 89.4,
"rl60": 89.5
},
"eval_ts": {
"base": 60.4,
"v1": 66.4,
"v2": 71.8,
"rl60": 72.3
},
"opendecisions": {
"base": 66.5,
"v1": 72.2,
"v2": 74.2,
"rl60": 73.9
},
"localllama": {
"base": 63.0,
"v1": 66.5,
"v2": 66.7,
"rl60": 66.3
}
}
},
{
"slug": "calibration",
"title": "Reliability diagrams, V2 + RL",
"caption": "Reliability diagrams (15 equal-width confidence bins; bottom: share of items per bin) for V2 + RL's fast readout on all 5,442 questions and its letter readout after the chain on the 1,498-question subset, both with per-type temperatures fitted on the calibration items. Post-chain ECE V2 + RL - V2: +0.014 (95% CI -0.006 to +0.028); source: saved predictions, results/final_stats.json.",
"key_numbers": {
"ece_fast": 0.0299,
"ece_post": 0.0568,
"post_n": 1495
}
},
{
"slug": "thinking_tokens",
"title": "Thinking tokens per chain, V2 vs V2 + RL",
"caption": "Thinking tokens per chain on the 1,498-question subset (log-spaced bins, share of all chains); red: chains that hit the length cap without an answer. Looped-chain rate V2 + RL - V2: -4.9 points (95% CI -6.0 to -3.8); source: saved predictions, results/final_stats.json.",
"key_numbers": {
"v2": {
"median": 1339.5,
"mean": 3748,
"loop_rate": 5.07,
"n_looped": 76,
"p90": 6580
},
"rl60": {
"median": 1095.5,
"mean": 1802,
"loop_rate": 0.2,
"n_looped": 3,
"p90": 4049
}
}
},
{
"slug": "fix_break",
"title": "Where thinking fixes and breaks the fast answer",
"caption": "V2 + RL on the 1,498-question subset: questions where thinking fixed a wrong fast answer or broke a right one, split by whether they fall in the lowest-margin 30% that routing sends to thinking. In the routed 30%, 146 fixes - 58 breaks = +88 questions (+5.9 points, 75.9% -> 81.8%); source: results/examples_analysis.json, recomputed from saved predictions.",
"key_numbers": {
"all": {
"n": 1498,
"routed": 449,
"fix": 214,
"fix_routed": 146,
"break": 106,
"break_routed": 58
},
"eval_proc": {
"n": 300,
"routed": 33,
"fix": 25,
"fix_routed": 18,
"break": 7,
"break_routed": 0
},
"eval_ts": {
"n": 300,
"routed": 112,
"fix": 36,
"fix_routed": 27,
"break": 34,
"break_routed": 20
},
"opendecisions": {
"n": 448,
"routed": 99,
"fix": 93,
"fix_routed": 47,
"break": 9,
"break_routed": 4
},
"localllama": {
"n": 450,
"routed": 205,
"fix": 60,
"fix_routed": 54,
"break": 56,
"break_routed": 34
}
}
},
{
"slug": "twob",
"title": "The 2B variants",
"caption": "MiniCPM5-2B variants next to the 4B V2 + RL (labels in bars: mean thinking tokens per question); the routed column uses the report's 2B-table rule (lowest probability gap to thinking, the chain's parsed letter as answer), which gives the 4B 81.9% / 742 tokens vs 81.8% / 729 under the headline rule. Hybrid at 50% routed: 78.0% at 411 tokens; source: saved predictions in runs/v2_2b and runs/v2.",
"key_numbers": {
"MiniCPM5-2B, untrained": {
"fast": 50.2,
"thinking": 79.4
},
"2B, V2 recipe": {
"fast": 65.1,
"thinking": 71.7,
"routed30": 71.9
},
"2B, readout-only": {
"fast": 66.0,
"thinking": 70.8,
"routed30": 71.0
},
"2B hybrid: readout-only fast + untrained thinking": {
"fast": 66.0,
"thinking": 79.4,
"routed30": 75.6
},
"4B V2 + RL (reference)": {
"fast": 72.5,
"thinking": 83.1,
"routed30": 81.9
},
"hybrid_routed50": [
78.0,
411
],
"loop_rates": {
"base2b": 0.1,
"v2b": 7.4,
"v2b_ro": 2.1
}
}
},
{
"slug": "jevals",
"title": "Jevals Decision Scores",
"caption": "Decision Score (Brier / RPS skill vs the label prior, x100, 95% CI) on the three Jevals 0.1.0 boards (300 items each): published entries in grey (ordered by mean score) and ours, scored by our reimplementation of the Jevals rules with calibration-fitted temperatures. Source: results/jevals.json.",
"key_numbers": {
"pubmedqa": {
"rl60": [
58.6,
[
47.3,
68.8
]
],
"rl60_think": [
63.5,
[
51.2,
74.8
]
],
"v2_2b_ro": [
44.6,
[
31.5,
56.5
]
]
},
"banking77": {
"rl60": [
61.3,
[
55.5,
67.3
]
],
"rl60_think": [
59.3,
[
53.6,
64.9
]
],
"v2_2b_ro": [
53.9,
[
48.4,
59.8
]
]
},
"helpsteer2": {
"rl60": [
2.4,
[
-7.2,
11.7
]
],
"rl60_think": [
-20.1,
[
-35.3,
-4.9
]
],
"v2_2b_ro": [
-0.4,
[
-8.2,
7.9
]
]
}
}
},
{
"slug": "s1mb_field",
"title": "S1MB against the published field",
"caption": "Left: S1MB English v1 Task Avg of the 211 complete published models (grey, ranked) and our five (fast readout); 74 published models rank above V2 + RL in the leaderboard's Borda order. Right: per-task scores of our 4B models vs the best published 4B (KnowLine-4B-Gen5, outlined) and the published median (dashed); source: results/s1mb.json (routed rows excluded).",
"key_numbers": {
"ours_task_avg": {
"base_2b": 18.1,
"base": 32.8,
"v2_2b_ro": 37.3,
"v1_joint": 43.5,
"rl60": 47.9
},
"ours_per_task": {
"base_2b": [
22.9,
29.5,
2.0
],
"base": [
33.8,
45.7,
18.9
],
"v2_2b_ro": [
42.3,
45.7,
24.0
],
"v1_joint": [
50.1,
52.3,
28.0
],
"rl60": [
50.7,
57.2,
35.8
]
},
"published_above_borda": {
"base_2b": 162,
"base": 134,
"v2_2b_ro": 123,
"v1_joint": 100,
"rl60": 74
},
"median": {
"task_avg": 41.2,
"noul": 47.9,
"choice": 51.6,
"score": 24.3
},
"best_4b": [
67.0,
65.7,
76.5,
58.8
]
}
},
{
"slug": "s1mb_seen_unseen",
"title": "S1MB gains by seen vs unseen source",
"caption": "Base -> V2 + RL on S1MB by whether a benchmark's source dataset contributed train-split items to our training pool (seen 41, unseen 96), with 95% CIs from 10,000 benchmark-level bootstrap resamples. Seen holds 1 score benchmark vs 20 unseen, so the task-matched contrast is the composition-fair one; source: results/final_stats.json.",
"key_numbers": {
"seen_gain": [
8.4,
[
4.7,
12.2
]
],
"unseen_gain": [
17.4,
[
13.4,
21.5
]
],
"unseen_minus_seen_gain": [
9.0,
[
3.4,
14.5
]
],
"unseen_minus_seen_gain_task_matched": [
8.2,
[
1.5,
14.7
]
],
"unseen_minus_seen_gain_noul_only": [
11.0,
[
-0.9,
22.6
]
],
"unseen_minus_seen_gain_choice_only": [
5.4,
[
-0.9,
11.9
]
]
}
},
{
"slug": "forest",
"title": "Paired-bootstrap differences",
"caption": "Differences in accuracy (and looped-chain rate), in points, with 95% percentile intervals from 10,000 paired bootstrap resamples of questions (same resample for every model, temperatures fixed); blue rows involve V2 + RL. Single training seed, so the intervals cover question sampling only; source: results/final_stats.json.",
"key_numbers": {
"Fast readout, 5,442 questions: V2 + RL \u2212 base": [
9.79,
[
8.43,
11.15
]
],
"Fast readout, 5,442 questions: V2 + RL \u2212 v1": [
3.0,
[
1.87,
4.15
]
],
"Fast readout, 5,442 questions: V2 \u2212 v1": [
3.23,
[
2.11,
4.39
]
],
"Fast readout, 5,442 questions: V2 + RL \u2212 V2": [
-0.24,
[
-0.5,
0.02
]
],
"Fast readout, 5,442 questions: RL step 40 \u2212 step 60": [
0.29,
[
0.04,
0.55
]
],
"Thinking, 1,498 questions: V2 + RL \u2212 V2": [
1.94,
[
0.33,
3.54
]
],
"Thinking, 1,498 questions: V2 + RL \u2212 base": [
-0.93,
[
-2.6,
0.73
]
],
"Thinking, 1,498 questions: V2 \u2212 base": [
-2.87,
[
-4.67,
-1.0
]
],
"Thinking, 1,498 questions: RL step 40 \u2212 step 60": [
0.47,
[
-1.07,
2.0
]
],
"Thinking, 1,498 questions: looped chains, V2 + RL \u2212 V2": [
-4.87,
[
-6.01,
-3.81
]
],
"Routed 30%, 1,498 questions: V2 + RL \u2212 base": [
8.81,
[
6.28,
10.75
]
],
"Routed 30%, 1,498 questions: V2 + RL \u2212 v1": [
3.87,
[
1.67,
5.61
]
],
"Routed 30%, 1,498 questions: V2 + RL \u2212 V2": [
0.07,
[
-1.27,
1.13
]
],
"Routed 30%, 1,498 questions: RL step 40 \u2212 step 60": [
0.2,
[
-1.07,
1.4
]
],
"Votes mixed in \u2212 gold only (V2 ablations): all 5,442": [
-1.19,
[
-2.13,
-0.28
]
],
"Votes mixed in \u2212 gold only (V2 ablations): in-distribution (1,125)": [
1.51,
[
-0.44,
3.38
]
],
"Votes mixed in \u2212 gold only (V2 ablations): out-of-distribution (4,317)": [
-1.9,
[
-2.94,
-0.86
]
],
"Votes mixed in \u2212 gold only (V2 ablations): in-distribution \u2212 OOD effect": [
3.41,
[
1.23,
5.54
]
]
}
},
{
"slug": "yesno",
"title": "Yes-rate vs gold yes-rate",
"caption": "Share of yes/no items the fast readout answers yes vs the gold yes share (our eval) or the target yes mass (S1MB); filled: per-type temperatures, open: plus one yes/no logit offset fitted on the 109 yes/no calibration items, notes: balanced accuracy (eval) or yes/no adjusted skill x100 (S1MB), without -> with the offset. Source: results/yesno_offset.json; S1MB targets from data/s1mb/items.jsonl.",
"key_numbers": {
"eval": {
"base": {
"yes_rate": 41.6,
"yes_rate_offset": 45.0,
"note": "bal. acc. 0.686 \u2192 0.704"
},
"v1_joint": {
"yes_rate": 58.4,
"yes_rate_offset": 53.3,
"note": "bal. acc. 0.786 \u2192 0.790"
},
"rl60": {
"yes_rate": 62.3,
"yes_rate_offset": 52.3,
"note": "bal. acc. 0.784 \u2192 0.836"
},
"gold_yes": 50.4
},
"s1mb": {
"base": {
"yes_rate": 18.4,
"yes_rate_offset": 20.6,
"note": "yes/no skill 33.8 \u2192 36.1"
},
"v1_joint": {
"yes_rate": 28.1,
"yes_rate_offset": 22.9,
"note": "yes/no skill 50.1 \u2192 47.6"
},
"rl60": {
"yes_rate": 31.6,
"yes_rate_offset": 25.0,
"note": "yes/no skill 50.7 \u2192 49.9"
},
"gold_yes": 27.5
}
}
}
]

Xet Storage Details

Size:
13.2 kB
·
Xet hash:
b1c865173acc1a109747f47dd897313dbc33357360bedbd8d18f13c298b1973d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.