Buckets:
| [ | |
| { | |
| "slug": "headline", | |
| "title": "Headline accuracy by mode", | |
| "caption": "Fast readout on all 5,442 eval questions; thinking and routed 30% (lowest-margin 30% answered by thinking; labels: mean thinking tokens per question) on the 1,498-question subset, where the fast readout scores 61.8 / 70.3 / 76.0 / 75.9%. Source: saved predictions, loaded and scored as in analysis/final_stats.py.", | |
| "key_numbers": { | |
| "fast": { | |
| "base": 62.7, | |
| "v1": 69.5, | |
| "v2": 72.7, | |
| "rl60": 72.5 | |
| }, | |
| "thinking": { | |
| "base": 84.0, | |
| "v1": 84.4, | |
| "v2": 81.2, | |
| "rl60": 83.1 | |
| }, | |
| "routed30": { | |
| "base": 73.0, | |
| "v1": 78.0, | |
| "v2": 81.8, | |
| "rl60": 81.8 | |
| }, | |
| "tokens30": { | |
| "base": 1069.7, | |
| "v1": 1166.3, | |
| "v2": 1749.4, | |
| "rl60": 729.0 | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "routing_curve", | |
| "title": "Accuracy vs thinking tokens under margin routing", | |
| "caption": "Accuracy vs mean thinking tokens per question as the lowest-margin share sent to thinking goes from 0% (left) to 100% (right) on the 1,498-question subset; a chain with no answer falls back to the readout (metrics_v2.py rule, hence V2's 84.2% at 100% vs 81.2% thinking-only), large dots mark 30%. Under this rule the 2B hybrid (readout-only 2B fast + untrained 2B thinking) gives 75.8% at 245 tokens at 30% vs 75.6% / 253 in the report's 2B table (probability-gap rule, no fallback); source: saved predictions, CI from results/final_stats.json.", | |
| "key_numbers": { | |
| "base": { | |
| "acc30": 73.03, | |
| "tok30": 1069.7, | |
| "acc100": 84.45, | |
| "tok100": 2772.7 | |
| }, | |
| "v1": { | |
| "acc30": 77.97, | |
| "tok30": 1166.3, | |
| "acc100": 85.11, | |
| "tok100": 2664.4 | |
| }, | |
| "v2": { | |
| "acc30": 81.78, | |
| "tok30": 1749.4, | |
| "acc100": 84.25, | |
| "tok100": 3748.2 | |
| }, | |
| "rl60": { | |
| "acc30": 81.84, | |
| "tok30": 729.0, | |
| "acc100": 83.38, | |
| "tok100": 1802.0 | |
| }, | |
| "hybrid2b": { | |
| "acc30": 75.83, | |
| "tok30": 245.1, | |
| "acc100": 79.51, | |
| "tok100": 763.2 | |
| }, | |
| "tokens30_rl60_minus_v2": [ | |
| -1020, | |
| [ | |
| -1262, | |
| -797 | |
| ] | |
| ] | |
| } | |
| }, | |
| { | |
| "slug": "per_set", | |
| "title": "Fast-readout accuracy by eval set", | |
| "caption": "Fast-readout accuracy on each eval set (5,442 questions in all), base -> v1 -> V2 -> V2 + RL; the two sets least like the training data (OpenDecisions, LocalLLaMA) gained least. Source: saved predictions (equal to runs/v2/results/metrics_rl.json).", | |
| "key_numbers": { | |
| "eval_proc": { | |
| "base": 46.8, | |
| "v1": 71.9, | |
| "v2": 89.4, | |
| "rl60": 89.5 | |
| }, | |
| "eval_ts": { | |
| "base": 60.4, | |
| "v1": 66.4, | |
| "v2": 71.8, | |
| "rl60": 72.3 | |
| }, | |
| "opendecisions": { | |
| "base": 66.5, | |
| "v1": 72.2, | |
| "v2": 74.2, | |
| "rl60": 73.9 | |
| }, | |
| "localllama": { | |
| "base": 63.0, | |
| "v1": 66.5, | |
| "v2": 66.7, | |
| "rl60": 66.3 | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "calibration", | |
| "title": "Reliability diagrams, V2 + RL", | |
| "caption": "Reliability diagrams (15 equal-width confidence bins; bottom: share of items per bin) for V2 + RL's fast readout on all 5,442 questions and its letter readout after the chain on the 1,498-question subset, both with per-type temperatures fitted on the calibration items. Post-chain ECE V2 + RL - V2: +0.014 (95% CI -0.006 to +0.028); source: saved predictions, results/final_stats.json.", | |
| "key_numbers": { | |
| "ece_fast": 0.0299, | |
| "ece_post": 0.0568, | |
| "post_n": 1495 | |
| } | |
| }, | |
| { | |
| "slug": "thinking_tokens", | |
| "title": "Thinking tokens per chain, V2 vs V2 + RL", | |
| "caption": "Thinking tokens per chain on the 1,498-question subset (log-spaced bins, share of all chains); red: chains that hit the length cap without an answer. Looped-chain rate V2 + RL - V2: -4.9 points (95% CI -6.0 to -3.8); source: saved predictions, results/final_stats.json.", | |
| "key_numbers": { | |
| "v2": { | |
| "median": 1339.5, | |
| "mean": 3748, | |
| "loop_rate": 5.07, | |
| "n_looped": 76, | |
| "p90": 6580 | |
| }, | |
| "rl60": { | |
| "median": 1095.5, | |
| "mean": 1802, | |
| "loop_rate": 0.2, | |
| "n_looped": 3, | |
| "p90": 4049 | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "fix_break", | |
| "title": "Where thinking fixes and breaks the fast answer", | |
| "caption": "V2 + RL on the 1,498-question subset: questions where thinking fixed a wrong fast answer or broke a right one, split by whether they fall in the lowest-margin 30% that routing sends to thinking. In the routed 30%, 146 fixes - 58 breaks = +88 questions (+5.9 points, 75.9% -> 81.8%); source: results/examples_analysis.json, recomputed from saved predictions.", | |
| "key_numbers": { | |
| "all": { | |
| "n": 1498, | |
| "routed": 449, | |
| "fix": 214, | |
| "fix_routed": 146, | |
| "break": 106, | |
| "break_routed": 58 | |
| }, | |
| "eval_proc": { | |
| "n": 300, | |
| "routed": 33, | |
| "fix": 25, | |
| "fix_routed": 18, | |
| "break": 7, | |
| "break_routed": 0 | |
| }, | |
| "eval_ts": { | |
| "n": 300, | |
| "routed": 112, | |
| "fix": 36, | |
| "fix_routed": 27, | |
| "break": 34, | |
| "break_routed": 20 | |
| }, | |
| "opendecisions": { | |
| "n": 448, | |
| "routed": 99, | |
| "fix": 93, | |
| "fix_routed": 47, | |
| "break": 9, | |
| "break_routed": 4 | |
| }, | |
| "localllama": { | |
| "n": 450, | |
| "routed": 205, | |
| "fix": 60, | |
| "fix_routed": 54, | |
| "break": 56, | |
| "break_routed": 34 | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "twob", | |
| "title": "The 2B variants", | |
| "caption": "MiniCPM5-2B variants next to the 4B V2 + RL (labels in bars: mean thinking tokens per question); the routed column uses the report's 2B-table rule (lowest probability gap to thinking, the chain's parsed letter as answer), which gives the 4B 81.9% / 742 tokens vs 81.8% / 729 under the headline rule. Hybrid at 50% routed: 78.0% at 411 tokens; source: saved predictions in runs/v2_2b and runs/v2.", | |
| "key_numbers": { | |
| "MiniCPM5-2B, untrained": { | |
| "fast": 50.2, | |
| "thinking": 79.4 | |
| }, | |
| "2B, V2 recipe": { | |
| "fast": 65.1, | |
| "thinking": 71.7, | |
| "routed30": 71.9 | |
| }, | |
| "2B, readout-only": { | |
| "fast": 66.0, | |
| "thinking": 70.8, | |
| "routed30": 71.0 | |
| }, | |
| "2B hybrid: readout-only fast + untrained thinking": { | |
| "fast": 66.0, | |
| "thinking": 79.4, | |
| "routed30": 75.6 | |
| }, | |
| "4B V2 + RL (reference)": { | |
| "fast": 72.5, | |
| "thinking": 83.1, | |
| "routed30": 81.9 | |
| }, | |
| "hybrid_routed50": [ | |
| 78.0, | |
| 411 | |
| ], | |
| "loop_rates": { | |
| "base2b": 0.1, | |
| "v2b": 7.4, | |
| "v2b_ro": 2.1 | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "jevals", | |
| "title": "Jevals Decision Scores", | |
| "caption": "Decision Score (Brier / RPS skill vs the label prior, x100, 95% CI) on the three Jevals 0.1.0 boards (300 items each): published entries in grey (ordered by mean score) and ours, scored by our reimplementation of the Jevals rules with calibration-fitted temperatures. Source: results/jevals.json.", | |
| "key_numbers": { | |
| "pubmedqa": { | |
| "rl60": [ | |
| 58.6, | |
| [ | |
| 47.3, | |
| 68.8 | |
| ] | |
| ], | |
| "rl60_think": [ | |
| 63.5, | |
| [ | |
| 51.2, | |
| 74.8 | |
| ] | |
| ], | |
| "v2_2b_ro": [ | |
| 44.6, | |
| [ | |
| 31.5, | |
| 56.5 | |
| ] | |
| ] | |
| }, | |
| "banking77": { | |
| "rl60": [ | |
| 61.3, | |
| [ | |
| 55.5, | |
| 67.3 | |
| ] | |
| ], | |
| "rl60_think": [ | |
| 59.3, | |
| [ | |
| 53.6, | |
| 64.9 | |
| ] | |
| ], | |
| "v2_2b_ro": [ | |
| 53.9, | |
| [ | |
| 48.4, | |
| 59.8 | |
| ] | |
| ] | |
| }, | |
| "helpsteer2": { | |
| "rl60": [ | |
| 2.4, | |
| [ | |
| -7.2, | |
| 11.7 | |
| ] | |
| ], | |
| "rl60_think": [ | |
| -20.1, | |
| [ | |
| -35.3, | |
| -4.9 | |
| ] | |
| ], | |
| "v2_2b_ro": [ | |
| -0.4, | |
| [ | |
| -8.2, | |
| 7.9 | |
| ] | |
| ] | |
| } | |
| } | |
| }, | |
| { | |
| "slug": "s1mb_field", | |
| "title": "S1MB against the published field", | |
| "caption": "Left: S1MB English v1 Task Avg of the 211 complete published models (grey, ranked) and our five (fast readout); 74 published models rank above V2 + RL in the leaderboard's Borda order. Right: per-task scores of our 4B models vs the best published 4B (KnowLine-4B-Gen5, outlined) and the published median (dashed); source: results/s1mb.json (routed rows excluded).", | |
| "key_numbers": { | |
| "ours_task_avg": { | |
| "base_2b": 18.1, | |
| "base": 32.8, | |
| "v2_2b_ro": 37.3, | |
| "v1_joint": 43.5, | |
| "rl60": 47.9 | |
| }, | |
| "ours_per_task": { | |
| "base_2b": [ | |
| 22.9, | |
| 29.5, | |
| 2.0 | |
| ], | |
| "base": [ | |
| 33.8, | |
| 45.7, | |
| 18.9 | |
| ], | |
| "v2_2b_ro": [ | |
| 42.3, | |
| 45.7, | |
| 24.0 | |
| ], | |
| "v1_joint": [ | |
| 50.1, | |
| 52.3, | |
| 28.0 | |
| ], | |
| "rl60": [ | |
| 50.7, | |
| 57.2, | |
| 35.8 | |
| ] | |
| }, | |
| "published_above_borda": { | |
| "base_2b": 162, | |
| "base": 134, | |
| "v2_2b_ro": 123, | |
| "v1_joint": 100, | |
| "rl60": 74 | |
| }, | |
| "median": { | |
| "task_avg": 41.2, | |
| "noul": 47.9, | |
| "choice": 51.6, | |
| "score": 24.3 | |
| }, | |
| "best_4b": [ | |
| 67.0, | |
| 65.7, | |
| 76.5, | |
| 58.8 | |
| ] | |
| } | |
| }, | |
| { | |
| "slug": "s1mb_seen_unseen", | |
| "title": "S1MB gains by seen vs unseen source", | |
| "caption": "Base -> V2 + RL on S1MB by whether a benchmark's source dataset contributed train-split items to our training pool (seen 41, unseen 96), with 95% CIs from 10,000 benchmark-level bootstrap resamples. Seen holds 1 score benchmark vs 20 unseen, so the task-matched contrast is the composition-fair one; source: results/final_stats.json.", | |
| "key_numbers": { | |
| "seen_gain": [ | |
| 8.4, | |
| [ | |
| 4.7, | |
| 12.2 | |
| ] | |
| ], | |
| "unseen_gain": [ | |
| 17.4, | |
| [ | |
| 13.4, | |
| 21.5 | |
| ] | |
| ], | |
| "unseen_minus_seen_gain": [ | |
| 9.0, | |
| [ | |
| 3.4, | |
| 14.5 | |
| ] | |
| ], | |
| "unseen_minus_seen_gain_task_matched": [ | |
| 8.2, | |
| [ | |
| 1.5, | |
| 14.7 | |
| ] | |
| ], | |
| "unseen_minus_seen_gain_noul_only": [ | |
| 11.0, | |
| [ | |
| -0.9, | |
| 22.6 | |
| ] | |
| ], | |
| "unseen_minus_seen_gain_choice_only": [ | |
| 5.4, | |
| [ | |
| -0.9, | |
| 11.9 | |
| ] | |
| ] | |
| } | |
| }, | |
| { | |
| "slug": "forest", | |
| "title": "Paired-bootstrap differences", | |
| "caption": "Differences in accuracy (and looped-chain rate), in points, with 95% percentile intervals from 10,000 paired bootstrap resamples of questions (same resample for every model, temperatures fixed); blue rows involve V2 + RL. Single training seed, so the intervals cover question sampling only; source: results/final_stats.json.", | |
| "key_numbers": { | |
| "Fast readout, 5,442 questions: V2 + RL \u2212 base": [ | |
| 9.79, | |
| [ | |
| 8.43, | |
| 11.15 | |
| ] | |
| ], | |
| "Fast readout, 5,442 questions: V2 + RL \u2212 v1": [ | |
| 3.0, | |
| [ | |
| 1.87, | |
| 4.15 | |
| ] | |
| ], | |
| "Fast readout, 5,442 questions: V2 \u2212 v1": [ | |
| 3.23, | |
| [ | |
| 2.11, | |
| 4.39 | |
| ] | |
| ], | |
| "Fast readout, 5,442 questions: V2 + RL \u2212 V2": [ | |
| -0.24, | |
| [ | |
| -0.5, | |
| 0.02 | |
| ] | |
| ], | |
| "Fast readout, 5,442 questions: RL step 40 \u2212 step 60": [ | |
| 0.29, | |
| [ | |
| 0.04, | |
| 0.55 | |
| ] | |
| ], | |
| "Thinking, 1,498 questions: V2 + RL \u2212 V2": [ | |
| 1.94, | |
| [ | |
| 0.33, | |
| 3.54 | |
| ] | |
| ], | |
| "Thinking, 1,498 questions: V2 + RL \u2212 base": [ | |
| -0.93, | |
| [ | |
| -2.6, | |
| 0.73 | |
| ] | |
| ], | |
| "Thinking, 1,498 questions: V2 \u2212 base": [ | |
| -2.87, | |
| [ | |
| -4.67, | |
| -1.0 | |
| ] | |
| ], | |
| "Thinking, 1,498 questions: RL step 40 \u2212 step 60": [ | |
| 0.47, | |
| [ | |
| -1.07, | |
| 2.0 | |
| ] | |
| ], | |
| "Thinking, 1,498 questions: looped chains, V2 + RL \u2212 V2": [ | |
| -4.87, | |
| [ | |
| -6.01, | |
| -3.81 | |
| ] | |
| ], | |
| "Routed 30%, 1,498 questions: V2 + RL \u2212 base": [ | |
| 8.81, | |
| [ | |
| 6.28, | |
| 10.75 | |
| ] | |
| ], | |
| "Routed 30%, 1,498 questions: V2 + RL \u2212 v1": [ | |
| 3.87, | |
| [ | |
| 1.67, | |
| 5.61 | |
| ] | |
| ], | |
| "Routed 30%, 1,498 questions: V2 + RL \u2212 V2": [ | |
| 0.07, | |
| [ | |
| -1.27, | |
| 1.13 | |
| ] | |
| ], | |
| "Routed 30%, 1,498 questions: RL step 40 \u2212 step 60": [ | |
| 0.2, | |
| [ | |
| -1.07, | |
| 1.4 | |
| ] | |
| ], | |
| "Votes mixed in \u2212 gold only (V2 ablations): all 5,442": [ | |
| -1.19, | |
| [ | |
| -2.13, | |
| -0.28 | |
| ] | |
| ], | |
| "Votes mixed in \u2212 gold only (V2 ablations): in-distribution (1,125)": [ | |
| 1.51, | |
| [ | |
| -0.44, | |
| 3.38 | |
| ] | |
| ], | |
| "Votes mixed in \u2212 gold only (V2 ablations): out-of-distribution (4,317)": [ | |
| -1.9, | |
| [ | |
| -2.94, | |
| -0.86 | |
| ] | |
| ], | |
| "Votes mixed in \u2212 gold only (V2 ablations): in-distribution \u2212 OOD effect": [ | |
| 3.41, | |
| [ | |
| 1.23, | |
| 5.54 | |
| ] | |
| ] | |
| } | |
| }, | |
| { | |
| "slug": "yesno", | |
| "title": "Yes-rate vs gold yes-rate", | |
| "caption": "Share of yes/no items the fast readout answers yes vs the gold yes share (our eval) or the target yes mass (S1MB); filled: per-type temperatures, open: plus one yes/no logit offset fitted on the 109 yes/no calibration items, notes: balanced accuracy (eval) or yes/no adjusted skill x100 (S1MB), without -> with the offset. Source: results/yesno_offset.json; S1MB targets from data/s1mb/items.jsonl.", | |
| "key_numbers": { | |
| "eval": { | |
| "base": { | |
| "yes_rate": 41.6, | |
| "yes_rate_offset": 45.0, | |
| "note": "bal. acc. 0.686 \u2192 0.704" | |
| }, | |
| "v1_joint": { | |
| "yes_rate": 58.4, | |
| "yes_rate_offset": 53.3, | |
| "note": "bal. acc. 0.786 \u2192 0.790" | |
| }, | |
| "rl60": { | |
| "yes_rate": 62.3, | |
| "yes_rate_offset": 52.3, | |
| "note": "bal. acc. 0.784 \u2192 0.836" | |
| }, | |
| "gold_yes": 50.4 | |
| }, | |
| "s1mb": { | |
| "base": { | |
| "yes_rate": 18.4, | |
| "yes_rate_offset": 20.6, | |
| "note": "yes/no skill 33.8 \u2192 36.1" | |
| }, | |
| "v1_joint": { | |
| "yes_rate": 28.1, | |
| "yes_rate_offset": 22.9, | |
| "note": "yes/no skill 50.1 \u2192 47.6" | |
| }, | |
| "rl60": { | |
| "yes_rate": 31.6, | |
| "yes_rate_offset": 25.0, | |
| "note": "yes/no skill 50.7 \u2192 49.9" | |
| }, | |
| "gold_yes": 27.5 | |
| } | |
| } | |
| } | |
| ] |
Xet Storage Details
- Size:
- 13.2 kB
- Xet hash:
- b1c865173acc1a109747f47dd897313dbc33357360bedbd8d18f13c298b1973d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.