| { |
| "what": "Manchego v3's release read: v3, Manchego v2.1 and the untrained Qwen3.5-4B scored in one runtime (bf16, one RTX 5090, padded batches of up to 8,192 tokens, the adapters applied to the base unmerged) on identical rows; aggregates only. JevBench numbers here are OUR runs on its 231 public decisions, not official JevBench scores.", |
| "names": { |
| "frozen (served)": "untrained Qwen3.5-4B under Manchego v2.1's served prompt policy (short prompt up to 26 options, state-first beyond)", |
| "frozen (SemIf)": "untrained Qwen3.5-4B under the SemIf prompt (state-first beyond 16 options)", |
| "Manchego v2.1 (served)": "Manchego v2.1 under its served prompt policy", |
| "v3m100b (SemIf)": "Manchego v3 (its adapter on the base, bf16) under the SemIf prompt (state-first beyond 16 options)" |
| }, |
| "runtime_check_passed": true, |
| "runtime_check": "v2.1 and the untrained base reproduced their archived records on the three card suites (tolerance |accuracy| <= 0.01, |cross-entropy| <= 0.02)", |
| "intervals": "paired 95% percentile bootstraps; the resampling unit is named per suite ('interval')", |
| "card": { |
| "jevbench_public": { |
| "rows": 231, |
| "interval": "scenario groups resampled", |
| "systems": { |
| "frozen (served)": { |
| "n": 231, |
| "accuracy": 0.7748917748917749, |
| "cross_entropy": 0.49286616487181245, |
| "brier": 0.2866513305940945, |
| "ece": 0.06231002798067052, |
| "mean_confidence": 0.8204758784756144, |
| "refused": 0 |
| }, |
| "frozen (SemIf)": { |
| "n": 231, |
| "accuracy": 0.8008658008658008, |
| "cross_entropy": 0.4852084330739462, |
| "brier": 0.25419321047505944, |
| "ece": 0.0588310701620669, |
| "mean_confidence": 0.8366482323997403, |
| "refused": 0 |
| }, |
| "Manchego v2.1 (served)": { |
| "n": 231, |
| "accuracy": 0.8051948051948052, |
| "cross_entropy": 0.4232890730630488, |
| "brier": 0.24566343385005784, |
| "ece": 0.07188272887054453, |
| "mean_confidence": 0.8538814495170696, |
| "refused": 0 |
| }, |
| "v3m100b (SemIf)": { |
| "n": 231, |
| "accuracy": 0.8571428571428571, |
| "cross_entropy": 0.3260731101395454, |
| "brier": 0.18328252671634565, |
| "ece": 0.05534072233865077, |
| "mean_confidence": 0.8536973872249488, |
| "refused": 0 |
| } |
| }, |
| "v3_minus_v21": { |
| "accuracy": { |
| "mean": 0.05194805194805195, |
| "ci95": [ |
| 0.004201680672268907, |
| 0.10222222222222223 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.09721596292350353, |
| "ci95": [ |
| -0.17117279532265608, |
| -0.03245969447592095 |
| ] |
| } |
| }, |
| "v3_minus_frozen_served": { |
| "accuracy": { |
| "mean": 0.08225108225108226, |
| "ci95": [ |
| 0.021645021645021644, |
| 0.14347826086956522 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.16679305473226716, |
| "ci95": [ |
| -0.2671700196376057, |
| -0.07280825498368075 |
| ] |
| } |
| } |
| }, |
| "public8": { |
| "rows": 1600, |
| "interval": "items resampled within dataset", |
| "systems": { |
| "frozen (served)": { |
| "n": 1600, |
| "accuracy": 0.68625, |
| "cross_entropy": 0.8430894766320378, |
| "brier": 0.42806658962520244, |
| "ece": 0.12246943241697027, |
| "mean_confidence": 0.8087194324169702, |
| "refused": 0 |
| }, |
| "frozen (SemIf)": { |
| "n": 1600, |
| "accuracy": 0.725, |
| "cross_entropy": 0.8471491321273774, |
| "brier": 0.3999620136837791, |
| "ece": 0.12153813578564186, |
| "mean_confidence": 0.8439632063936646, |
| "refused": 0 |
| }, |
| "Manchego v2.1 (served)": { |
| "n": 1600, |
| "accuracy": 0.731875, |
| "cross_entropy": 0.6837753863639332, |
| "brier": 0.3671370658953875, |
| "ece": 0.05316138522715013, |
| "mean_confidence": 0.7847182733650019, |
| "refused": 0 |
| }, |
| "v3m100b (SemIf)": { |
| "n": 1600, |
| "accuracy": 0.753125, |
| "cross_entropy": 0.6085345175694903, |
| "brier": 0.3268992227572392, |
| "ece": 0.05543283372305037, |
| "mean_confidence": 0.8085578337230505, |
| "refused": 0 |
| } |
| }, |
| "v3_minus_v21": { |
| "accuracy": { |
| "mean": 0.02125, |
| "ci95": [ |
| 0.001875, |
| 0.039375 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.07524086879444299, |
| "ci95": [ |
| -0.10700733270210035, |
| -0.04398069430837902 |
| ] |
| } |
| }, |
| "v3_minus_frozen_served": { |
| "accuracy": { |
| "mean": 0.066875, |
| "ci95": [ |
| 0.04625, |
| 0.08625 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.23455495906254767, |
| "ci95": [ |
| -0.27699129955904445, |
| -0.19314965712030813 |
| ] |
| } |
| } |
| }, |
| "transfer": { |
| "rows": 1561, |
| "interval": "the 27 source-clean tasks (v2.1 card), tasks resampled", |
| "systems": { |
| "frozen (served)": { |
| "n": 1561, |
| "accuracy": 0.6777706598334401, |
| "cross_entropy": 0.9189782184185465, |
| "brier": 0.43554421632973334, |
| "ece": 0.10705508985338476, |
| "mean_confidence": 0.7840551579972183, |
| "refused": 0 |
| }, |
| "frozen (SemIf)": { |
| "n": 1561, |
| "accuracy": 0.6777706598334401, |
| "cross_entropy": 1.055959778577908, |
| "brier": 0.46561897696066334, |
| "ece": 0.11924407258759004, |
| "mean_confidence": 0.7967923988846295, |
| "refused": 0 |
| }, |
| "Manchego v2.1 (served)": { |
| "n": 1561, |
| "accuracy": 0.7014734144778988, |
| "cross_entropy": 0.9285998666239199, |
| "brier": 0.42428692850163635, |
| "ece": 0.08289752004742494, |
| "mean_confidence": 0.7837359571214004, |
| "refused": 0 |
| }, |
| "v3m100b (SemIf)": { |
| "n": 1561, |
| "accuracy": 0.7085201793721974, |
| "cross_entropy": 0.9408154991498171, |
| "brier": 0.4318082093390665, |
| "ece": 0.12920627298362403, |
| "mean_confidence": 0.8292848345361473, |
| "refused": 0 |
| } |
| }, |
| "v3_minus_v21": { |
| "accuracy": { |
| "mean": 0.007046764894298526, |
| "ci95": [ |
| -0.01585288522511097, |
| 0.03132992327365729 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": 0.012215632525897134, |
| "ci95": [ |
| -0.20211863222827312, |
| 0.15664732848488094 |
| ] |
| } |
| }, |
| "v3_minus_frozen_served": { |
| "accuracy": { |
| "mean": 0.03074951953875721, |
| "ci95": [ |
| 0.0019157088122605363, |
| 0.06482670089858793 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": 0.02183728073127055, |
| "ci95": [ |
| -0.08835037289694243, |
| 0.15047515066869643 |
| ] |
| } |
| } |
| } |
| }, |
| "sealed": { |
| "confirm_b": { |
| "what": "24 Natural Instructions tasks from 13 source clusters no Manchego model trained on, read once; the contrast is the mean over tasks of the per-task accuracy difference, source clusters resampled; 'systems' are pooled over rows", |
| "claim": { |
| "v3_minus_v21": { |
| "tasks": 24, |
| "clusters": 13, |
| "rows": 5397, |
| "accuracy": { |
| "mean": 0.04227827381234498, |
| "ci95": [ |
| -0.017932316870179994, |
| 0.1068637823646576 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": 0.0024332706544883936, |
| "ci95": [ |
| -0.10795744285557755, |
| 0.1274440740433659 |
| ] |
| } |
| }, |
| "v3_minus_frozen_served": { |
| "tasks": 24, |
| "clusters": 13, |
| "rows": 5397, |
| "accuracy": { |
| "mean": 0.05955055602143098, |
| "ci95": [ |
| -0.019663513011422513, |
| 0.14306252573617587 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.055827519937997915, |
| "ci95": [ |
| -0.15976968646042974, |
| 0.07125067495963548 |
| ] |
| } |
| }, |
| "systems": { |
| "frozen (served)": { |
| "n": 5397, |
| "accuracy": 0.6409116175653141, |
| "cross_entropy": 0.9178368709979563, |
| "brier": 0.48264586072708016, |
| "ece": 0.15134806828254388, |
| "mean_confidence": 0.7922596858478579, |
| "refused": 0 |
| }, |
| "frozen (SemIf)": { |
| "n": 5397, |
| "accuracy": 0.6740781915879193, |
| "cross_entropy": 0.9107246561067318, |
| "brier": 0.46026264250869503, |
| "ece": 0.1475488337401401, |
| "mean_confidence": 0.8213640799547319, |
| "refused": 0 |
| }, |
| "Manchego v2.1 (served)": { |
| "n": 5397, |
| "accuracy": 0.6588845654993515, |
| "cross_entropy": 0.8486214459422593, |
| "brier": 0.46015495267198037, |
| "ece": 0.09823534911045727, |
| "mean_confidence": 0.7571199146098088, |
| "refused": 0 |
| }, |
| "v3m100b (SemIf)": { |
| "n": 5397, |
| "accuracy": 0.6950157494904576, |
| "cross_entropy": 0.8607399041168844, |
| "brier": 0.44547195424681324, |
| "ece": 0.14094955334255857, |
| "mean_confidence": 0.8359653028330162, |
| "refused": 0 |
| } |
| } |
| } |
| }, |
| "rep_sealed": { |
| "what": "fresh out-of-distribution draws of four of the project's own computed families (policy decisions, lookup chains, temporal, answer adequacy), 400 instances, read once; instances resampled", |
| "rows": 1193, |
| "instances": 400, |
| "systems": { |
| "frozen (served)": { |
| "n": 1193, |
| "accuracy": 0.4450963956412406, |
| "cross_entropy": 1.1611160039257353, |
| "brier": 0.6460967922428869, |
| "ece": 0.1405380034173992, |
| "mean_confidence": 0.583342473485591, |
| "refused": 0 |
| }, |
| "frozen (SemIf)": { |
| "n": 1193, |
| "accuracy": 0.5079631181894384, |
| "cross_entropy": 1.0912352055652879, |
| "brier": 0.5978876795541787, |
| "ece": 0.07519011377082045, |
| "mean_confidence": 0.5831532319602589, |
| "refused": 0 |
| }, |
| "Manchego v2.1 (served)": { |
| "n": 1193, |
| "accuracy": 0.7946353730092205, |
| "cross_entropy": 0.4659314334385817, |
| "brier": 0.2733922531947281, |
| "ece": 0.051271641242572615, |
| "mean_confidence": 0.8437516230506135, |
| "refused": 0 |
| }, |
| "v3m100b (SemIf)": { |
| "n": 1193, |
| "accuracy": 0.7619446772841576, |
| "cross_entropy": 0.5777628650382641, |
| "brier": 0.3190441194412629, |
| "ece": 0.0495448301892173, |
| "mean_confidence": 0.8004757787551626, |
| "refused": 0 |
| } |
| }, |
| "v3_minus_v21": { |
| "accuracy": { |
| "mean": -0.03269069572506287, |
| "ci95": [ |
| -0.060085836909871244, |
| -0.005106382978723404 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": 0.11183143159968244, |
| "ci95": [ |
| 0.04482470675698149, |
| 0.18231048096005373 |
| ] |
| } |
| }, |
| "v3_minus_frozen_served": { |
| "accuracy": { |
| "mean": 0.316848281642917, |
| "ci95": [ |
| 0.2802013422818792, |
| 0.35313807531380753 |
| ] |
| }, |
| "cross_entropy": { |
| "mean": -0.5833531388874713, |
| "ci95": [ |
| -0.6449727284665607, |
| -0.519783716913897 |
| ] |
| } |
| } |
| } |
| }, |
| "flags": { |
| "runtime_check_passed": true, |
| "jevbench_v3_below_v21_point": false, |
| "resolved_below_v21": [ |
| "rep_sealed" |
| ] |
| } |
| } |
|
|
