{
 "what": "Manchego v3's release read: v3, Manchego v2.1 and the untrained Qwen3.5-4B scored in one runtime (bf16, one RTX 5090, padded batches of up to 8,192 tokens, the adapters applied to the base unmerged) on identical rows; aggregates only. JevBench numbers here are OUR runs on its 231 public decisions, not official JevBench scores.",
 "names": {
  "frozen (served)": "untrained Qwen3.5-4B under Manchego v2.1's served prompt policy (short prompt up to 26 options, state-first beyond)",
  "frozen (SemIf)": "untrained Qwen3.5-4B under the SemIf prompt (state-first beyond 16 options)",
  "Manchego v2.1 (served)": "Manchego v2.1 under its served prompt policy",
  "v3m100b (SemIf)": "Manchego v3 (its adapter on the base, bf16) under the SemIf prompt (state-first beyond 16 options)"
 },
 "runtime_check_passed": true,
 "runtime_check": "v2.1 and the untrained base reproduced their archived records on the three card suites (tolerance |accuracy| <= 0.01, |cross-entropy| <= 0.02)",
 "intervals": "paired 95% percentile bootstraps; the resampling unit is named per suite ('interval')",
 "card": {
  "jevbench_public": {
   "rows": 231,
   "interval": "scenario groups resampled",
   "systems": {
    "frozen (served)": {
     "n": 231,
     "accuracy": 0.7748917748917749,
     "cross_entropy": 0.49286616487181245,
     "brier": 0.2866513305940945,
     "ece": 0.06231002798067052,
     "mean_confidence": 0.8204758784756144,
     "refused": 0
    },
    "frozen (SemIf)": {
     "n": 231,
     "accuracy": 0.8008658008658008,
     "cross_entropy": 0.4852084330739462,
     "brier": 0.25419321047505944,
     "ece": 0.0588310701620669,
     "mean_confidence": 0.8366482323997403,
     "refused": 0
    },
    "Manchego v2.1 (served)": {
     "n": 231,
     "accuracy": 0.8051948051948052,
     "cross_entropy": 0.4232890730630488,
     "brier": 0.24566343385005784,
     "ece": 0.07188272887054453,
     "mean_confidence": 0.8538814495170696,
     "refused": 0
    },
    "v3m100b (SemIf)": {
     "n": 231,
     "accuracy": 0.8571428571428571,
     "cross_entropy": 0.3260731101395454,
     "brier": 0.18328252671634565,
     "ece": 0.05534072233865077,
     "mean_confidence": 0.8536973872249488,
     "refused": 0
    }
   },
   "v3_minus_v21": {
    "accuracy": {
     "mean": 0.05194805194805195,
     "ci95": [
      0.004201680672268907,
      0.10222222222222223
     ]
    },
    "cross_entropy": {
     "mean": -0.09721596292350353,
     "ci95": [
      -0.17117279532265608,
      -0.03245969447592095
     ]
    }
   },
   "v3_minus_frozen_served": {
    "accuracy": {
     "mean": 0.08225108225108226,
     "ci95": [
      0.021645021645021644,
      0.14347826086956522
     ]
    },
    "cross_entropy": {
     "mean": -0.16679305473226716,
     "ci95": [
      -0.2671700196376057,
      -0.07280825498368075
     ]
    }
   }
  },
  "public8": {
   "rows": 1600,
   "interval": "items resampled within dataset",
   "systems": {
    "frozen (served)": {
     "n": 1600,
     "accuracy": 0.68625,
     "cross_entropy": 0.8430894766320378,
     "brier": 0.42806658962520244,
     "ece": 0.12246943241697027,
     "mean_confidence": 0.8087194324169702,
     "refused": 0
    },
    "frozen (SemIf)": {
     "n": 1600,
     "accuracy": 0.725,
     "cross_entropy": 0.8471491321273774,
     "brier": 0.3999620136837791,
     "ece": 0.12153813578564186,
     "mean_confidence": 0.8439632063936646,
     "refused": 0
    },
    "Manchego v2.1 (served)": {
     "n": 1600,
     "accuracy": 0.731875,
     "cross_entropy": 0.6837753863639332,
     "brier": 0.3671370658953875,
     "ece": 0.05316138522715013,
     "mean_confidence": 0.7847182733650019,
     "refused": 0
    },
    "v3m100b (SemIf)": {
     "n": 1600,
     "accuracy": 0.753125,
     "cross_entropy": 0.6085345175694903,
     "brier": 0.3268992227572392,
     "ece": 0.05543283372305037,
     "mean_confidence": 0.8085578337230505,
     "refused": 0
    }
   },
   "v3_minus_v21": {
    "accuracy": {
     "mean": 0.02125,
     "ci95": [
      0.001875,
      0.039375
     ]
    },
    "cross_entropy": {
     "mean": -0.07524086879444299,
     "ci95": [
      -0.10700733270210035,
      -0.04398069430837902
     ]
    }
   },
   "v3_minus_frozen_served": {
    "accuracy": {
     "mean": 0.066875,
     "ci95": [
      0.04625,
      0.08625
     ]
    },
    "cross_entropy": {
     "mean": -0.23455495906254767,
     "ci95": [
      -0.27699129955904445,
      -0.19314965712030813
     ]
    }
   }
  },
  "transfer": {
   "rows": 1561,
   "interval": "the 27 source-clean tasks (v2.1 card), tasks resampled",
   "systems": {
    "frozen (served)": {
     "n": 1561,
     "accuracy": 0.6777706598334401,
     "cross_entropy": 0.9189782184185465,
     "brier": 0.43554421632973334,
     "ece": 0.10705508985338476,
     "mean_confidence": 0.7840551579972183,
     "refused": 0
    },
    "frozen (SemIf)": {
     "n": 1561,
     "accuracy": 0.6777706598334401,
     "cross_entropy": 1.055959778577908,
     "brier": 0.46561897696066334,
     "ece": 0.11924407258759004,
     "mean_confidence": 0.7967923988846295,
     "refused": 0
    },
    "Manchego v2.1 (served)": {
     "n": 1561,
     "accuracy": 0.7014734144778988,
     "cross_entropy": 0.9285998666239199,
     "brier": 0.42428692850163635,
     "ece": 0.08289752004742494,
     "mean_confidence": 0.7837359571214004,
     "refused": 0
    },
    "v3m100b (SemIf)": {
     "n": 1561,
     "accuracy": 0.7085201793721974,
     "cross_entropy": 0.9408154991498171,
     "brier": 0.4318082093390665,
     "ece": 0.12920627298362403,
     "mean_confidence": 0.8292848345361473,
     "refused": 0
    }
   },
   "v3_minus_v21": {
    "accuracy": {
     "mean": 0.007046764894298526,
     "ci95": [
      -0.01585288522511097,
      0.03132992327365729
     ]
    },
    "cross_entropy": {
     "mean": 0.012215632525897134,
     "ci95": [
      -0.20211863222827312,
      0.15664732848488094
     ]
    }
   },
   "v3_minus_frozen_served": {
    "accuracy": {
     "mean": 0.03074951953875721,
     "ci95": [
      0.0019157088122605363,
      0.06482670089858793
     ]
    },
    "cross_entropy": {
     "mean": 0.02183728073127055,
     "ci95": [
      -0.08835037289694243,
      0.15047515066869643
     ]
    }
   }
  }
 },
 "sealed": {
  "confirm_b": {
   "what": "24 Natural Instructions tasks from 13 source clusters no Manchego model trained on, read once; the contrast is the mean over tasks of the per-task accuracy difference, source clusters resampled; 'systems' are pooled over rows",
   "claim": {
    "v3_minus_v21": {
     "tasks": 24,
     "clusters": 13,
     "rows": 5397,
     "accuracy": {
      "mean": 0.04227827381234498,
      "ci95": [
       -0.017932316870179994,
       0.1068637823646576
      ]
     },
     "cross_entropy": {
      "mean": 0.0024332706544883936,
      "ci95": [
       -0.10795744285557755,
       0.1274440740433659
      ]
     }
    },
    "v3_minus_frozen_served": {
     "tasks": 24,
     "clusters": 13,
     "rows": 5397,
     "accuracy": {
      "mean": 0.05955055602143098,
      "ci95": [
       -0.019663513011422513,
       0.14306252573617587
      ]
     },
     "cross_entropy": {
      "mean": -0.055827519937997915,
      "ci95": [
       -0.15976968646042974,
       0.07125067495963548
      ]
     }
    },
    "systems": {
     "frozen (served)": {
      "n": 5397,
      "accuracy": 0.6409116175653141,
      "cross_entropy": 0.9178368709979563,
      "brier": 0.48264586072708016,
      "ece": 0.15134806828254388,
      "mean_confidence": 0.7922596858478579,
      "refused": 0
     },
     "frozen (SemIf)": {
      "n": 5397,
      "accuracy": 0.6740781915879193,
      "cross_entropy": 0.9107246561067318,
      "brier": 0.46026264250869503,
      "ece": 0.1475488337401401,
      "mean_confidence": 0.8213640799547319,
      "refused": 0
     },
     "Manchego v2.1 (served)": {
      "n": 5397,
      "accuracy": 0.6588845654993515,
      "cross_entropy": 0.8486214459422593,
      "brier": 0.46015495267198037,
      "ece": 0.09823534911045727,
      "mean_confidence": 0.7571199146098088,
      "refused": 0
     },
     "v3m100b (SemIf)": {
      "n": 5397,
      "accuracy": 0.6950157494904576,
      "cross_entropy": 0.8607399041168844,
      "brier": 0.44547195424681324,
      "ece": 0.14094955334255857,
      "mean_confidence": 0.8359653028330162,
      "refused": 0
     }
    }
   }
  },
  "rep_sealed": {
   "what": "fresh out-of-distribution draws of four of the project's own computed families (policy decisions, lookup chains, temporal, answer adequacy), 400 instances, read once; instances resampled",
   "rows": 1193,
   "instances": 400,
   "systems": {
    "frozen (served)": {
     "n": 1193,
     "accuracy": 0.4450963956412406,
     "cross_entropy": 1.1611160039257353,
     "brier": 0.6460967922428869,
     "ece": 0.1405380034173992,
     "mean_confidence": 0.583342473485591,
     "refused": 0
    },
    "frozen (SemIf)": {
     "n": 1193,
     "accuracy": 0.5079631181894384,
     "cross_entropy": 1.0912352055652879,
     "brier": 0.5978876795541787,
     "ece": 0.07519011377082045,
     "mean_confidence": 0.5831532319602589,
     "refused": 0
    },
    "Manchego v2.1 (served)": {
     "n": 1193,
     "accuracy": 0.7946353730092205,
     "cross_entropy": 0.4659314334385817,
     "brier": 0.2733922531947281,
     "ece": 0.051271641242572615,
     "mean_confidence": 0.8437516230506135,
     "refused": 0
    },
    "v3m100b (SemIf)": {
     "n": 1193,
     "accuracy": 0.7619446772841576,
     "cross_entropy": 0.5777628650382641,
     "brier": 0.3190441194412629,
     "ece": 0.0495448301892173,
     "mean_confidence": 0.8004757787551626,
     "refused": 0
    }
   },
   "v3_minus_v21": {
    "accuracy": {
     "mean": -0.03269069572506287,
     "ci95": [
      -0.060085836909871244,
      -0.005106382978723404
     ]
    },
    "cross_entropy": {
     "mean": 0.11183143159968244,
     "ci95": [
      0.04482470675698149,
      0.18231048096005373
     ]
    }
   },
   "v3_minus_frozen_served": {
    "accuracy": {
     "mean": 0.316848281642917,
     "ci95": [
      0.2802013422818792,
      0.35313807531380753
     ]
    },
    "cross_entropy": {
     "mean": -0.5833531388874713,
     "ci95": [
      -0.6449727284665607,
      -0.519783716913897
     ]
    }
   }
  }
 },
 "flags": {
  "runtime_check_passed": true,
  "jevbench_v3_below_v21_point": false,
  "resolved_below_v21": [
   "rep_sealed"
  ]
 }
}