{
 "benchmark": "RuBench v1",
 "split": "dev",
 "n_answers": 200,
 "n_criteria": 604,
 "built_at": "2026-11-04T00:15:58",
 "conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), no JSON schema enforcement, invalid JSON counts as wrong. RuBio: SYSTEM_PROMPT only; base and general models: SYSTEM_PROMPT+POLICY. RuBio and Qwen base: temperature 0, max_tokens 1024, thinking off. General models: vendor CLI at the listed reasoning effort, one fixed run.",
 "panel": {
  "round1": [
   "gpt",
   "opus",
   "grok"
  ],
  "round2": [
   "gpt",
   "opus",
   "gemini"
  ],
  "student_answer_author": "gemini"
 },
 "caveats": [
  "GPT and Opus were round-1 gold panelists: the r1-3/3 tier (521 of 604 dev criteria) contains their own blind judgments, so their scores include self-agreement and are upper bounds.",
  "Round-2 majority (r2-2/3) items include GPT, Opus and Gemini votes; see criterion_acc_excl_r2_2of3.",
  "Gemini wrote the synthetic student answers with intended labels; reported as reference only (DESIGN 4-1).",
  "Grok 4.7 (round-1 panelist) was not evaluated as a model.",
  "RuBio-27B is the RuIso-v6 retrain (main since 2026-11-04), run on a RunPod A100 Pod with vLLM 0.32.0 under the same generation settings; RuBio-4B v2 was run on RunPod Serverless (worker-v1-vllm, vLLM 0.32.0). The decision to replace the 27B main weights used this same dev split."
 ],
 "models": {
  "4b": {
   "key": "4b",
   "label": "RuBio-4B v2",
   "hf": "mulkzz/RuBio-4B@ba9e00c8",
   "role": "finetuned",
   "size": "4B",
   "name": "RuBio-4B v2",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 0,
   "criterion_acc": {
    "mean": 0.7716666666666667,
    "ci95": [
     0.7339481076683567,
     0.8093852256649767
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.79875,
    "ci95": [
     0.759669235815335,
     0.837830764184665
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 467,
    "n": 604
   },
   "misconception": {
    "k": 106,
    "n": 133,
    "p": 0.7969924812030075,
    "ci95": [
     0.7207474553277116,
     0.8565630300901761
    ]
   },
   "misconception_with_id": {
    "k": 65,
    "n": 79,
    "p": 0.8227848101265823,
    "ci95": [
     0.7242382665030478,
     0.8913956243248582
    ]
   },
   "json_valid": {
    "k": 200,
    "n": 200,
    "p": 1.0,
    "ci95": [
     0.981154748802476,
     1.0
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 156,
     "n": 206,
     "p": 0.7572815533980582,
     "ci95": [
      0.6943735885025178,
      0.8107697158073134
     ]
    },
    "II": {
     "k": 154,
     "n": 200,
     "p": 0.77,
     "ci95": [
      0.7069170932155301,
      0.822906471137807
     ]
    },
    "III": {
     "k": 157,
     "n": 198,
     "p": 0.7929292929292929,
     "ci95": [
      0.7311760804230768,
      0.8435324539592243
     ]
    }
   },
   "other_issues": {
    "k": 153,
    "n": 200,
    "p": 0.765,
    "ci95": [
     0.7015864484736013,
     0.8184255683917109
    ]
   },
   "answer_status": {
    "k": 124,
    "n": 200,
    "p": 0.62,
    "ci95": [
     0.5510673223171103,
     0.684409817395484
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 431,
     "n": 521,
     "p": 0.8272552783109405,
     "ci95": [
      0.7924302993782009,
      0.8572897340091328
     ]
    },
    "r2-3/3": {
     "k": 10,
     "n": 23,
     "p": 0.43478260869565216,
     "ci95": [
      0.2563466486442807,
      0.6318858887246053
     ]
    },
    "r2-2/3": {
     "k": 23,
     "n": 51,
     "p": 0.45098039215686275,
     "ci95": [
      0.32267542174059904,
      0.5861526522832785
     ]
    },
    "human": {
     "k": 3,
     "n": 9,
     "p": 0.3333333333333333,
     "ci95": [
      0.12058406396498972,
      0.6457973346907151
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 10,
     "n": 53,
     "p": 0.18867924528301888,
     "ci95": [
      0.10585337627628057,
      0.3135843061386674
     ]
    },
    "no_error_specificity": {
     "k": 143,
     "n": 147,
     "p": 0.9727891156462585,
     "ci95": [
      0.9321289915585997,
      0.9893684235398896
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 121,
    "contradicted->missing": 5,
    "contradicted->supported": 3,
    "contradicted->unclear": 4,
    "missing->contradicted": 12,
    "missing->missing": 92,
    "missing->supported": 18,
    "missing->unclear": 20,
    "supported->contradicted": 14,
    "supported->missing": 9,
    "supported->supported": 232,
    "supported->unclear": 28,
    "unclear->contradicted": 8,
    "unclear->missing": 7,
    "unclear->supported": 9,
    "unclear->unclear": 22
   },
   "cli_seconds_p50": null
  },
  "qwen4b": {
   "key": "qwen4b",
   "label": "Qwen3.5-4B (base)",
   "hf": "Qwen/Qwen3.5-4B",
   "role": "base",
   "size": "4B",
   "name": "Qwen3.5-4B (기본)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 0,
   "criterion_acc": {
    "mean": 0.7091666666666667,
    "ci95": [
     0.673031876586196,
     0.7453014567471374
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.7454166666666667,
    "ci95": [
     0.7095199602308042,
     0.7813133731025292
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 427,
    "n": 604
   },
   "misconception": {
    "k": 78,
    "n": 133,
    "p": 0.5864661654135338,
    "ci95": [
     0.501491500687543,
     0.666586235573361
    ]
   },
   "misconception_with_id": {
    "k": 55,
    "n": 79,
    "p": 0.6962025316455697,
    "ci95": [
     0.5876536225817802,
     0.7865552130194958
    ]
   },
   "json_valid": {
    "k": 200,
    "n": 200,
    "p": 1.0,
    "ci95": [
     0.981154748802476,
     1.0
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 144,
     "n": 206,
     "p": 0.6990291262135923,
     "ci95": [
      0.633218801872325,
      0.7575524335371671
     ]
    },
    "II": {
     "k": 142,
     "n": 200,
     "p": 0.71,
     "ci95": [
      0.6436253597813291,
      0.7684596347157107
     ]
    },
    "III": {
     "k": 141,
     "n": 198,
     "p": 0.7121212121212122,
     "ci95": [
      0.6454906562919455,
      0.770677592743514
     ]
    }
   },
   "other_issues": {
    "k": 150,
    "n": 200,
    "p": 0.75,
    "ci95": [
     0.6856591553575013,
     0.8049182190437367
    ]
   },
   "answer_status": {
    "k": 89,
    "n": 200,
    "p": 0.445,
    "ci95": [
     0.37780614202227114,
     0.5142668356094566
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 402,
     "n": 521,
     "p": 0.7715930902111324,
     "ci95": [
      0.7336347625660318,
      0.8055757050252159
     ]
    },
    "r2-3/3": {
     "k": 4,
     "n": 23,
     "p": 0.17391304347826086,
     "ci95": [
      0.06978666088359675,
      0.37137602596083263
     ]
    },
    "r2-2/3": {
     "k": 15,
     "n": 51,
     "p": 0.29411764705882354,
     "ci95": [
      0.1870874425221019,
      0.4299904683781837
     ]
    },
    "human": {
     "k": 6,
     "n": 9,
     "p": 0.6666666666666666,
     "ci95": [
      0.35420266530928485,
      0.8794159360350102
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 4,
     "n": 53,
     "p": 0.07547169811320754,
     "ci95": [
      0.029739608356547,
      0.17858450402747283
     ]
    },
    "no_error_specificity": {
     "k": 146,
     "n": 147,
     "p": 0.9931972789115646,
     "ci95": [
      0.9624761437110765,
      0.9987981382261968
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 100,
    "contradicted->missing": 20,
    "contradicted->supported": 10,
    "contradicted->unclear": 3,
    "missing->contradicted": 22,
    "missing->missing": 79,
    "missing->supported": 37,
    "missing->unclear": 4,
    "supported->contradicted": 12,
    "supported->missing": 16,
    "supported->supported": 244,
    "supported->unclear": 11,
    "unclear->contradicted": 7,
    "unclear->missing": 9,
    "unclear->supported": 26,
    "unclear->unclear": 4
   },
   "cli_seconds_p50": null
  },
  "27b": {
   "key": "27b",
   "src": "27b-v6-s3407",
   "label": "RuBio-27B",
   "hf": "mulkzz/RuBio-27B@c1cfefdb",
   "role": "finetuned",
   "size": "27B",
   "name": "RuBio-27B RuIso-v6 (시드 3407)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 0,
   "criterion_acc": {
    "mean": 0.86875,
    "ci95": [
     0.8381287620959553,
     0.8993712379040447
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.9,
    "ci95": [
     0.8714503967862466,
     0.9285496032137535
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 524,
    "n": 604
   },
   "misconception": {
    "k": 118,
    "n": 133,
    "p": 0.8872180451127819,
    "ci95": [
     0.8222450524506153,
     0.9304508968916938
    ]
   },
   "misconception_with_id": {
    "k": 73,
    "n": 79,
    "p": 0.9240506329113924,
    "ci95": [
     0.8440460366216044,
     0.9647278983876056
    ]
   },
   "json_valid": {
    "k": 200,
    "n": 200,
    "p": 1.0,
    "ci95": [
     0.981154748802476,
     1.0
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 185,
     "n": 206,
     "p": 0.8980582524271845,
     "ci95": [
      0.8491894424118119,
      0.9323530284071725
     ]
    },
    "II": {
     "k": 166,
     "n": 200,
     "p": 0.83,
     "ci95": [
      0.7718412936135373,
      0.8757208405960968
     ]
    },
    "III": {
     "k": 173,
     "n": 198,
     "p": 0.8737373737373737,
     "ci95": [
      0.8202540537968194,
      0.9129947659323233
     ]
    }
   },
   "other_issues": {
    "k": 173,
    "n": 200,
    "p": 0.865,
    "ci95": [
     0.810708740295804,
     0.9055342263300035
    ]
   },
   "answer_status": {
    "k": 154,
    "n": 200,
    "p": 0.77,
    "ci95": [
     0.7069170932155301,
     0.822906471137807
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 483,
     "n": 521,
     "p": 0.927063339731286,
     "ci95": [
      0.9014725811198147,
      0.9464025357639493
     ]
    },
    "r2-3/3": {
     "k": 11,
     "n": 23,
     "p": 0.4782608695652174,
     "ci95": [
      0.29237189265791597,
      0.670372286465046
     ]
    },
    "r2-2/3": {
     "k": 23,
     "n": 51,
     "p": 0.45098039215686275,
     "ci95": [
      0.32267542174059904,
      0.5861526522832785
     ]
    },
    "human": {
     "k": 7,
     "n": 9,
     "p": 0.7777777777777778,
     "ci95": [
      0.45258959147532657,
      0.9367747440984988
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 46,
     "n": 53,
     "p": 0.8679245283018868,
     "ci95": [
      0.7515977273649117,
      0.9345213752356045
     ]
    },
    "no_error_specificity": {
     "k": 127,
     "n": 147,
     "p": 0.8639455782312925,
     "ci95": [
      0.7991850746700077,
      0.9101690506216352
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 127,
    "contradicted->missing": 4,
    "contradicted->supported": 1,
    "contradicted->unclear": 1,
    "missing->contradicted": 8,
    "missing->missing": 125,
    "missing->supported": 2,
    "missing->unclear": 7,
    "supported->contradicted": 4,
    "supported->missing": 20,
    "supported->supported": 253,
    "supported->unclear": 6,
    "unclear->contradicted": 4,
    "unclear->missing": 15,
    "unclear->supported": 8,
    "unclear->unclear": 19
   },
   "cli_seconds_p50": null
  },
  "qwen27b": {
   "key": "qwen27b",
   "label": "Qwen3.8-27B (base)",
   "hf": "Qwen/Qwen3.8-27B",
   "role": "base",
   "size": "27B",
   "name": "Qwen3.8-27B (기본)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 5,
   "criterion_acc": {
    "mean": 0.8179166666666667,
    "ci95": [
     0.7816876269997497,
     0.8541457063335838
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.8495833333333332,
    "ci95": [
     0.8142417703596521,
     0.8849248963070144
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 495,
    "n": 604
   },
   "misconception": {
    "k": 107,
    "n": 133,
    "p": 0.8045112781954887,
    "ci95": [
     0.7289698915278552,
     0.8629560492170677
    ]
   },
   "misconception_with_id": {
    "k": 68,
    "n": 79,
    "p": 0.8607594936708861,
    "ci95": [
     0.7676269042027796,
     0.9204345031931153
    ]
   },
   "json_valid": {
    "k": 195,
    "n": 200,
    "p": 0.975,
    "ci95": [
     0.9428217485145032,
     0.989275262847849
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 165,
     "n": 206,
     "p": 0.8009708737864077,
     "ci95": [
      0.7411592356541727,
      0.8497631203309128
     ]
    },
    "II": {
     "k": 167,
     "n": 200,
     "p": 0.835,
     "ci95": [
      0.777342313757747,
      0.8800313679399118
     ]
    },
    "III": {
     "k": 163,
     "n": 198,
     "p": 0.8232323232323232,
     "ci95": [
      0.764095715325512,
      0.8700654260618547
     ]
    }
   },
   "other_issues": {
    "k": 156,
    "n": 200,
    "p": 0.78,
    "ci95": [
     0.7176121367571497,
     0.8318345225722368
    ]
   },
   "answer_status": {
    "k": 120,
    "n": 200,
    "p": 0.6,
    "ci95": [
     0.530836862213719,
     0.6653940875467761
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 461,
     "n": 521,
     "p": 0.8848368522072937,
     "ci95": [
      0.854565276901407,
      0.9094749969646364
     ]
    },
    "r2-3/3": {
     "k": 5,
     "n": 23,
     "p": 0.21739130434782608,
     "ci95": [
      0.09663993783788177,
      0.4190343907606237
     ]
    },
    "r2-2/3": {
     "k": 24,
     "n": 51,
     "p": 0.47058823529411764,
     "ci95": [
      0.3405301441444071,
      0.6047667002699194
     ]
    },
    "human": {
     "k": 5,
     "n": 9,
     "p": 0.5555555555555556,
     "ci95": [
      0.26665173025527866,
      0.8112211368594864
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 29,
     "n": 53,
     "p": 0.5471698113207547,
     "ci95": [
      0.41454001247278066,
      0.6734239750401059
     ]
    },
    "no_error_specificity": {
     "k": 127,
     "n": 147,
     "p": 0.8639455782312925,
     "ci95": [
      0.7991850746700077,
      0.9101690506216352
     ]
    }
   },
   "confusion": {
    "contradicted->(없음)": 6,
    "contradicted->contradicted": 112,
    "contradicted->missing": 7,
    "contradicted->supported": 5,
    "contradicted->unclear": 3,
    "missing->(없음)": 8,
    "missing->contradicted": 3,
    "missing->missing": 95,
    "missing->supported": 20,
    "missing->unclear": 16,
    "supported->(없음)": 3,
    "supported->contradicted": 4,
    "supported->missing": 1,
    "supported->supported": 269,
    "supported->unclear": 6,
    "unclear->(없음)": 1,
    "unclear->missing": 3,
    "unclear->supported": 23,
    "unclear->unclear": 19
   },
   "cli_seconds_p50": 9.7
  },
  "gpt": {
   "key": "gpt",
   "label": "GPT-6 Astra (xhigh)",
   "hf": null,
   "role": "general",
   "size": null,
   "name": "GPT-6 Astra (xhigh)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 0,
   "criterion_acc": {
    "mean": 0.9116666666666667,
    "ci95": [
     0.8858637301001623,
     0.9374696032331712
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.9558333333333333,
    "ci95": [
     0.9362898427686828,
     0.9753768238979839
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 550,
    "n": 604
   },
   "misconception": {
    "k": 129,
    "n": 133,
    "p": 0.9699248120300752,
    "ci95": [
     0.9252225048958136,
     0.9882434530438822
    ]
   },
   "misconception_with_id": {
    "k": 78,
    "n": 79,
    "p": 0.9873417721518988,
    "ci95": [
     0.9317244628529116,
     0.9977619997696132
    ]
   },
   "json_valid": {
    "k": 200,
    "n": 200,
    "p": 1.0,
    "ci95": [
     0.981154748802476,
     1.0
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 184,
     "n": 206,
     "p": 0.8932038834951457,
     "ci95": [
      0.8436020225775144,
      0.928409442499775
     ]
    },
    "II": {
     "k": 185,
     "n": 200,
     "p": 0.925,
     "ci95": [
      0.8799564990407998,
      0.9540250374413048
     ]
    },
    "III": {
     "k": 181,
     "n": 198,
     "p": 0.9141414141414141,
     "ci95": [
      0.866814758905608,
      0.9457042034969555
     ]
    }
   },
   "other_issues": {
    "k": 177,
    "n": 200,
    "p": 0.885,
    "ci95": [
     0.8333529508707886,
     0.9221362057071179
    ]
   },
   "answer_status": {
    "k": 176,
    "n": 200,
    "p": 0.88,
    "ci95": [
     0.8276575999704266,
     0.9180200091194551
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 510,
     "n": 521,
     "p": 0.9788867562380038,
     "ci95": [
      0.9625927664904859,
      0.9881705668241167
     ]
    },
    "r2-3/3": {
     "k": 14,
     "n": 23,
     "p": 0.6086956521739131,
     "ci95": [
      0.4078556031551567,
      0.7784235012300337
     ]
    },
    "r2-2/3": {
     "k": 22,
     "n": 51,
     "p": 0.43137254901960786,
     "ci95": [
      0.30501259121603025,
      0.5673467124173983
     ]
    },
    "human": {
     "k": 4,
     "n": 9,
     "p": 0.4444444444444444,
     "ci95": [
      0.18877886314051356,
      0.7333482697447213
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 33,
     "n": 53,
     "p": 0.6226415094339622,
     "ci95": [
      0.48806923125215484,
      0.7406371362813504
     ]
    },
    "no_error_specificity": {
     "k": 144,
     "n": 147,
     "p": 0.9795918367346939,
     "ci95": [
      0.9417208980015789,
      0.9930354727098384
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 129,
    "contradicted->missing": 3,
    "contradicted->unclear": 1,
    "missing->contradicted": 1,
    "missing->missing": 135,
    "missing->supported": 2,
    "missing->unclear": 4,
    "supported->contradicted": 3,
    "supported->missing": 11,
    "supported->supported": 269,
    "unclear->contradicted": 2,
    "unclear->missing": 23,
    "unclear->supported": 4,
    "unclear->unclear": 17
   },
   "cli_seconds_p50": 575.6
  },
  "opus": {
   "key": "opus",
   "label": "Claude Opus 5.5 (xhigh)",
   "hf": null,
   "role": "general",
   "size": null,
   "name": "Claude Opus 5.5 (xhigh)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 0,
   "criterion_acc": {
    "mean": 0.9420833333333333,
    "ci95": [
     0.9203542491963614,
     0.9638124174703051
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.9608333333333333,
    "ci95": [
     0.9417409178385642,
     0.9799257488281025
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 570,
    "n": 604
   },
   "misconception": {
    "k": 127,
    "n": 133,
    "p": 0.9548872180451128,
    "ci95": [
     0.9050722874953598,
     0.9791627597902657
    ]
   },
   "misconception_with_id": {
    "k": 77,
    "n": 79,
    "p": 0.9746835443037974,
    "ci95": [
     0.9123143405749654,
     0.9930296165248962
    ]
   },
   "json_valid": {
    "k": 200,
    "n": 200,
    "p": 1.0,
    "ci95": [
     0.981154748802476,
     1.0
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 193,
     "n": 206,
     "p": 0.9368932038834952,
     "ci95": [
      0.895037892168079,
      0.9627526245844646
     ]
    },
    "II": {
     "k": 189,
     "n": 200,
     "p": 0.945,
     "ci95": [
      0.9042131036406227,
      0.9690146227935807
     ]
    },
    "III": {
     "k": 188,
     "n": 198,
     "p": 0.9494949494949495,
     "ci95": [
      0.9095419269028148,
      0.9723384103389922
     ]
    }
   },
   "other_issues": {
    "k": 195,
    "n": 200,
    "p": 0.975,
    "ci95": [
     0.9428217485145032,
     0.989275262847849
    ]
   },
   "answer_status": {
    "k": 184,
    "n": 200,
    "p": 0.92,
    "ci95": [
     0.8740106225357106,
     0.9501593664583693
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 510,
     "n": 521,
     "p": 0.9788867562380038,
     "ci95": [
      0.9625927664904859,
      0.9881705668241167
     ]
    },
    "r2-3/3": {
     "k": 18,
     "n": 23,
     "p": 0.782608695652174,
     "ci95": [
      0.5809656092393763,
      0.9033600621621182
     ]
    },
    "r2-2/3": {
     "k": 37,
     "n": 51,
     "p": 0.7254901960784313,
     "ci95": [
      0.5905337813556237,
      0.8288570781345395
     ]
    },
    "human": {
     "k": 5,
     "n": 9,
     "p": 0.5555555555555556,
     "ci95": [
      0.26665173025527866,
      0.8112211368594864
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 52,
     "n": 53,
     "p": 0.9811320754716981,
     "ci95": [
      0.9005711355345968,
      0.9966615370968476
     ]
    },
    "no_error_specificity": {
     "k": 143,
     "n": 147,
     "p": 0.9727891156462585,
     "ci95": [
      0.9321289915585997,
      0.9893684235398896
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 128,
    "contradicted->missing": 3,
    "contradicted->supported": 1,
    "contradicted->unclear": 1,
    "missing->contradicted": 1,
    "missing->missing": 123,
    "missing->supported": 7,
    "missing->unclear": 11,
    "supported->missing": 3,
    "supported->supported": 279,
    "supported->unclear": 1,
    "unclear->supported": 6,
    "unclear->unclear": 40
   },
   "cli_seconds_p50": 41.3
  },
  "gemini": {
   "key": "gemini",
   "label": "Gemini 4 Argon (High)",
   "hf": null,
   "role": "reference",
   "size": null,
   "name": "Gemini 4 Argon (High)",
   "answers": 200,
   "missing_output": 0,
   "json_invalid": 2,
   "criterion_acc": {
    "mean": 0.9125,
    "ci95": [
     0.8850333281496484,
     0.9399666718503515
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.9358333333333333,
    "ci95": [
     0.9098731921940174,
     0.9617934744726492
    ],
    "n_answers": 200
   },
   "criterion_micro": {
    "hit": 551,
    "n": 604
   },
   "misconception": {
    "k": 122,
    "n": 133,
    "p": 0.9172932330827067,
    "ci95": [
     0.8579662495837602,
     0.9531915210666896
    ]
   },
   "misconception_with_id": {
    "k": 75,
    "n": 79,
    "p": 0.9493670886075949,
    "ci95": [
     0.8769230243999817,
     0.980135921654554
    ]
   },
   "json_valid": {
    "k": 198,
    "n": 200,
    "p": 0.99,
    "ci95": [
     0.9642783183248235,
     0.997253335501603
    ]
   },
   "unit_criterion_micro": {
    "I": {
     "k": 187,
     "n": 206,
     "p": 0.9077669902912622,
     "ci95": [
      0.8604469235517486,
      0.9401575587506256
     ]
    },
    "II": {
     "k": 184,
     "n": 200,
     "p": 0.92,
     "ci95": [
      0.8740106225357106,
      0.9501593664583693
     ]
    },
    "III": {
     "k": 180,
     "n": 198,
     "p": 0.9090909090909091,
     "ci95": [
      0.8608883910901738,
      0.9417218034782121
     ]
    }
   },
   "other_issues": {
    "k": 190,
    "n": 200,
    "p": 0.95,
    "ci95": [
     0.9104219523342212,
     0.9726173215880072
    ]
   },
   "answer_status": {
    "k": 179,
    "n": 200,
    "p": 0.895,
    "ci95": [
     0.8448197976153835,
     0.9302924539385726
    ]
   },
   "gold_tier_criterion_acc": {
    "r1-3/3": {
     "k": 496,
     "n": 521,
     "p": 0.9520153550863724,
     "ci95": [
      0.9301245809465594,
      0.9672893068113121
     ]
    },
    "r2-3/3": {
     "k": 19,
     "n": 23,
     "p": 0.8260869565217391,
     "ci95": [
      0.6286239740391674,
      0.9302133391164032
     ]
    },
    "r2-2/3": {
     "k": 32,
     "n": 51,
     "p": 0.6274509803921569,
     "ci95": [
      0.49025241212438464,
      0.7467945954135337
     ]
    },
    "human": {
     "k": 4,
     "n": 9,
     "p": 0.4444444444444444,
     "ci95": [
      0.18877886314051356,
      0.7333482697447213
     ]
    }
   },
   "other_issues_split": {
    "has_error_recall": {
     "k": 45,
     "n": 53,
     "p": 0.8490566037735849,
     "ci95": [
      0.7294577896504801,
      0.9214757179448814
     ]
    },
    "no_error_specificity": {
     "k": 145,
     "n": 147,
     "p": 0.9863945578231292,
     "ci95": [
      0.9517544022691328,
      0.9962609240552124
     ]
    }
   },
   "confusion": {
    "contradicted->contradicted": 125,
    "contradicted->missing": 5,
    "contradicted->supported": 1,
    "contradicted->unclear": 2,
    "missing->(없음)": 2,
    "missing->missing": 125,
    "missing->supported": 10,
    "missing->unclear": 5,
    "supported->(없음)": 3,
    "supported->contradicted": 2,
    "supported->missing": 6,
    "supported->supported": 267,
    "supported->unclear": 5,
    "unclear->(없음)": 1,
    "unclear->contradicted": 1,
    "unclear->missing": 9,
    "unclear->supported": 1,
    "unclear->unclear": 34
   },
   "cli_seconds_p50": 87.9
  }
 },
 "paired": {
  "qwen4b-minus-4b": {
   "all": {
    "diff": -0.06250000000000001,
    "ci95": [
     -0.09929215430834135,
     -0.02570784569165868
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.05333333333333334,
    "ci95": [
     -0.0949195943790324,
     -0.011747072287634275
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "27b-minus-4b": {
   "all": {
    "diff": 0.09708333333333334,
    "ci95": [
     0.06123184381518337,
     0.1329348228514833
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.10125,
    "ci95": [
     0.06521017094468781,
     0.1372898290553122
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "qwen27b-minus-4b": {
   "all": {
    "diff": 0.04625,
    "ci95": [
     0.007906503441078273,
     0.08459349655892173
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.05083333333333334,
    "ci95": [
     0.008389559707535299,
     0.09327710695913138
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gpt-minus-4b": {
   "all": {
    "diff": 0.14,
    "ci95": [
     0.1028811816212637,
     0.17711881837873633
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.15708333333333335,
    "ci95": [
     0.11841813107448726,
     0.19574853559217945
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "opus-minus-4b": {
   "all": {
    "diff": 0.1704166666666667,
    "ci95": [
     0.1349501475307221,
     0.20588318580261128
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.16208333333333336,
    "ci95": [
     0.12777146352786023,
     0.19639520313880648
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gemini-minus-4b": {
   "all": {
    "diff": 0.14083333333333334,
    "ci95": [
     0.10129028178518129,
     0.18037638488148539
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.13708333333333333,
    "ci95": [
     0.0986490412756628,
     0.17551762539100385
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "4b-minus-qwen4b": {
   "all": {
    "diff": 0.06250000000000001,
    "ci95": [
     0.02570784569165868,
     0.09929215430834135
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.05333333333333334,
    "ci95": [
     0.011747072287634275,
     0.0949195943790324
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "27b-minus-qwen4b": {
   "all": {
    "diff": 0.15958333333333333,
    "ci95": [
     0.12437326122557398,
     0.1947934054410927
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.15458333333333335,
    "ci95": [
     0.11312043866157052,
     0.1960462280050962
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "qwen27b-minus-qwen4b": {
   "all": {
    "diff": 0.10875,
    "ci95": [
     0.06948238748995589,
     0.1480176125100441
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.10416666666666669,
    "ci95": [
     0.06221595810731039,
     0.14611737522602297
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gpt-minus-qwen4b": {
   "all": {
    "diff": 0.2025,
    "ci95": [
     0.16509784766297997,
     0.23990215233702006
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.21041666666666667,
    "ci95": [
     0.1727924563143645,
     0.24804087701896885
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "opus-minus-qwen4b": {
   "all": {
    "diff": 0.2329166666666667,
    "ci95": [
     0.1962971271528449,
     0.26953620618048846
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.21541666666666667,
    "ci95": [
     0.17708051297608593,
     0.25375282035724744
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gemini-minus-qwen4b": {
   "all": {
    "diff": 0.20333333333333337,
    "ci95": [
     0.16215851002079762,
     0.2445081566458691
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.19041666666666668,
    "ci95": [
     0.14755506490751477,
     0.23327826842581859
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "4b-minus-27b": {
   "all": {
    "diff": -0.09708333333333334,
    "ci95": [
     -0.1329348228514833,
     -0.06123184381518337
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.10125,
    "ci95": [
     -0.1372898290553122,
     -0.06521017094468781
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen4b-minus-27b": {
   "all": {
    "diff": -0.15958333333333333,
    "ci95": [
     -0.1947934054410927,
     -0.12437326122557398
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.15458333333333335,
    "ci95": [
     -0.1960462280050962,
     -0.11312043866157052
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen27b-minus-27b": {
   "all": {
    "diff": -0.05083333333333334,
    "ci95": [
     -0.09092183669301501,
     -0.010744829973651678
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.05041666666666667,
    "ci95": [
     -0.0903058270121205,
     -0.01052750632121284
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "gpt-minus-27b": {
   "all": {
    "diff": 0.04291666666666667,
    "ci95": [
     0.01078839936599886,
     0.07504493396733448
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.05583333333333334,
    "ci95": [
     0.02437694687489072,
     0.08728971979177597
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "opus-minus-27b": {
   "all": {
    "diff": 0.07333333333333333,
    "ci95": [
     0.04152834968415308,
     0.10513831698251358
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.060833333333333336,
    "ci95": [
     0.030981954682252844,
     0.09068471198441383
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gemini-minus-27b": {
   "all": {
    "diff": 0.04375,
    "ci95": [
     0.009717845060980758,
     0.07778215493901924
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.035833333333333335,
    "ci95": [
     0.001730887172231424,
     0.06993577949443525
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "4b-minus-qwen27b": {
   "all": {
    "diff": -0.04625,
    "ci95": [
     -0.08459349655892173,
     -0.007906503441078273
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.05083333333333334,
    "ci95": [
     -0.09327710695913138,
     -0.008389559707535299
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen4b-minus-qwen27b": {
   "all": {
    "diff": -0.10875,
    "ci95": [
     -0.1480176125100441,
     -0.06948238748995589
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.10416666666666669,
    "ci95": [
     -0.14611737522602297,
     -0.06221595810731039
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "27b-minus-qwen27b": {
   "all": {
    "diff": 0.05083333333333334,
    "ci95": [
     0.010744829973651678,
     0.09092183669301501
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.05041666666666667,
    "ci95": [
     0.01052750632121284,
     0.0903058270121205
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gpt-minus-qwen27b": {
   "all": {
    "diff": 0.09375,
    "ci95": [
     0.05730219163645278,
     0.13019780836354722
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.10625,
    "ci95": [
     0.07279390330302032,
     0.13970609669697967
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "opus-minus-qwen27b": {
   "all": {
    "diff": 0.12416666666666668,
    "ci95": [
     0.09197561056896073,
     0.15635772276437263
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.11125,
    "ci95": [
     0.07880409709964073,
     0.14369590290035927
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "gemini-minus-qwen27b": {
   "all": {
    "diff": 0.09458333333333334,
    "ci95": [
     0.0551882445624855,
     0.13397842210418118
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.08625,
    "ci95": [
     0.04819286835030037,
     0.12430713164969961
    ],
    "n": 200,
    "verdict": "개선"
   }
  },
  "4b-minus-gpt": {
   "all": {
    "diff": -0.14,
    "ci95": [
     -0.17711881837873633,
     -0.1028811816212637
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.15708333333333335,
    "ci95": [
     -0.19574853559217945,
     -0.11841813107448726
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen4b-minus-gpt": {
   "all": {
    "diff": -0.2025,
    "ci95": [
     -0.23990215233702006,
     -0.16509784766297997
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.21041666666666667,
    "ci95": [
     -0.24804087701896885,
     -0.1727924563143645
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "27b-minus-gpt": {
   "all": {
    "diff": -0.04291666666666667,
    "ci95": [
     -0.07504493396733448,
     -0.01078839936599886
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.05583333333333334,
    "ci95": [
     -0.08728971979177597,
     -0.02437694687489072
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen27b-minus-gpt": {
   "all": {
    "diff": -0.09375,
    "ci95": [
     -0.13019780836354722,
     -0.05730219163645278
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.10625,
    "ci95": [
     -0.13970609669697967,
     -0.07279390330302032
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "opus-minus-gpt": {
   "all": {
    "diff": 0.030416666666666668,
    "ci95": [
     0.004444457390938891,
     0.056388875942394445
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.005000000000000001,
    "ci95": [
     -0.015406897160101511,
     0.025406897160101513
    ],
    "n": 200,
    "verdict": "차이 없음"
   }
  },
  "gemini-minus-gpt": {
   "all": {
    "diff": 0.0008333333333333328,
    "ci95": [
     -0.029734282082786483,
     0.03140094874945315
    ],
    "n": 200,
    "verdict": "차이 없음"
   },
   "no_majority": {
    "diff": -0.02,
    "ci95": [
     -0.048200934523245254,
     0.008200934523245256
    ],
    "n": 200,
    "verdict": "차이 없음"
   }
  },
  "4b-minus-opus": {
   "all": {
    "diff": -0.1704166666666667,
    "ci95": [
     -0.20588318580261128,
     -0.1349501475307221
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.16208333333333336,
    "ci95": [
     -0.19639520313880648,
     -0.12777146352786023
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen4b-minus-opus": {
   "all": {
    "diff": -0.2329166666666667,
    "ci95": [
     -0.26953620618048846,
     -0.1962971271528449
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.21541666666666667,
    "ci95": [
     -0.25375282035724744,
     -0.17708051297608593
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "27b-minus-opus": {
   "all": {
    "diff": -0.07333333333333333,
    "ci95": [
     -0.10513831698251358,
     -0.04152834968415308
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.060833333333333336,
    "ci95": [
     -0.09068471198441383,
     -0.030981954682252844
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen27b-minus-opus": {
   "all": {
    "diff": -0.12416666666666668,
    "ci95": [
     -0.15635772276437263,
     -0.09197561056896073
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.11125,
    "ci95": [
     -0.14369590290035927,
     -0.07880409709964073
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "gpt-minus-opus": {
   "all": {
    "diff": -0.030416666666666668,
    "ci95": [
     -0.056388875942394445,
     -0.004444457390938891
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.005000000000000001,
    "ci95": [
     -0.025406897160101513,
     0.015406897160101511
    ],
    "n": 200,
    "verdict": "차이 없음"
   }
  },
  "gemini-minus-opus": {
   "all": {
    "diff": -0.029583333333333336,
    "ci95": [
     -0.056084893999743986,
     -0.003081772666922687
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.025,
    "ci95": [
     -0.04693792318061361,
     -0.003062076819386396
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "4b-minus-gemini": {
   "all": {
    "diff": -0.14083333333333334,
    "ci95": [
     -0.18037638488148539,
     -0.10129028178518129
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.13708333333333333,
    "ci95": [
     -0.17551762539100385,
     -0.0986490412756628
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen4b-minus-gemini": {
   "all": {
    "diff": -0.20333333333333337,
    "ci95": [
     -0.2445081566458691,
     -0.16215851002079762
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.19041666666666668,
    "ci95": [
     -0.23327826842581859,
     -0.14755506490751477
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "27b-minus-gemini": {
   "all": {
    "diff": -0.04375,
    "ci95": [
     -0.07778215493901924,
     -0.009717845060980758
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.035833333333333335,
    "ci95": [
     -0.06993577949443525,
     -0.001730887172231424
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "qwen27b-minus-gemini": {
   "all": {
    "diff": -0.09458333333333334,
    "ci95": [
     -0.13397842210418118,
     -0.0551882445624855
    ],
    "n": 200,
    "verdict": "악화"
   },
   "no_majority": {
    "diff": -0.08625,
    "ci95": [
     -0.12430713164969961,
     -0.04819286835030037
    ],
    "n": 200,
    "verdict": "악화"
   }
  },
  "gpt-minus-gemini": {
   "all": {
    "diff": -0.0008333333333333328,
    "ci95": [
     -0.03140094874945315,
     0.029734282082786483
    ],
    "n": 200,
    "verdict": "차이 없음"
   },
   "no_majority": {
    "diff": 0.02,
    "ci95": [
     -0.008200934523245256,
     0.048200934523245254
    ],
    "n": 200,
    "verdict": "차이 없음"
   }
  },
  "opus-minus-gemini": {
   "all": {
    "diff": 0.029583333333333336,
    "ci95": [
     0.003081772666922687,
     0.056084893999743986
    ],
    "n": 200,
    "verdict": "개선"
   },
   "no_majority": {
    "diff": 0.025,
    "ci95": [
     0.003062076819386396,
     0.04693792318061361
    ],
    "n": 200,
    "verdict": "개선"
   }
  }
 },
 "supplementary": {
  "qwen27b_system_prompt_only": {
   "label": "Qwen3.8-27B (base, SYSTEM_PROMPT only)",
   "criterion_acc": {
    "mean": 0.8433333333333333,
    "ci95": [
     0.8119954335270959,
     0.8746712331395706
    ],
    "n_answers": 200
   },
   "criterion_acc_excl_r2_2of3": {
    "mean": 0.8729166666666667,
    "ci95": [
     0.8426809286496472,
     0.9031524046836862
    ],
    "n_answers": 200
   },
   "json_invalid": 0,
   "paired_27b_minus_this": {
    "all": {
     "diff": 0.02541666666666667,
     "ci95": [
      -0.009341613239777467,
      0.06017494657311081
     ],
     "n": 200,
     "verdict": "차이 없음"
    },
    "no_majority": {
     "diff": 0.027083333333333334,
     "ci95": [
      -0.008110744018988085,
      0.062277410685654754
     ],
     "n": 200,
     "verdict": "차이 없음"
    }
   }
  }
 }
}