{
"benchmark": "RuBench v1",
"split": "dev",
"n_answers": 200,
"n_criteria": 604,
"built_at": "2026-11-04T00:15:59",
"conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), no JSON schema enforcement, invalid JSON counts as wrong. RuBio: SYSTEM_PROMPT only; base and general models: SYSTEM_PROMPT+POLICY. RuBio and Qwen base: temperature 0, max_tokens 1024, thinking off. General models: vendor CLI at the listed reasoning effort, one fixed run.",
"panel": {
"round1": [
"gpt",
"opus",
"grok"
],
"round2": [
"gpt",
"opus",
"gemini"
],
"student_answer_author": "gemini"
},
"caveats": [
"GPT and Opus were round-1 gold panelists: the r1-3/3 tier (521 of 604 dev criteria) contains their own blind judgments, so their scores include self-agreement and are upper bounds.",
"Round-2 majority (r2-2/3) items include GPT, Opus and Gemini votes; see criterion_acc_excl_r2_2of3.",
"Gemini wrote the synthetic student answers with intended labels; reported as reference only (DESIGN 4-1).",
"Grok 4.7 (round-1 panelist) was not evaluated as a model.",
"RuBio-27B is the RuIso-v6 retrain (main since 2026-11-04), run on a RunPod A100 Pod with vLLM 0.32.0 under the same generation settings; RuBio-4B v2 was run on RunPod Serverless (worker-v1-vllm, vLLM 0.32.0). The decision to replace the 27B main weights used this same dev split."
],
"selection_note": "RuBio-9B (RuIso-v6 retrain) seed was chosen by lowest best_eval_loss before RuBench scoring; the decision to replace the main weights used this same dev split.",
"models": {
"4b": {
"key": "4b",
"label": "RuBio-4B v2",
"hf": "mulkzz/RuBio-4B@ba9e00c8",
"role": "finetuned",
"size": "4B",
"name": "RuBio-4B v2",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.7716666666666667,
"ci95": [
0.7339481076683567,
0.8093852256649767
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.79875,
"ci95": [
0.759669235815335,
0.837830764184665
],
"n_answers": 200
},
"criterion_micro": {
"hit": 467,
"n": 604
},
"misconception": {
"k": 106,
"n": 133,
"p": 0.7969924812030075,
"ci95": [
0.7207474553277116,
0.8565630300901761
]
},
"misconception_with_id": {
"k": 65,
"n": 79,
"p": 0.8227848101265823,
"ci95": [
0.7242382665030478,
0.8913956243248582
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 156,
"n": 206,
"p": 0.7572815533980582,
"ci95": [
0.6943735885025178,
0.8107697158073134
]
},
"II": {
"k": 154,
"n": 200,
"p": 0.77,
"ci95": [
0.7069170932155301,
0.822906471137807
]
},
"III": {
"k": 157,
"n": 198,
"p": 0.7929292929292929,
"ci95": [
0.7311760804230768,
0.8435324539592243
]
}
},
"other_issues": {
"k": 153,
"n": 200,
"p": 0.765,
"ci95": [
0.7015864484736013,
0.8184255683917109
]
},
"answer_status": {
"k": 124,
"n": 200,
"p": 0.62,
"ci95": [
0.5510673223171103,
0.684409817395484
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 431,
"n": 521,
"p": 0.8272552783109405,
"ci95": [
0.7924302993782009,
0.8572897340091328
]
},
"r2-3/3": {
"k": 10,
"n": 23,
"p": 0.43478260869565216,
"ci95": [
0.2563466486442807,
0.6318858887246053
]
},
"r2-2/3": {
"k": 23,
"n": 51,
"p": 0.45098039215686275,
"ci95": [
0.32267542174059904,
0.5861526522832785
]
},
"human": {
"k": 3,
"n": 9,
"p": 0.3333333333333333,
"ci95": [
0.12058406396498972,
0.6457973346907151
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 10,
"n": 53,
"p": 0.18867924528301888,
"ci95": [
0.10585337627628057,
0.3135843061386674
]
},
"no_error_specificity": {
"k": 143,
"n": 147,
"p": 0.9727891156462585,
"ci95": [
0.9321289915585997,
0.9893684235398896
]
}
},
"confusion": {
"contradicted->contradicted": 121,
"contradicted->missing": 5,
"contradicted->supported": 3,
"contradicted->unclear": 4,
"missing->contradicted": 12,
"missing->missing": 92,
"missing->supported": 18,
"missing->unclear": 20,
"supported->contradicted": 14,
"supported->missing": 9,
"supported->supported": 232,
"supported->unclear": 28,
"unclear->contradicted": 8,
"unclear->missing": 7,
"unclear->supported": 9,
"unclear->unclear": 22
},
"cli_seconds_p50": null
},
"qwen4b": {
"key": "qwen4b",
"label": "Qwen3.5-4B (base)",
"hf": "Qwen/Qwen3.5-4B",
"role": "base",
"size": "4B",
"name": "Qwen3.5-4B (기본)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.7091666666666667,
"ci95": [
0.673031876586196,
0.7453014567471374
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.7454166666666667,
"ci95": [
0.7095199602308042,
0.7813133731025292
],
"n_answers": 200
},
"criterion_micro": {
"hit": 427,
"n": 604
},
"misconception": {
"k": 78,
"n": 133,
"p": 0.5864661654135338,
"ci95": [
0.501491500687543,
0.666586235573361
]
},
"misconception_with_id": {
"k": 55,
"n": 79,
"p": 0.6962025316455697,
"ci95": [
0.5876536225817802,
0.7865552130194958
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 144,
"n": 206,
"p": 0.6990291262135923,
"ci95": [
0.633218801872325,
0.7575524335371671
]
},
"II": {
"k": 142,
"n": 200,
"p": 0.71,
"ci95": [
0.6436253597813291,
0.7684596347157107
]
},
"III": {
"k": 141,
"n": 198,
"p": 0.7121212121212122,
"ci95": [
0.6454906562919455,
0.770677592743514
]
}
},
"other_issues": {
"k": 150,
"n": 200,
"p": 0.75,
"ci95": [
0.6856591553575013,
0.8049182190437367
]
},
"answer_status": {
"k": 89,
"n": 200,
"p": 0.445,
"ci95": [
0.37780614202227114,
0.5142668356094566
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 402,
"n": 521,
"p": 0.7715930902111324,
"ci95": [
0.7336347625660318,
0.8055757050252159
]
},
"r2-3/3": {
"k": 4,
"n": 23,
"p": 0.17391304347826086,
"ci95": [
0.06978666088359675,
0.37137602596083263
]
},
"r2-2/3": {
"k": 15,
"n": 51,
"p": 0.29411764705882354,
"ci95": [
0.1870874425221019,
0.4299904683781837
]
},
"human": {
"k": 6,
"n": 9,
"p": 0.6666666666666666,
"ci95": [
0.35420266530928485,
0.8794159360350102
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 4,
"n": 53,
"p": 0.07547169811320754,
"ci95": [
0.029739608356547,
0.17858450402747283
]
},
"no_error_specificity": {
"k": 146,
"n": 147,
"p": 0.9931972789115646,
"ci95": [
0.9624761437110765,
0.9987981382261968
]
}
},
"confusion": {
"contradicted->contradicted": 100,
"contradicted->missing": 20,
"contradicted->supported": 10,
"contradicted->unclear": 3,
"missing->contradicted": 22,
"missing->missing": 79,
"missing->supported": 37,
"missing->unclear": 4,
"supported->contradicted": 12,
"supported->missing": 16,
"supported->supported": 244,
"supported->unclear": 11,
"unclear->contradicted": 7,
"unclear->missing": 9,
"unclear->supported": 26,
"unclear->unclear": 4
},
"cli_seconds_p50": null
},
"9b-wt": {
"key": "9b-wt",
"src": "9b-v6-s3408",
"label": "RuBio-9B",
"hf": "mulkzz/RuBio-9B@f61ac9cb",
"role": "finetuned",
"size": "9B",
"name": "RuBio-9B RuIso-v6 (시드 3408)",
"answers": 200,
"missing_output": 0,
"json_invalid": 1,
"criterion_acc": {
"mean": 0.84875,
"ci95": [
0.8161433560231208,
0.8813566439768792
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.8770833333333333,
"ci95": [
0.8464292781603502,
0.9077373885063165
],
"n_answers": 200
},
"criterion_micro": {
"hit": 512,
"n": 604
},
"misconception": {
"k": 111,
"n": 133,
"p": 0.8345864661654135,
"ci95": [
0.7622364035210294,
0.888151358532034
]
},
"misconception_with_id": {
"k": 65,
"n": 79,
"p": 0.8227848101265823,
"ci95": [
0.7242382665030478,
0.8913956243248582
]
},
"json_valid": {
"k": 199,
"n": 200,
"p": 0.995,
"ci95": [
0.9722263725443985,
0.9991168287700527
]
},
"unit_criterion_micro": {
"I": {
"k": 170,
"n": 206,
"p": 0.8252427184466019,
"ci95": [
0.7675629836155808,
0.8710144010779794
]
},
"II": {
"k": 171,
"n": 200,
"p": 0.855,
"ci95": [
0.7995134505428394,
0.8971064211069184
]
},
"III": {
"k": 171,
"n": 198,
"p": 0.8636363636363636,
"ci95": [
0.8088693626231386,
0.9045619214376489
]
}
},
"other_issues": {
"k": 153,
"n": 200,
"p": 0.765,
"ci95": [
0.7015864484736013,
0.8184255683917109
]
},
"answer_status": {
"k": 144,
"n": 200,
"p": 0.72,
"ci95": [
0.6540770944519325,
0.7776309950211568
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 467,
"n": 521,
"p": 0.8963531669865643,
"ci95": [
0.8672147027925656,
0.9196896191692199
]
},
"r2-3/3": {
"k": 13,
"n": 23,
"p": 0.5652173913043478,
"ci95": [
0.36811411127539473,
0.7436533513557193
]
},
"r2-2/3": {
"k": 26,
"n": 51,
"p": 0.5098039215686274,
"ci95": [
0.37680949168569783,
0.6414248935095266
]
},
"human": {
"k": 6,
"n": 9,
"p": 0.6666666666666666,
"ci95": [
0.35420266530928485,
0.8794159360350102
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 42,
"n": 53,
"p": 0.7924528301886793,
"ci95": [
0.6654228381471919,
0.8799538844327056
]
},
"no_error_specificity": {
"k": 111,
"n": 147,
"p": 0.7551020408163265,
"ci95": [
0.6796736041927686,
0.8175372312920278
]
}
},
"confusion": {
"contradicted->(없음)": 1,
"contradicted->contradicted": 124,
"contradicted->missing": 5,
"contradicted->supported": 2,
"contradicted->unclear": 1,
"missing->contradicted": 7,
"missing->missing": 102,
"missing->supported": 22,
"missing->unclear": 11,
"supported->contradicted": 6,
"supported->missing": 9,
"supported->supported": 263,
"supported->unclear": 5,
"unclear->(없음)": 2,
"unclear->contradicted": 6,
"unclear->missing": 3,
"unclear->supported": 12,
"unclear->unclear": 23
},
"cli_seconds_p50": null
},
"qwen9b": {
"key": "qwen9b",
"label": "Qwen3.5-9B (base)",
"hf": "Qwen/Qwen3.5-9B",
"role": "base",
"size": "9B",
"name": "Qwen3.5-9B (기본)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.76625,
"ci95": [
0.7304187083476741,
0.8020812916523259
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.81375,
"ci95": [
0.7792670360036391,
0.8482329639963608
],
"n_answers": 200
},
"criterion_micro": {
"hit": 463,
"n": 604
},
"misconception": {
"k": 86,
"n": 133,
"p": 0.6466165413533834,
"ci95": [
0.5623037851806594,
0.7226975936965256
]
},
"misconception_with_id": {
"k": 62,
"n": 79,
"p": 0.7848101265822784,
"ci95": [
0.6821285476156216,
0.8610778266442956
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 152,
"n": 206,
"p": 0.7378640776699029,
"ci95": [
0.6738455884077236,
0.793173692935328
]
},
"II": {
"k": 156,
"n": 200,
"p": 0.78,
"ci95": [
0.7176121367571497,
0.8318345225722368
]
},
"III": {
"k": 155,
"n": 198,
"p": 0.7828282828282829,
"ci95": [
0.7203091094117178,
0.8345818893022281
]
}
},
"other_issues": {
"k": 153,
"n": 200,
"p": 0.765,
"ci95": [
0.7015864484736013,
0.8184255683917109
]
},
"answer_status": {
"k": 96,
"n": 200,
"p": 0.48,
"ci95": [
0.41179179908760966,
0.5489620109602914
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 440,
"n": 521,
"p": 0.8445297504798465,
"ci95": [
0.810905474589728,
0.8731106309412187
]
},
"r2-3/3": {
"k": 7,
"n": 23,
"p": 0.30434782608695654,
"ci95": [
0.15604046185489862,
0.5086571502517591
]
},
"r2-2/3": {
"k": 13,
"n": 51,
"p": 0.2549019607843137,
"ci95": [
0.15545784189248602,
0.38868252822690164
]
},
"human": {
"k": 3,
"n": 9,
"p": 0.3333333333333333,
"ci95": [
0.12058406396498972,
0.6457973346907151
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 7,
"n": 53,
"p": 0.1320754716981132,
"ci95": [
0.06547862476439556,
0.24840227263508838
]
},
"no_error_specificity": {
"k": 146,
"n": 147,
"p": 0.9931972789115646,
"ci95": [
0.9624761437110765,
0.9987981382261968
]
}
},
"confusion": {
"contradicted->contradicted": 113,
"contradicted->missing": 9,
"contradicted->supported": 9,
"contradicted->unclear": 2,
"missing->contradicted": 13,
"missing->missing": 72,
"missing->supported": 47,
"missing->unclear": 10,
"supported->contradicted": 5,
"supported->missing": 2,
"supported->supported": 270,
"supported->unclear": 6,
"unclear->contradicted": 7,
"unclear->missing": 3,
"unclear->supported": 28,
"unclear->unclear": 8
},
"cli_seconds_p50": null
},
"27b": {
"key": "27b",
"src": "27b-v6-s3407",
"label": "RuBio-27B",
"hf": "mulkzz/RuBio-27B@c1cfefdb",
"role": "finetuned",
"size": "27B",
"name": "RuBio-27B RuIso-v6 (시드 3407)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.86875,
"ci95": [
0.8381287620959553,
0.8993712379040447
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.9,
"ci95": [
0.8714503967862466,
0.9285496032137535
],
"n_answers": 200
},
"criterion_micro": {
"hit": 524,
"n": 604
},
"misconception": {
"k": 118,
"n": 133,
"p": 0.8872180451127819,
"ci95": [
0.8222450524506153,
0.9304508968916938
]
},
"misconception_with_id": {
"k": 73,
"n": 79,
"p": 0.9240506329113924,
"ci95": [
0.8440460366216044,
0.9647278983876056
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 185,
"n": 206,
"p": 0.8980582524271845,
"ci95": [
0.8491894424118119,
0.9323530284071725
]
},
"II": {
"k": 166,
"n": 200,
"p": 0.83,
"ci95": [
0.7718412936135373,
0.8757208405960968
]
},
"III": {
"k": 173,
"n": 198,
"p": 0.8737373737373737,
"ci95": [
0.8202540537968194,
0.9129947659323233
]
}
},
"other_issues": {
"k": 173,
"n": 200,
"p": 0.865,
"ci95": [
0.810708740295804,
0.9055342263300035
]
},
"answer_status": {
"k": 154,
"n": 200,
"p": 0.77,
"ci95": [
0.7069170932155301,
0.822906471137807
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 483,
"n": 521,
"p": 0.927063339731286,
"ci95": [
0.9014725811198147,
0.9464025357639493
]
},
"r2-3/3": {
"k": 11,
"n": 23,
"p": 0.4782608695652174,
"ci95": [
0.29237189265791597,
0.670372286465046
]
},
"r2-2/3": {
"k": 23,
"n": 51,
"p": 0.45098039215686275,
"ci95": [
0.32267542174059904,
0.5861526522832785
]
},
"human": {
"k": 7,
"n": 9,
"p": 0.7777777777777778,
"ci95": [
0.45258959147532657,
0.9367747440984988
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 46,
"n": 53,
"p": 0.8679245283018868,
"ci95": [
0.7515977273649117,
0.9345213752356045
]
},
"no_error_specificity": {
"k": 127,
"n": 147,
"p": 0.8639455782312925,
"ci95": [
0.7991850746700077,
0.9101690506216352
]
}
},
"confusion": {
"contradicted->contradicted": 127,
"contradicted->missing": 4,
"contradicted->supported": 1,
"contradicted->unclear": 1,
"missing->contradicted": 8,
"missing->missing": 125,
"missing->supported": 2,
"missing->unclear": 7,
"supported->contradicted": 4,
"supported->missing": 20,
"supported->supported": 253,
"supported->unclear": 6,
"unclear->contradicted": 4,
"unclear->missing": 15,
"unclear->supported": 8,
"unclear->unclear": 19
},
"cli_seconds_p50": null
},
"qwen27b": {
"key": "qwen27b",
"label": "Qwen3.8-27B (base)",
"hf": "Qwen/Qwen3.8-27B",
"role": "base",
"size": "27B",
"name": "Qwen3.8-27B (기본)",
"answers": 200,
"missing_output": 0,
"json_invalid": 5,
"criterion_acc": {
"mean": 0.8179166666666667,
"ci95": [
0.7816876269997497,
0.8541457063335838
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.8495833333333332,
"ci95": [
0.8142417703596521,
0.8849248963070144
],
"n_answers": 200
},
"criterion_micro": {
"hit": 495,
"n": 604
},
"misconception": {
"k": 107,
"n": 133,
"p": 0.8045112781954887,
"ci95": [
0.7289698915278552,
0.8629560492170677
]
},
"misconception_with_id": {
"k": 68,
"n": 79,
"p": 0.8607594936708861,
"ci95": [
0.7676269042027796,
0.9204345031931153
]
},
"json_valid": {
"k": 195,
"n": 200,
"p": 0.975,
"ci95": [
0.9428217485145032,
0.989275262847849
]
},
"unit_criterion_micro": {
"I": {
"k": 165,
"n": 206,
"p": 0.8009708737864077,
"ci95": [
0.7411592356541727,
0.8497631203309128
]
},
"II": {
"k": 167,
"n": 200,
"p": 0.835,
"ci95": [
0.777342313757747,
0.8800313679399118
]
},
"III": {
"k": 163,
"n": 198,
"p": 0.8232323232323232,
"ci95": [
0.764095715325512,
0.8700654260618547
]
}
},
"other_issues": {
"k": 156,
"n": 200,
"p": 0.78,
"ci95": [
0.7176121367571497,
0.8318345225722368
]
},
"answer_status": {
"k": 120,
"n": 200,
"p": 0.6,
"ci95": [
0.530836862213719,
0.6653940875467761
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 461,
"n": 521,
"p": 0.8848368522072937,
"ci95": [
0.854565276901407,
0.9094749969646364
]
},
"r2-3/3": {
"k": 5,
"n": 23,
"p": 0.21739130434782608,
"ci95": [
0.09663993783788177,
0.4190343907606237
]
},
"r2-2/3": {
"k": 24,
"n": 51,
"p": 0.47058823529411764,
"ci95": [
0.3405301441444071,
0.6047667002699194
]
},
"human": {
"k": 5,
"n": 9,
"p": 0.5555555555555556,
"ci95": [
0.26665173025527866,
0.8112211368594864
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 29,
"n": 53,
"p": 0.5471698113207547,
"ci95": [
0.41454001247278066,
0.6734239750401059
]
},
"no_error_specificity": {
"k": 127,
"n": 147,
"p": 0.8639455782312925,
"ci95": [
0.7991850746700077,
0.9101690506216352
]
}
},
"confusion": {
"contradicted->(없음)": 6,
"contradicted->contradicted": 112,
"contradicted->missing": 7,
"contradicted->supported": 5,
"contradicted->unclear": 3,
"missing->(없음)": 8,
"missing->contradicted": 3,
"missing->missing": 95,
"missing->supported": 20,
"missing->unclear": 16,
"supported->(없음)": 3,
"supported->contradicted": 4,
"supported->missing": 1,
"supported->supported": 269,
"supported->unclear": 6,
"unclear->(없음)": 1,
"unclear->missing": 3,
"unclear->supported": 23,
"unclear->unclear": 19
},
"cli_seconds_p50": 9.7
},
"gpt": {
"key": "gpt",
"label": "GPT-6 Astra (xhigh)",
"hf": null,
"role": "general",
"size": null,
"name": "GPT-6 Astra (xhigh)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.9116666666666667,
"ci95": [
0.8858637301001623,
0.9374696032331712
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.9558333333333333,
"ci95": [
0.9362898427686828,
0.9753768238979839
],
"n_answers": 200
},
"criterion_micro": {
"hit": 550,
"n": 604
},
"misconception": {
"k": 129,
"n": 133,
"p": 0.9699248120300752,
"ci95": [
0.9252225048958136,
0.9882434530438822
]
},
"misconception_with_id": {
"k": 78,
"n": 79,
"p": 0.9873417721518988,
"ci95": [
0.9317244628529116,
0.9977619997696132
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 184,
"n": 206,
"p": 0.8932038834951457,
"ci95": [
0.8436020225775144,
0.928409442499775
]
},
"II": {
"k": 185,
"n": 200,
"p": 0.925,
"ci95": [
0.8799564990407998,
0.9540250374413048
]
},
"III": {
"k": 181,
"n": 198,
"p": 0.9141414141414141,
"ci95": [
0.866814758905608,
0.9457042034969555
]
}
},
"other_issues": {
"k": 177,
"n": 200,
"p": 0.885,
"ci95": [
0.8333529508707886,
0.9221362057071179
]
},
"answer_status": {
"k": 176,
"n": 200,
"p": 0.88,
"ci95": [
0.8276575999704266,
0.9180200091194551
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 510,
"n": 521,
"p": 0.9788867562380038,
"ci95": [
0.9625927664904859,
0.9881705668241167
]
},
"r2-3/3": {
"k": 14,
"n": 23,
"p": 0.6086956521739131,
"ci95": [
0.4078556031551567,
0.7784235012300337
]
},
"r2-2/3": {
"k": 22,
"n": 51,
"p": 0.43137254901960786,
"ci95": [
0.30501259121603025,
0.5673467124173983
]
},
"human": {
"k": 4,
"n": 9,
"p": 0.4444444444444444,
"ci95": [
0.18877886314051356,
0.7333482697447213
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 33,
"n": 53,
"p": 0.6226415094339622,
"ci95": [
0.48806923125215484,
0.7406371362813504
]
},
"no_error_specificity": {
"k": 144,
"n": 147,
"p": 0.9795918367346939,
"ci95": [
0.9417208980015789,
0.9930354727098384
]
}
},
"confusion": {
"contradicted->contradicted": 129,
"contradicted->missing": 3,
"contradicted->unclear": 1,
"missing->contradicted": 1,
"missing->missing": 135,
"missing->supported": 2,
"missing->unclear": 4,
"supported->contradicted": 3,
"supported->missing": 11,
"supported->supported": 269,
"unclear->contradicted": 2,
"unclear->missing": 23,
"unclear->supported": 4,
"unclear->unclear": 17
},
"cli_seconds_p50": 575.6
},
"opus": {
"key": "opus",
"label": "Claude Opus 5.5 (xhigh)",
"hf": null,
"role": "general",
"size": null,
"name": "Claude Opus 5.5 (xhigh)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.9420833333333333,
"ci95": [
0.9203542491963614,
0.9638124174703051
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.9608333333333333,
"ci95": [
0.9417409178385642,
0.9799257488281025
],
"n_answers": 200
},
"criterion_micro": {
"hit": 570,
"n": 604
},
"misconception": {
"k": 127,
"n": 133,
"p": 0.9548872180451128,
"ci95": [
0.9050722874953598,
0.9791627597902657
]
},
"misconception_with_id": {
"k": 77,
"n": 79,
"p": 0.9746835443037974,
"ci95": [
0.9123143405749654,
0.9930296165248962
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 193,
"n": 206,
"p": 0.9368932038834952,
"ci95": [
0.895037892168079,
0.9627526245844646
]
},
"II": {
"k": 189,
"n": 200,
"p": 0.945,
"ci95": [
0.9042131036406227,
0.9690146227935807
]
},
"III": {
"k": 188,
"n": 198,
"p": 0.9494949494949495,
"ci95": [
0.9095419269028148,
0.9723384103389922
]
}
},
"other_issues": {
"k": 195,
"n": 200,
"p": 0.975,
"ci95": [
0.9428217485145032,
0.989275262847849
]
},
"answer_status": {
"k": 184,
"n": 200,
"p": 0.92,
"ci95": [
0.8740106225357106,
0.9501593664583693
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 510,
"n": 521,
"p": 0.9788867562380038,
"ci95": [
0.9625927664904859,
0.9881705668241167
]
},
"r2-3/3": {
"k": 18,
"n": 23,
"p": 0.782608695652174,
"ci95": [
0.5809656092393763,
0.9033600621621182
]
},
"r2-2/3": {
"k": 37,
"n": 51,
"p": 0.7254901960784313,
"ci95": [
0.5905337813556237,
0.8288570781345395
]
},
"human": {
"k": 5,
"n": 9,
"p": 0.5555555555555556,
"ci95": [
0.26665173025527866,
0.8112211368594864
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 52,
"n": 53,
"p": 0.9811320754716981,
"ci95": [
0.9005711355345968,
0.9966615370968476
]
},
"no_error_specificity": {
"k": 143,
"n": 147,
"p": 0.9727891156462585,
"ci95": [
0.9321289915585997,
0.9893684235398896
]
}
},
"confusion": {
"contradicted->contradicted": 128,
"contradicted->missing": 3,
"contradicted->supported": 1,
"contradicted->unclear": 1,
"missing->contradicted": 1,
"missing->missing": 123,
"missing->supported": 7,
"missing->unclear": 11,
"supported->missing": 3,
"supported->supported": 279,
"supported->unclear": 1,
"unclear->supported": 6,
"unclear->unclear": 40
},
"cli_seconds_p50": 41.3
},
"gemini": {
"key": "gemini",
"label": "Gemini 4 Argon (High)",
"hf": null,
"role": "reference",
"size": null,
"name": "Gemini 4 Argon (High)",
"answers": 200,
"missing_output": 0,
"json_invalid": 2,
"criterion_acc": {
"mean": 0.9125,
"ci95": [
0.8850333281496484,
0.9399666718503515
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.9358333333333333,
"ci95": [
0.9098731921940174,
0.9617934744726492
],
"n_answers": 200
},
"criterion_micro": {
"hit": 551,
"n": 604
},
"misconception": {
"k": 122,
"n": 133,
"p": 0.9172932330827067,
"ci95": [
0.8579662495837602,
0.9531915210666896
]
},
"misconception_with_id": {
"k": 75,
"n": 79,
"p": 0.9493670886075949,
"ci95": [
0.8769230243999817,
0.980135921654554
]
},
"json_valid": {
"k": 198,
"n": 200,
"p": 0.99,
"ci95": [
0.9642783183248235,
0.997253335501603
]
},
"unit_criterion_micro": {
"I": {
"k": 187,
"n": 206,
"p": 0.9077669902912622,
"ci95": [
0.8604469235517486,
0.9401575587506256
]
},
"II": {
"k": 184,
"n": 200,
"p": 0.92,
"ci95": [
0.8740106225357106,
0.9501593664583693
]
},
"III": {
"k": 180,
"n": 198,
"p": 0.9090909090909091,
"ci95": [
0.8608883910901738,
0.9417218034782121
]
}
},
"other_issues": {
"k": 190,
"n": 200,
"p": 0.95,
"ci95": [
0.9104219523342212,
0.9726173215880072
]
},
"answer_status": {
"k": 179,
"n": 200,
"p": 0.895,
"ci95": [
0.8448197976153835,
0.9302924539385726
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 496,
"n": 521,
"p": 0.9520153550863724,
"ci95": [
0.9301245809465594,
0.9672893068113121
]
},
"r2-3/3": {
"k": 19,
"n": 23,
"p": 0.8260869565217391,
"ci95": [
0.6286239740391674,
0.9302133391164032
]
},
"r2-2/3": {
"k": 32,
"n": 51,
"p": 0.6274509803921569,
"ci95": [
0.49025241212438464,
0.7467945954135337
]
},
"human": {
"k": 4,
"n": 9,
"p": 0.4444444444444444,
"ci95": [
0.18877886314051356,
0.7333482697447213
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 45,
"n": 53,
"p": 0.8490566037735849,
"ci95": [
0.7294577896504801,
0.9214757179448814
]
},
"no_error_specificity": {
"k": 145,
"n": 147,
"p": 0.9863945578231292,
"ci95": [
0.9517544022691328,
0.9962609240552124
]
}
},
"confusion": {
"contradicted->contradicted": 125,
"contradicted->missing": 5,
"contradicted->supported": 1,
"contradicted->unclear": 2,
"missing->(없음)": 2,
"missing->missing": 125,
"missing->supported": 10,
"missing->unclear": 5,
"supported->(없음)": 3,
"supported->contradicted": 2,
"supported->missing": 6,
"supported->supported": 267,
"supported->unclear": 5,
"unclear->(없음)": 1,
"unclear->contradicted": 1,
"unclear->missing": 9,
"unclear->supported": 1,
"unclear->unclear": 34
},
"cli_seconds_p50": 87.9
}
},
"paired": {
"qwen4b-minus-4b": {
"all": {
"diff": -0.06250000000000001,
"ci95": [
-0.09929215430834135,
-0.02570784569165868
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.05333333333333334,
"ci95": [
-0.0949195943790324,
-0.011747072287634275
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-4b": {
"all": {
"diff": 0.07708333333333334,
"ci95": [
0.04658272750965517,
0.1075839391570115
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.07833333333333334,
"ci95": [
0.043278517520531544,
0.11338814914613513
],
"n": 200,
"verdict": "개선"
}
},
"qwen9b-minus-4b": {
"all": {
"diff": -0.005416666666666669,
"ci95": [
-0.04459678358809187,
0.033763450254758534
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": 0.015,
"ci95": [
-0.028534757736911935,
0.058534757736911934
],
"n": 200,
"verdict": "차이 없음"
}
},
"27b-minus-4b": {
"all": {
"diff": 0.09708333333333334,
"ci95": [
0.06123184381518337,
0.1329348228514833
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.10125,
"ci95": [
0.06521017094468781,
0.1372898290553122
],
"n": 200,
"verdict": "개선"
}
},
"qwen27b-minus-4b": {
"all": {
"diff": 0.04625,
"ci95": [
0.007906503441078273,
0.08459349655892173
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05083333333333334,
"ci95": [
0.008389559707535299,
0.09327710695913138
],
"n": 200,
"verdict": "개선"
}
},
"gpt-minus-4b": {
"all": {
"diff": 0.14,
"ci95": [
0.1028811816212637,
0.17711881837873633
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.15708333333333335,
"ci95": [
0.11841813107448726,
0.19574853559217945
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-4b": {
"all": {
"diff": 0.1704166666666667,
"ci95": [
0.1349501475307221,
0.20588318580261128
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.16208333333333336,
"ci95": [
0.12777146352786023,
0.19639520313880648
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-4b": {
"all": {
"diff": 0.14083333333333334,
"ci95": [
0.10129028178518129,
0.18037638488148539
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.13708333333333333,
"ci95": [
0.0986490412756628,
0.17551762539100385
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-qwen4b": {
"all": {
"diff": 0.06250000000000001,
"ci95": [
0.02570784569165868,
0.09929215430834135
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05333333333333334,
"ci95": [
0.011747072287634275,
0.0949195943790324
],
"n": 200,
"verdict": "개선"
}
},
"9b-wt-minus-qwen4b": {
"all": {
"diff": 0.13958333333333334,
"ci95": [
0.10343097151649225,
0.17573569515017443
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.13166666666666668,
"ci95": [
0.0910702383211025,
0.17226309501223086
],
"n": 200,
"verdict": "개선"
}
},
"qwen9b-minus-qwen4b": {
"all": {
"diff": 0.05708333333333334,
"ci95": [
0.023976872759691126,
0.09018979390697555
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.06833333333333334,
"ci95": [
0.032557035692015616,
0.10410963097465106
],
"n": 200,
"verdict": "개선"
}
},
"27b-minus-qwen4b": {
"all": {
"diff": 0.15958333333333333,
"ci95": [
0.12437326122557398,
0.1947934054410927
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.15458333333333335,
"ci95": [
0.11312043866157052,
0.1960462280050962
],
"n": 200,
"verdict": "개선"
}
},
"qwen27b-minus-qwen4b": {
"all": {
"diff": 0.10875,
"ci95": [
0.06948238748995589,
0.1480176125100441
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.10416666666666669,
"ci95": [
0.06221595810731039,
0.14611737522602297
],
"n": 200,
"verdict": "개선"
}
},
"gpt-minus-qwen4b": {
"all": {
"diff": 0.2025,
"ci95": [
0.16509784766297997,
0.23990215233702006
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.21041666666666667,
"ci95": [
0.1727924563143645,
0.24804087701896885
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-qwen4b": {
"all": {
"diff": 0.2329166666666667,
"ci95": [
0.1962971271528449,
0.26953620618048846
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.21541666666666667,
"ci95": [
0.17708051297608593,
0.25375282035724744
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-qwen4b": {
"all": {
"diff": 0.20333333333333337,
"ci95": [
0.16215851002079762,
0.2445081566458691
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.19041666666666668,
"ci95": [
0.14755506490751477,
0.23327826842581859
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-9b-wt": {
"all": {
"diff": -0.07708333333333334,
"ci95": [
-0.1075839391570115,
-0.04658272750965517
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.07833333333333334,
"ci95": [
-0.11338814914613513,
-0.043278517520531544
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-9b-wt": {
"all": {
"diff": -0.13958333333333334,
"ci95": [
-0.17573569515017443,
-0.10343097151649225
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.13166666666666668,
"ci95": [
-0.17226309501223086,
-0.0910702383211025
],
"n": 200,
"verdict": "악화"
}
},
"qwen9b-minus-9b-wt": {
"all": {
"diff": -0.0825,
"ci95": [
-0.11567548042449224,
-0.04932451957550776
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.06333333333333334,
"ci95": [
-0.09993351090564791,
-0.02673315576101877
],
"n": 200,
"verdict": "악화"
}
},
"27b-minus-9b-wt": {
"all": {
"diff": 0.02,
"ci95": [
-0.014332229414202247,
0.05433222941420225
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": 0.02291666666666667,
"ci95": [
-0.011448688434242939,
0.057282021767576276
],
"n": 200,
"verdict": "차이 없음"
}
},
"qwen27b-minus-9b-wt": {
"all": {
"diff": -0.030833333333333334,
"ci95": [
-0.06614857086946481,
0.004481904202798138
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": -0.0275,
"ci95": [
-0.06235288624359042,
0.007352886243590425
],
"n": 200,
"verdict": "차이 없음"
}
},
"gpt-minus-9b-wt": {
"all": {
"diff": 0.06291666666666668,
"ci95": [
0.03084018163541989,
0.09499315169791346
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.07875000000000001,
"ci95": [
0.04891013146731345,
0.10858986853268658
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-9b-wt": {
"all": {
"diff": 0.09333333333333334,
"ci95": [
0.061867359403474584,
0.12479930726319209
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.08375,
"ci95": [
0.05559051266998988,
0.11190948733001013
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-9b-wt": {
"all": {
"diff": 0.06375,
"ci95": [
0.02575279935154528,
0.10174720064845472
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05875,
"ci95": [
0.024038033740232924,
0.09346196625976708
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-qwen9b": {
"all": {
"diff": 0.005416666666666669,
"ci95": [
-0.033763450254758534,
0.04459678358809187
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": -0.015,
"ci95": [
-0.058534757736911934,
0.028534757736911935
],
"n": 200,
"verdict": "차이 없음"
}
},
"qwen4b-minus-qwen9b": {
"all": {
"diff": -0.05708333333333334,
"ci95": [
-0.09018979390697555,
-0.023976872759691126
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.06833333333333334,
"ci95": [
-0.10410963097465106,
-0.032557035692015616
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-qwen9b": {
"all": {
"diff": 0.0825,
"ci95": [
0.04932451957550776,
0.11567548042449224
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.06333333333333334,
"ci95": [
0.02673315576101877,
0.09993351090564791
],
"n": 200,
"verdict": "개선"
}
},
"27b-minus-qwen9b": {
"all": {
"diff": 0.1025,
"ci95": [
0.06413880766177524,
0.14086119233822475
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.08625,
"ci95": [
0.04496564082615172,
0.12753435917384826
],
"n": 200,
"verdict": "개선"
}
},
"qwen27b-minus-qwen9b": {
"all": {
"diff": 0.051666666666666666,
"ci95": [
0.017563237751625577,
0.08577009558170776
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.035833333333333335,
"ci95": [
0.00021194102234852602,
0.07145472564431815
],
"n": 200,
"verdict": "개선"
}
},
"gpt-minus-qwen9b": {
"all": {
"diff": 0.14541666666666667,
"ci95": [
0.10780920665808538,
0.18302412667524795
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.14208333333333334,
"ci95": [
0.10584097494818269,
0.178325691718484
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-qwen9b": {
"all": {
"diff": 0.17583333333333337,
"ci95": [
0.14348026996831242,
0.20818639669835431
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.14708333333333334,
"ci95": [
0.11374685963285322,
0.18041980703381347
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-qwen9b": {
"all": {
"diff": 0.14625000000000002,
"ci95": [
0.10733733957973716,
0.18516266042026286
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.12208333333333334,
"ci95": [
0.08337135245263665,
0.16079531421403004
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-27b": {
"all": {
"diff": -0.09708333333333334,
"ci95": [
-0.1329348228514833,
-0.06123184381518337
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.10125,
"ci95": [
-0.1372898290553122,
-0.06521017094468781
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-27b": {
"all": {
"diff": -0.15958333333333333,
"ci95": [
-0.1947934054410927,
-0.12437326122557398
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.15458333333333335,
"ci95": [
-0.1960462280050962,
-0.11312043866157052
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-27b": {
"all": {
"diff": -0.02,
"ci95": [
-0.05433222941420225,
0.014332229414202247
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": -0.02291666666666667,
"ci95": [
-0.057282021767576276,
0.011448688434242939
],
"n": 200,
"verdict": "차이 없음"
}
},
"qwen9b-minus-27b": {
"all": {
"diff": -0.1025,
"ci95": [
-0.14086119233822475,
-0.06413880766177524
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.08625,
"ci95": [
-0.12753435917384826,
-0.04496564082615172
],
"n": 200,
"verdict": "악화"
}
},
"qwen27b-minus-27b": {
"all": {
"diff": -0.05083333333333334,
"ci95": [
-0.09092183669301501,
-0.010744829973651678
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.05041666666666667,
"ci95": [
-0.0903058270121205,
-0.01052750632121284
],
"n": 200,
"verdict": "악화"
}
},
"gpt-minus-27b": {
"all": {
"diff": 0.04291666666666667,
"ci95": [
0.01078839936599886,
0.07504493396733448
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05583333333333334,
"ci95": [
0.02437694687489072,
0.08728971979177597
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-27b": {
"all": {
"diff": 0.07333333333333333,
"ci95": [
0.04152834968415308,
0.10513831698251358
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.060833333333333336,
"ci95": [
0.030981954682252844,
0.09068471198441383
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-27b": {
"all": {
"diff": 0.04375,
"ci95": [
0.009717845060980758,
0.07778215493901924
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.035833333333333335,
"ci95": [
0.001730887172231424,
0.06993577949443525
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-qwen27b": {
"all": {
"diff": -0.04625,
"ci95": [
-0.08459349655892173,
-0.007906503441078273
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.05083333333333334,
"ci95": [
-0.09327710695913138,
-0.008389559707535299
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-qwen27b": {
"all": {
"diff": -0.10875,
"ci95": [
-0.1480176125100441,
-0.06948238748995589
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.10416666666666669,
"ci95": [
-0.14611737522602297,
-0.06221595810731039
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-qwen27b": {
"all": {
"diff": 0.030833333333333334,
"ci95": [
-0.004481904202798138,
0.06614857086946481
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": 0.0275,
"ci95": [
-0.007352886243590425,
0.06235288624359042
],
"n": 200,
"verdict": "차이 없음"
}
},
"qwen9b-minus-qwen27b": {
"all": {
"diff": -0.051666666666666666,
"ci95": [
-0.08577009558170776,
-0.017563237751625577
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.035833333333333335,
"ci95": [
-0.07145472564431815,
-0.00021194102234852602
],
"n": 200,
"verdict": "악화"
}
},
"27b-minus-qwen27b": {
"all": {
"diff": 0.05083333333333334,
"ci95": [
0.010744829973651678,
0.09092183669301501
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05041666666666667,
"ci95": [
0.01052750632121284,
0.0903058270121205
],
"n": 200,
"verdict": "개선"
}
},
"gpt-minus-qwen27b": {
"all": {
"diff": 0.09375,
"ci95": [
0.05730219163645278,
0.13019780836354722
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.10625,
"ci95": [
0.07279390330302032,
0.13970609669697967
],
"n": 200,
"verdict": "개선"
}
},
"opus-minus-qwen27b": {
"all": {
"diff": 0.12416666666666668,
"ci95": [
0.09197561056896073,
0.15635772276437263
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.11125,
"ci95": [
0.07880409709964073,
0.14369590290035927
],
"n": 200,
"verdict": "개선"
}
},
"gemini-minus-qwen27b": {
"all": {
"diff": 0.09458333333333334,
"ci95": [
0.0551882445624855,
0.13397842210418118
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.08625,
"ci95": [
0.04819286835030037,
0.12430713164969961
],
"n": 200,
"verdict": "개선"
}
},
"4b-minus-gpt": {
"all": {
"diff": -0.14,
"ci95": [
-0.17711881837873633,
-0.1028811816212637
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.15708333333333335,
"ci95": [
-0.19574853559217945,
-0.11841813107448726
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-gpt": {
"all": {
"diff": -0.2025,
"ci95": [
-0.23990215233702006,
-0.16509784766297997
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.21041666666666667,
"ci95": [
-0.24804087701896885,
-0.1727924563143645
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-gpt": {
"all": {
"diff": -0.06291666666666668,
"ci95": [
-0.09499315169791346,
-0.03084018163541989
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.07875000000000001,
"ci95": [
-0.10858986853268658,
-0.04891013146731345
],
"n": 200,
"verdict": "악화"
}
},
"qwen9b-minus-gpt": {
"all": {
"diff": -0.14541666666666667,
"ci95": [
-0.18302412667524795,
-0.10780920665808538
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.14208333333333334,
"ci95": [
-0.178325691718484,
-0.10584097494818269
],
"n": 200,
"verdict": "악화"
}
},
"27b-minus-gpt": {
"all": {
"diff": -0.04291666666666667,
"ci95": [
-0.07504493396733448,
-0.01078839936599886
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.05583333333333334,
"ci95": [
-0.08728971979177597,
-0.02437694687489072
],
"n": 200,
"verdict": "악화"
}
},
"qwen27b-minus-gpt": {
"all": {
"diff": -0.09375,
"ci95": [
-0.13019780836354722,
-0.05730219163645278
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.10625,
"ci95": [
-0.13970609669697967,
-0.07279390330302032
],
"n": 200,
"verdict": "악화"
}
},
"opus-minus-gpt": {
"all": {
"diff": 0.030416666666666668,
"ci95": [
0.004444457390938891,
0.056388875942394445
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.005000000000000001,
"ci95": [
-0.015406897160101511,
0.025406897160101513
],
"n": 200,
"verdict": "차이 없음"
}
},
"gemini-minus-gpt": {
"all": {
"diff": 0.0008333333333333328,
"ci95": [
-0.029734282082786483,
0.03140094874945315
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": -0.02,
"ci95": [
-0.048200934523245254,
0.008200934523245256
],
"n": 200,
"verdict": "차이 없음"
}
},
"4b-minus-opus": {
"all": {
"diff": -0.1704166666666667,
"ci95": [
-0.20588318580261128,
-0.1349501475307221
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.16208333333333336,
"ci95": [
-0.19639520313880648,
-0.12777146352786023
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-opus": {
"all": {
"diff": -0.2329166666666667,
"ci95": [
-0.26953620618048846,
-0.1962971271528449
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.21541666666666667,
"ci95": [
-0.25375282035724744,
-0.17708051297608593
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-opus": {
"all": {
"diff": -0.09333333333333334,
"ci95": [
-0.12479930726319209,
-0.061867359403474584
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.08375,
"ci95": [
-0.11190948733001013,
-0.05559051266998988
],
"n": 200,
"verdict": "악화"
}
},
"qwen9b-minus-opus": {
"all": {
"diff": -0.17583333333333337,
"ci95": [
-0.20818639669835431,
-0.14348026996831242
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.14708333333333334,
"ci95": [
-0.18041980703381347,
-0.11374685963285322
],
"n": 200,
"verdict": "악화"
}
},
"27b-minus-opus": {
"all": {
"diff": -0.07333333333333333,
"ci95": [
-0.10513831698251358,
-0.04152834968415308
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.060833333333333336,
"ci95": [
-0.09068471198441383,
-0.030981954682252844
],
"n": 200,
"verdict": "악화"
}
},
"qwen27b-minus-opus": {
"all": {
"diff": -0.12416666666666668,
"ci95": [
-0.15635772276437263,
-0.09197561056896073
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.11125,
"ci95": [
-0.14369590290035927,
-0.07880409709964073
],
"n": 200,
"verdict": "악화"
}
},
"gpt-minus-opus": {
"all": {
"diff": -0.030416666666666668,
"ci95": [
-0.056388875942394445,
-0.004444457390938891
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.005000000000000001,
"ci95": [
-0.025406897160101513,
0.015406897160101511
],
"n": 200,
"verdict": "차이 없음"
}
},
"gemini-minus-opus": {
"all": {
"diff": -0.029583333333333336,
"ci95": [
-0.056084893999743986,
-0.003081772666922687
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.025,
"ci95": [
-0.04693792318061361,
-0.003062076819386396
],
"n": 200,
"verdict": "악화"
}
},
"4b-minus-gemini": {
"all": {
"diff": -0.14083333333333334,
"ci95": [
-0.18037638488148539,
-0.10129028178518129
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.13708333333333333,
"ci95": [
-0.17551762539100385,
-0.0986490412756628
],
"n": 200,
"verdict": "악화"
}
},
"qwen4b-minus-gemini": {
"all": {
"diff": -0.20333333333333337,
"ci95": [
-0.2445081566458691,
-0.16215851002079762
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.19041666666666668,
"ci95": [
-0.23327826842581859,
-0.14755506490751477
],
"n": 200,
"verdict": "악화"
}
},
"9b-wt-minus-gemini": {
"all": {
"diff": -0.06375,
"ci95": [
-0.10174720064845472,
-0.02575279935154528
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.05875,
"ci95": [
-0.09346196625976708,
-0.024038033740232924
],
"n": 200,
"verdict": "악화"
}
},
"qwen9b-minus-gemini": {
"all": {
"diff": -0.14625000000000002,
"ci95": [
-0.18516266042026286,
-0.10733733957973716
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.12208333333333334,
"ci95": [
-0.16079531421403004,
-0.08337135245263665
],
"n": 200,
"verdict": "악화"
}
},
"27b-minus-gemini": {
"all": {
"diff": -0.04375,
"ci95": [
-0.07778215493901924,
-0.009717845060980758
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.035833333333333335,
"ci95": [
-0.06993577949443525,
-0.001730887172231424
],
"n": 200,
"verdict": "악화"
}
},
"qwen27b-minus-gemini": {
"all": {
"diff": -0.09458333333333334,
"ci95": [
-0.13397842210418118,
-0.0551882445624855
],
"n": 200,
"verdict": "악화"
},
"no_majority": {
"diff": -0.08625,
"ci95": [
-0.12430713164969961,
-0.04819286835030037
],
"n": 200,
"verdict": "악화"
}
},
"gpt-minus-gemini": {
"all": {
"diff": -0.0008333333333333328,
"ci95": [
-0.03140094874945315,
0.029734282082786483
],
"n": 200,
"verdict": "차이 없음"
},
"no_majority": {
"diff": 0.02,
"ci95": [
-0.008200934523245256,
0.048200934523245254
],
"n": 200,
"verdict": "차이 없음"
}
},
"opus-minus-gemini": {
"all": {
"diff": 0.029583333333333336,
"ci95": [
0.003081772666922687,
0.056084893999743986
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.025,
"ci95": [
0.003062076819386396,
0.04693792318061361
],
"n": 200,
"verdict": "개선"
}
}
},
"supplementary": {
"qwen9b_system_prompt_only": {
"label": "Qwen3.5-9B (base, SYSTEM_PROMPT only)",
"strict": {
"criterion_acc": {
"mean": 0.0375,
"ci95": [
0.013275135056749963,
0.06172486494325004
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.04166666666666667,
"ci95": [
0.01559599220278532,
0.06773734113054802
],
"n_answers": 200
},
"criterion_micro": {
"hit": 21,
"n": 604
},
"json_invalid": 0,
"paired_9b_minus_this": {
"all": {
"diff": 0.81125,
"ci95": [
0.7708834392338799,
0.8516165607661201
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.8354166666666667,
"ci95": [
0.7953006259012946,
0.8755327074320388
],
"n": 200,
"verdict": "개선"
}
}
},
"lenient_checks_dict": {
"criterion_acc": {
"mean": 0.76,
"ci95": [
0.7225611269837042,
0.7974388730162958
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.8008333333333333,
"ci95": [
0.7643698503804338,
0.8372968162862328
],
"n_answers": 200
},
"criterion_micro": {
"hit": 458,
"n": 604
},
"json_invalid": 0,
"paired_9b_minus_this": {
"all": {
"diff": 0.08875,
"ci95": [
0.051052105431661846,
0.12644789456833816
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.07625000000000001,
"ci95": [
0.03594492540328466,
0.11655507459671537
],
"n": 200,
"verdict": "개선"
}
}
}
}
}
}
{
"benchmark": "RuBench v1",
"split": "dev",
"n_answers": 200,
"n_criteria": 604,
"built_at": "2026-11-04T00:15:59",
"conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), no JSON schema enforcement, invalid JSON counts as wrong. RuBio: SYSTEM_PROMPT only; base and general models: SYSTEM_PROMPT+POLICY. RuBio and Qwen base: temperature 0, max_tokens 1024, thinking off. General models: vendor CLI at the listed reasoning effort, one fixed run.",
"panel": {
"round1": [
"gpt",
"opus",
"grok"
],
"round2": [
"gpt",
"opus",
"gemini"
],
"student_answer_author": "gemini"
},
"caveats": [
"GPT and Opus were round-1 gold panelists: the r1-3/3 tier (521 of 604 dev criteria) contains their own blind judgments, so their scores include self-agreement and are upper bounds.",
"Round-2 majority (r2-2/3) items include GPT, Opus and Gemini votes; see criterion_acc_excl_r2_2of3.",
"Gemini wrote the synthetic student answers with intended labels; reported as reference only (DESIGN 4-1).",
"Grok 4.7 (round-1 panelist) was not evaluated as a model.",
"RuBio-27B is the RuIso-v6 retrain (main since 2026-11-04), run on a RunPod A100 Pod with vLLM 0.32.0 under the same generation settings; RuBio-4B v2 was run on RunPod Serverless (worker-v1-vllm, vLLM 0.32.0). The decision to replace the 27B main weights used this same dev split."
],
"selection_note": "RuBio-9B (RuIso-v6 retrain) seed was chosen by lowest best_eval_loss before RuBench scoring; the decision to replace the main weights used this same dev split.",
"models": {
"4b": {
"key": "4b",
"label": "RuBio-4B v2",
"hf": "mulkzz/RuBio-4B@ba9e00c8",
"role": "finetuned",
"size": "4B",
"name": "RuBio-4B v2",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.7716666666666667,
"ci95": [
0.7339481076683567,
0.8093852256649767
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.79875,
"ci95": [
0.759669235815335,
0.837830764184665
],
"n_answers": 200
},
"criterion_micro": {
"hit": 467,
"n": 604
},
"misconception": {
"k": 106,
"n": 133,
"p": 0.7969924812030075,
"ci95": [
0.7207474553277116,
0.8565630300901761
]
},
"misconception_with_id": {
"k": 65,
"n": 79,
"p": 0.8227848101265823,
"ci95": [
0.7242382665030478,
0.8913956243248582
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 156,
"n": 206,
"p": 0.7572815533980582,
"ci95": [
0.6943735885025178,
0.8107697158073134
]
},
"II": {
"k": 154,
"n": 200,
"p": 0.77,