| { |
| "benchmark": "RuBench v1", |
| "split": "dev", |
| "n_answers": 200, |
| "n_criteria": 604, |
| "built_at": "2026-11-04T00:14:40", |
| "conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), temperature 0, max_tokens 1024, enable_thinking=false, no JSON schema enforcement. RuBio: SYSTEM_PROMPT only; base: SYSTEM_PROMPT+POLICY. Invalid JSON counts as wrong.", |
| "finetuned": { |
| "key": "9b-v6-s3408", |
| "label": "RuBio-9B", |
| "hf": "mulkzz/RuBio-9B@f61ac9cb", |
| "in_dist_train_dev_criterion_match": 0.9743260590500642, |
| "name": "RuBio-9B RuIso-v6 (시드 3408)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 1, |
| "criterion_acc": { |
| "mean": 0.84875, |
| "ci95": [ |
| 0.8161433560231208, |
| 0.8813566439768792 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.8770833333333333, |
| "ci95": [ |
| 0.8464292781603502, |
| 0.9077373885063165 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 512, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 111, |
| "n": 133, |
| "p": 0.8345864661654135, |
| "ci95": [ |
| 0.7622364035210294, |
| 0.888151358532034 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 65, |
| "n": 79, |
| "p": 0.8227848101265823, |
| "ci95": [ |
| 0.7242382665030478, |
| 0.8913956243248582 |
| ] |
| }, |
| "json_valid": { |
| "k": 199, |
| "n": 200, |
| "p": 0.995, |
| "ci95": [ |
| 0.9722263725443985, |
| 0.9991168287700527 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 170, |
| "n": 206, |
| "p": 0.8252427184466019, |
| "ci95": [ |
| 0.7675629836155808, |
| 0.8710144010779794 |
| ] |
| }, |
| "II": { |
| "k": 171, |
| "n": 200, |
| "p": 0.855, |
| "ci95": [ |
| 0.7995134505428394, |
| 0.8971064211069184 |
| ] |
| }, |
| "III": { |
| "k": 171, |
| "n": 198, |
| "p": 0.8636363636363636, |
| "ci95": [ |
| 0.8088693626231386, |
| 0.9045619214376489 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 153, |
| "n": 200, |
| "p": 0.765, |
| "ci95": [ |
| 0.7015864484736013, |
| 0.8184255683917109 |
| ] |
| }, |
| "answer_status": { |
| "k": 144, |
| "n": 200, |
| "p": 0.72, |
| "ci95": [ |
| 0.6540770944519325, |
| 0.7776309950211568 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 467, |
| "n": 521, |
| "p": 0.8963531669865643, |
| "ci95": [ |
| 0.8672147027925656, |
| 0.9196896191692199 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 13, |
| "n": 23, |
| "p": 0.5652173913043478, |
| "ci95": [ |
| 0.36811411127539473, |
| 0.7436533513557193 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 26, |
| "n": 51, |
| "p": 0.5098039215686274, |
| "ci95": [ |
| 0.37680949168569783, |
| 0.6414248935095266 |
| ] |
| }, |
| "human": { |
| "k": 6, |
| "n": 9, |
| "p": 0.6666666666666666, |
| "ci95": [ |
| 0.35420266530928485, |
| 0.8794159360350102 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 42, |
| "n": 53, |
| "p": 0.7924528301886793, |
| "ci95": [ |
| 0.6654228381471919, |
| 0.8799538844327056 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 111, |
| "n": 147, |
| "p": 0.7551020408163265, |
| "ci95": [ |
| 0.6796736041927686, |
| 0.8175372312920278 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->(없음)": 1, |
| "contradicted->contradicted": 124, |
| "contradicted->missing": 5, |
| "contradicted->supported": 2, |
| "contradicted->unclear": 1, |
| "missing->contradicted": 7, |
| "missing->missing": 102, |
| "missing->supported": 22, |
| "missing->unclear": 11, |
| "supported->contradicted": 6, |
| "supported->missing": 9, |
| "supported->supported": 263, |
| "supported->unclear": 5, |
| "unclear->(없음)": 2, |
| "unclear->contradicted": 6, |
| "unclear->missing": 3, |
| "unclear->supported": 12, |
| "unclear->unclear": 23 |
| } |
| }, |
| "base": { |
| "key": "qwen9b", |
| "label": "Qwen3.5-9B (base)", |
| "hf": "Qwen/Qwen3.5-9B", |
| "serving": "RunPod A40 Pod, vLLM 0.32.0, bf16", |
| "name": "Qwen3.5-9B (기본)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.76625, |
| "ci95": [ |
| 0.7304187083476741, |
| 0.8020812916523259 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.81375, |
| "ci95": [ |
| 0.7792670360036391, |
| 0.8482329639963608 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 463, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 86, |
| "n": 133, |
| "p": 0.6466165413533834, |
| "ci95": [ |
| 0.5623037851806594, |
| 0.7226975936965256 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 62, |
| "n": 79, |
| "p": 0.7848101265822784, |
| "ci95": [ |
| 0.6821285476156216, |
| 0.8610778266442956 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 152, |
| "n": 206, |
| "p": 0.7378640776699029, |
| "ci95": [ |
| 0.6738455884077236, |
| 0.793173692935328 |
| ] |
| }, |
| "II": { |
| "k": 156, |
| "n": 200, |
| "p": 0.78, |
| "ci95": [ |
| 0.7176121367571497, |
| 0.8318345225722368 |
| ] |
| }, |
| "III": { |
| "k": 155, |
| "n": 198, |
| "p": 0.7828282828282829, |
| "ci95": [ |
| 0.7203091094117178, |
| 0.8345818893022281 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 153, |
| "n": 200, |
| "p": 0.765, |
| "ci95": [ |
| 0.7015864484736013, |
| 0.8184255683917109 |
| ] |
| }, |
| "answer_status": { |
| "k": 96, |
| "n": 200, |
| "p": 0.48, |
| "ci95": [ |
| 0.41179179908760966, |
| 0.5489620109602914 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 440, |
| "n": 521, |
| "p": 0.8445297504798465, |
| "ci95": [ |
| 0.810905474589728, |
| 0.8731106309412187 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 7, |
| "n": 23, |
| "p": 0.30434782608695654, |
| "ci95": [ |
| 0.15604046185489862, |
| 0.5086571502517591 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 13, |
| "n": 51, |
| "p": 0.2549019607843137, |
| "ci95": [ |
| 0.15545784189248602, |
| 0.38868252822690164 |
| ] |
| }, |
| "human": { |
| "k": 3, |
| "n": 9, |
| "p": 0.3333333333333333, |
| "ci95": [ |
| 0.12058406396498972, |
| 0.6457973346907151 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 7, |
| "n": 53, |
| "p": 0.1320754716981132, |
| "ci95": [ |
| 0.06547862476439556, |
| 0.24840227263508838 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 146, |
| "n": 147, |
| "p": 0.9931972789115646, |
| "ci95": [ |
| 0.9624761437110765, |
| 0.9987981382261968 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 113, |
| "contradicted->missing": 9, |
| "contradicted->supported": 9, |
| "contradicted->unclear": 2, |
| "missing->contradicted": 13, |
| "missing->missing": 72, |
| "missing->supported": 47, |
| "missing->unclear": 10, |
| "supported->contradicted": 5, |
| "supported->missing": 2, |
| "supported->supported": 270, |
| "supported->unclear": 6, |
| "unclear->contradicted": 7, |
| "unclear->missing": 3, |
| "unclear->supported": 28, |
| "unclear->unclear": 8 |
| } |
| }, |
| "paired_ft_minus_base": { |
| "all": { |
| "diff": 0.0825, |
| "ci95": [ |
| 0.04932451957550776, |
| 0.11567548042449224 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.06333333333333334, |
| "ci95": [ |
| 0.02673315576101877, |
| 0.09993351090564791 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| } |
| } |
