{ "benchmark": "RuBench v1", "split": "dev", "n_answers": 200, "n_criteria": 604, "built_at": "2026-11-04T00:34:27", "conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), temperature 0, max_tokens 1024, enable_thinking=false, no JSON schema enforcement. RuBio: SYSTEM_PROMPT only; base: SYSTEM_PROMPT+POLICY. Invalid JSON counts as wrong.", "finetuned": { "key": "27b-v6-s3407", "label": "RuBio-27B", "hf": "mulkzz/RuBio-27B@c1cfefdb", "in_dist_train_dev_criterion_match": 0.9769230769230769, "name": "RuBio-27B RuIso-v6 (시드 3407)", "answers": 200, "missing_output": 0, "json_invalid": 0, "criterion_acc": { "mean": 0.86875, "ci95": [ 0.8381287620959553, 0.8993712379040447 ], "n_answers": 200 }, "criterion_acc_excl_r2_2of3": { "mean": 0.9, "ci95": [ 0.8714503967862466, 0.9285496032137535 ], "n_answers": 200 }, "criterion_micro": { "hit": 524, "n": 604 }, "misconception": { "k": 118, "n": 133, "p": 0.8872180451127819, "ci95": [ 0.8222450524506153, 0.9304508968916938 ] }, "misconception_with_id": { "k": 73, "n": 79, "p": 0.9240506329113924, "ci95": [ 0.8440460366216044, 0.9647278983876056 ] }, "json_valid": { "k": 200, "n": 200, "p": 1.0, "ci95": [ 0.981154748802476, 1.0 ] }, "unit_criterion_micro": { "I": { "k": 185, "n": 206, "p": 0.8980582524271845, "ci95": [ 0.8491894424118119, 0.9323530284071725 ] }, "II": { "k": 166, "n": 200, "p": 0.83, "ci95": [ 0.7718412936135373, 0.8757208405960968 ] }, "III": { "k": 173, "n": 198, "p": 0.8737373737373737, "ci95": [ 0.8202540537968194, 0.9129947659323233 ] } }, "other_issues": { "k": 173, "n": 200, "p": 0.865, "ci95": [ 0.810708740295804, 0.9055342263300035 ] }, "answer_status": { "k": 154, "n": 200, "p": 0.77, "ci95": [ 0.7069170932155301, 0.822906471137807 ] }, "gold_tier_criterion_acc": { "r1-3/3": { "k": 483, "n": 521, "p": 0.927063339731286, "ci95": [ 0.9014725811198147, 0.9464025357639493 ] }, "r2-3/3": { "k": 11, "n": 23, "p": 0.4782608695652174, "ci95": [ 0.29237189265791597, 0.670372286465046 ] }, "r2-2/3": { "k": 23, "n": 51, "p": 0.45098039215686275, "ci95": [ 0.32267542174059904, 0.5861526522832785 ] }, "human": { "k": 7, "n": 9, "p": 0.7777777777777778, "ci95": [ 0.45258959147532657, 0.9367747440984988 ] } }, "other_issues_split": { "has_error_recall": { "k": 46, "n": 53, "p": 0.8679245283018868, "ci95": [ 0.7515977273649117, 0.9345213752356045 ] }, "no_error_specificity": { "k": 127, "n": 147, "p": 0.8639455782312925, "ci95": [ 0.7991850746700077, 0.9101690506216352 ] } }, "confusion": { "contradicted->contradicted": 127, "contradicted->missing": 4, "contradicted->supported": 1, "contradicted->unclear": 1, "missing->contradicted": 8, "missing->missing": 125, "missing->supported": 2, "missing->unclear": 7, "supported->contradicted": 4, "supported->missing": 20, "supported->supported": 253, "supported->unclear": 6, "unclear->contradicted": 4, "unclear->missing": 15, "unclear->supported": 8, "unclear->unclear": 19 } }, "base": { "key": "qwen27b", "label": "Qwen3.8-27B (base)", "hf": "Qwen/Qwen3.8-27B", "serving": "OpenRouter qwen/qwen3.8-27b, DeepInfra bf16 고정", "name": "Qwen3.8-27B (기본)", "answers": 200, "missing_output": 0, "json_invalid": 5, "criterion_acc": { "mean": 0.8179166666666667, "ci95": [ 0.7816876269997497, 0.8541457063335838 ], "n_answers": 200 }, "criterion_acc_excl_r2_2of3": { "mean": 0.8495833333333332, "ci95": [ 0.8142417703596521, 0.8849248963070144 ], "n_answers": 200 }, "criterion_micro": { "hit": 495, "n": 604 }, "misconception": { "k": 107, "n": 133, "p": 0.8045112781954887, "ci95": [ 0.7289698915278552, 0.8629560492170677 ] }, "misconception_with_id": { "k": 68, "n": 79, "p": 0.8607594936708861, "ci95": [ 0.7676269042027796, 0.9204345031931153 ] }, "json_valid": { "k": 195, "n": 200, "p": 0.975, "ci95": [ 0.9428217485145032, 0.989275262847849 ] }, "unit_criterion_micro": { "I": { "k": 165, "n": 206, "p": 0.8009708737864077, "ci95": [ 0.7411592356541727, 0.8497631203309128 ] }, "II": { "k": 167, "n": 200, "p": 0.835, "ci95": [ 0.777342313757747, 0.8800313679399118 ] }, "III": { "k": 163, "n": 198, "p": 0.8232323232323232, "ci95": [ 0.764095715325512, 0.8700654260618547 ] } }, "other_issues": { "k": 156, "n": 200, "p": 0.78, "ci95": [ 0.7176121367571497, 0.8318345225722368 ] }, "answer_status": { "k": 120, "n": 200, "p": 0.6, "ci95": [ 0.530836862213719, 0.6653940875467761 ] }, "gold_tier_criterion_acc": { "r1-3/3": { "k": 461, "n": 521, "p": 0.8848368522072937, "ci95": [ 0.854565276901407, 0.9094749969646364 ] }, "r2-3/3": { "k": 5, "n": 23, "p": 0.21739130434782608, "ci95": [ 0.09663993783788177, 0.4190343907606237 ] }, "r2-2/3": { "k": 24, "n": 51, "p": 0.47058823529411764, "ci95": [ 0.3405301441444071, 0.6047667002699194 ] }, "human": { "k": 5, "n": 9, "p": 0.5555555555555556, "ci95": [ 0.26665173025527866, 0.8112211368594864 ] } }, "other_issues_split": { "has_error_recall": { "k": 29, "n": 53, "p": 0.5471698113207547, "ci95": [ 0.41454001247278066, 0.6734239750401059 ] }, "no_error_specificity": { "k": 127, "n": 147, "p": 0.8639455782312925, "ci95": [ 0.7991850746700077, 0.9101690506216352 ] } }, "confusion": { "contradicted->(없음)": 6, "contradicted->contradicted": 112, "contradicted->missing": 7, "contradicted->supported": 5, "contradicted->unclear": 3, "missing->(없음)": 8, "missing->contradicted": 3, "missing->missing": 95, "missing->supported": 20, "missing->unclear": 16, "supported->(없음)": 3, "supported->contradicted": 4, "supported->missing": 1, "supported->supported": 269, "supported->unclear": 6, "unclear->(없음)": 1, "unclear->missing": 3, "unclear->supported": 23, "unclear->unclear": 19 } }, "paired_ft_minus_base": { "all": { "diff": 0.05083333333333334, "ci95": [ 0.010744829973651678, 0.09092183669301501 ], "n": 200, "verdict": "개선" }, "no_majority": { "diff": 0.05041666666666667, "ci95": [ 0.01052750632121284, 0.0903058270121205 ], "n": 200, "verdict": "개선" } } }