RuBio-27B / figures /data /rubench_v1_dev200_27b.json
mulkzz's picture
main 가중치를 RuIso-v6 재학습본(ruiso-v6@c1cfefdb, 시드 3407)으로 교체하고 카드·그림 갱신(Gemini 4 Argon High 작성)
fa96259 verified
Raw History Blame Contribute Delete
7.19 kB
{
"benchmark": "RuBench v1",
"split": "dev",
"n_answers": 200,
"n_criteria": 604,
"built_at": "2026-11-04T00:34:27",
"conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), temperature 0, max_tokens 1024, enable_thinking=false, no JSON schema enforcement. RuBio: SYSTEM_PROMPT only; base: SYSTEM_PROMPT+POLICY. Invalid JSON counts as wrong.",
"finetuned": {
"key": "27b-v6-s3407",
"label": "RuBio-27B",
"hf": "mulkzz/RuBio-27B@c1cfefdb",
"in_dist_train_dev_criterion_match": 0.9769230769230769,
"name": "RuBio-27B RuIso-v6 (시드 3407)",
"answers": 200,
"missing_output": 0,
"json_invalid": 0,
"criterion_acc": {
"mean": 0.86875,
"ci95": [
0.8381287620959553,
0.8993712379040447
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.9,
"ci95": [
0.8714503967862466,
0.9285496032137535
],
"n_answers": 200
},
"criterion_micro": {
"hit": 524,
"n": 604
},
"misconception": {
"k": 118,
"n": 133,
"p": 0.8872180451127819,
"ci95": [
0.8222450524506153,
0.9304508968916938
]
},
"misconception_with_id": {
"k": 73,
"n": 79,
"p": 0.9240506329113924,
"ci95": [
0.8440460366216044,
0.9647278983876056
]
},
"json_valid": {
"k": 200,
"n": 200,
"p": 1.0,
"ci95": [
0.981154748802476,
1.0
]
},
"unit_criterion_micro": {
"I": {
"k": 185,
"n": 206,
"p": 0.8980582524271845,
"ci95": [
0.8491894424118119,
0.9323530284071725
]
},
"II": {
"k": 166,
"n": 200,
"p": 0.83,
"ci95": [
0.7718412936135373,
0.8757208405960968
]
},
"III": {
"k": 173,
"n": 198,
"p": 0.8737373737373737,
"ci95": [
0.8202540537968194,
0.9129947659323233
]
}
},
"other_issues": {
"k": 173,
"n": 200,
"p": 0.865,
"ci95": [
0.810708740295804,
0.9055342263300035
]
},
"answer_status": {
"k": 154,
"n": 200,
"p": 0.77,
"ci95": [
0.7069170932155301,
0.822906471137807
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 483,
"n": 521,
"p": 0.927063339731286,
"ci95": [
0.9014725811198147,
0.9464025357639493
]
},
"r2-3/3": {
"k": 11,
"n": 23,
"p": 0.4782608695652174,
"ci95": [
0.29237189265791597,
0.670372286465046
]
},
"r2-2/3": {
"k": 23,
"n": 51,
"p": 0.45098039215686275,
"ci95": [
0.32267542174059904,
0.5861526522832785
]
},
"human": {
"k": 7,
"n": 9,
"p": 0.7777777777777778,
"ci95": [
0.45258959147532657,
0.9367747440984988
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 46,
"n": 53,
"p": 0.8679245283018868,
"ci95": [
0.7515977273649117,
0.9345213752356045
]
},
"no_error_specificity": {
"k": 127,
"n": 147,
"p": 0.8639455782312925,
"ci95": [
0.7991850746700077,
0.9101690506216352
]
}
},
"confusion": {
"contradicted->contradicted": 127,
"contradicted->missing": 4,
"contradicted->supported": 1,
"contradicted->unclear": 1,
"missing->contradicted": 8,
"missing->missing": 125,
"missing->supported": 2,
"missing->unclear": 7,
"supported->contradicted": 4,
"supported->missing": 20,
"supported->supported": 253,
"supported->unclear": 6,
"unclear->contradicted": 4,
"unclear->missing": 15,
"unclear->supported": 8,
"unclear->unclear": 19
}
},
"base": {
"key": "qwen27b",
"label": "Qwen3.8-27B (base)",
"hf": "Qwen/Qwen3.8-27B",
"serving": "OpenRouter qwen/qwen3.8-27b, DeepInfra bf16 고정",
"name": "Qwen3.8-27B (기본)",
"answers": 200,
"missing_output": 0,
"json_invalid": 5,
"criterion_acc": {
"mean": 0.8179166666666667,
"ci95": [
0.7816876269997497,
0.8541457063335838
],
"n_answers": 200
},
"criterion_acc_excl_r2_2of3": {
"mean": 0.8495833333333332,
"ci95": [
0.8142417703596521,
0.8849248963070144
],
"n_answers": 200
},
"criterion_micro": {
"hit": 495,
"n": 604
},
"misconception": {
"k": 107,
"n": 133,
"p": 0.8045112781954887,
"ci95": [
0.7289698915278552,
0.8629560492170677
]
},
"misconception_with_id": {
"k": 68,
"n": 79,
"p": 0.8607594936708861,
"ci95": [
0.7676269042027796,
0.9204345031931153
]
},
"json_valid": {
"k": 195,
"n": 200,
"p": 0.975,
"ci95": [
0.9428217485145032,
0.989275262847849
]
},
"unit_criterion_micro": {
"I": {
"k": 165,
"n": 206,
"p": 0.8009708737864077,
"ci95": [
0.7411592356541727,
0.8497631203309128
]
},
"II": {
"k": 167,
"n": 200,
"p": 0.835,
"ci95": [
0.777342313757747,
0.8800313679399118
]
},
"III": {
"k": 163,
"n": 198,
"p": 0.8232323232323232,
"ci95": [
0.764095715325512,
0.8700654260618547
]
}
},
"other_issues": {
"k": 156,
"n": 200,
"p": 0.78,
"ci95": [
0.7176121367571497,
0.8318345225722368
]
},
"answer_status": {
"k": 120,
"n": 200,
"p": 0.6,
"ci95": [
0.530836862213719,
0.6653940875467761
]
},
"gold_tier_criterion_acc": {
"r1-3/3": {
"k": 461,
"n": 521,
"p": 0.8848368522072937,
"ci95": [
0.854565276901407,
0.9094749969646364
]
},
"r2-3/3": {
"k": 5,
"n": 23,
"p": 0.21739130434782608,
"ci95": [
0.09663993783788177,
0.4190343907606237
]
},
"r2-2/3": {
"k": 24,
"n": 51,
"p": 0.47058823529411764,
"ci95": [
0.3405301441444071,
0.6047667002699194
]
},
"human": {
"k": 5,
"n": 9,
"p": 0.5555555555555556,
"ci95": [
0.26665173025527866,
0.8112211368594864
]
}
},
"other_issues_split": {
"has_error_recall": {
"k": 29,
"n": 53,
"p": 0.5471698113207547,
"ci95": [
0.41454001247278066,
0.6734239750401059
]
},
"no_error_specificity": {
"k": 127,
"n": 147,
"p": 0.8639455782312925,
"ci95": [
0.7991850746700077,
0.9101690506216352
]
}
},
"confusion": {
"contradicted->(없음)": 6,
"contradicted->contradicted": 112,
"contradicted->missing": 7,
"contradicted->supported": 5,
"contradicted->unclear": 3,
"missing->(없음)": 8,
"missing->contradicted": 3,
"missing->missing": 95,
"missing->supported": 20,
"missing->unclear": 16,
"supported->(없음)": 3,
"supported->contradicted": 4,
"supported->missing": 1,
"supported->supported": 269,
"supported->unclear": 6,
"unclear->(없음)": 1,
"unclear->missing": 3,
"unclear->supported": 23,
"unclear->unclear": 19
}
},
"paired_ft_minus_base": {
"all": {
"diff": 0.05083333333333334,
"ci95": [
0.010744829973651678,
0.09092183669301501
],
"n": 200,
"verdict": "개선"
},
"no_majority": {
"diff": 0.05041666666666667,
"ci95": [
0.01052750632121284,
0.0903058270121205
],
"n": 200,
"verdict": "개선"
}
}
}