| { |
| "benchmark": "RuBench v1", |
| "split": "dev", |
| "n_answers": 200, |
| "n_criteria": 604, |
| "built_at": "2026-11-04T00:15:10", |
| "conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), temperature 0, max_tokens 1024, enable_thinking=false, no JSON schema enforcement. RuBio: SYSTEM_PROMPT only; base: SYSTEM_PROMPT+POLICY. Invalid JSON counts as wrong.", |
| "finetuned": { |
| "key": "8b-v6-s3407", |
| "label": "RuBio-8B", |
| "hf": "mulkzz/RuBio-8B@21e6bf3d", |
| "in_dist_train_dev_criterion_match": 0.9395109395109396, |
| "name": "RuBio-8B RuIso-v6 (μλ 3407)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 1, |
| "criterion_acc": { |
| "mean": 0.7595833333333333, |
| "ci95": [ |
| 0.7214769175142441, |
| 0.7976897491524224 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.7816666666666667, |
| "ci95": [ |
| 0.7422055735860975, |
| 0.821127759747236 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 458, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 86, |
| "n": 133, |
| "p": 0.6466165413533834, |
| "ci95": [ |
| 0.5623037851806594, |
| 0.7226975936965256 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 53, |
| "n": 79, |
| "p": 0.6708860759493671, |
| "ci95": [ |
| 0.5614659956181937, |
| 0.7644578289377567 |
| ] |
| }, |
| "json_valid": { |
| "k": 199, |
| "n": 200, |
| "p": 0.995, |
| "ci95": [ |
| 0.9722263725443985, |
| 0.9991168287700527 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 153, |
| "n": 206, |
| "p": 0.7427184466019418, |
| "ci95": [ |
| 0.6789634796217361, |
| 0.7975868074630104 |
| ] |
| }, |
| "II": { |
| "k": 154, |
| "n": 200, |
| "p": 0.77, |
| "ci95": [ |
| 0.7069170932155301, |
| 0.822906471137807 |
| ] |
| }, |
| "III": { |
| "k": 151, |
| "n": 198, |
| "p": 0.7626262626262627, |
| "ci95": [ |
| 0.6987187578979108, |
| 0.8165371694793246 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 155, |
| "n": 200, |
| "p": 0.775, |
| "ci95": [ |
| 0.7122589040583975, |
| 0.8273762077829643 |
| ] |
| }, |
| "answer_status": { |
| "k": 125, |
| "n": 200, |
| "p": 0.625, |
| "ci95": [ |
| 0.5561431792007809, |
| 0.6891455079998381 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 422, |
| "n": 521, |
| "p": 0.8099808061420346, |
| "ci95": [ |
| 0.7740717294391473, |
| 0.8413522318045736 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 13, |
| "n": 23, |
| "p": 0.5652173913043478, |
| "ci95": [ |
| 0.36811411127539473, |
| 0.7436533513557193 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 19, |
| "n": 51, |
| "p": 0.37254901960784315, |
| "ci95": [ |
| 0.25320540458646623, |
| 0.5097475878756154 |
| ] |
| }, |
| "human": { |
| "k": 4, |
| "n": 9, |
| "p": 0.4444444444444444, |
| "ci95": [ |
| 0.18877886314051356, |
| 0.7333482697447213 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 35, |
| "n": 53, |
| "p": 0.660377358490566, |
| "ci95": [ |
| 0.5259481435437473, |
| 0.7731294140000674 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 120, |
| "n": 147, |
| "p": 0.8163265306122449, |
| "ci95": [ |
| 0.745954432211289, |
| 0.8705870037898586 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->(μμ)": 1, |
| "contradicted->contradicted": 107, |
| "contradicted->missing": 13, |
| "contradicted->supported": 8, |
| "contradicted->unclear": 4, |
| "missing->(μμ)": 1, |
| "missing->contradicted": 21, |
| "missing->missing": 98, |
| "missing->supported": 18, |
| "missing->unclear": 4, |
| "supported->(μμ)": 2, |
| "supported->contradicted": 16, |
| "supported->missing": 18, |
| "supported->supported": 241, |
| "supported->unclear": 6, |
| "unclear->(μμ)": 1, |
| "unclear->contradicted": 9, |
| "unclear->missing": 11, |
| "unclear->supported": 13, |
| "unclear->unclear": 12 |
| } |
| }, |
| "base": { |
| "key": "8b-base", |
| "label": "Qwen3-8B-MegaScience (unfenced)", |
| "hf": "MegaScience/Qwen3-8B-MegaScience", |
| "serving": "RunPod A40 Pod, vLLM 0.32.0, bf16", |
| "name": "Qwen3-8B-MegaScience (κΈ°λ³Έ)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 8, |
| "criterion_acc": { |
| "mean": 0.5620833333333333, |
| "ci95": [ |
| 0.5168683002478879, |
| 0.6072983664187787 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.5779166666666666, |
| "ci95": [ |
| 0.5315152183706893, |
| 0.624318114962644 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 342, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 26, |
| "n": 133, |
| "p": 0.19548872180451127, |
| "ci95": [ |
| 0.1370439507829323, |
| 0.2710301084721449 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 26, |
| "n": 79, |
| "p": 0.3291139240506329, |
| "ci95": [ |
| 0.23554217106224323, |
| 0.4385340043818064 |
| ] |
| }, |
| "json_valid": { |
| "k": 192, |
| "n": 200, |
| "p": 0.96, |
| "ci95": [ |
| 0.9230680280606571, |
| 0.9795943408376206 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 120, |
| "n": 206, |
| "p": 0.5825242718446602, |
| "ci95": [ |
| 0.5142736573106079, |
| 0.6477534402982059 |
| ] |
| }, |
| "II": { |
| "k": 111, |
| "n": 200, |
| "p": 0.555, |
| "ci95": [ |
| 0.4857331643905435, |
| 0.622193857977729 |
| ] |
| }, |
| "III": { |
| "k": 111, |
| "n": 198, |
| "p": 0.5606060606060606, |
| "ci95": [ |
| 0.49097326500716265, |
| 0.6279319490029686 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 141, |
| "n": 200, |
| "p": 0.705, |
| "ci95": [ |
| 0.6384129957526774, |
| 0.7638604512563376 |
| ] |
| }, |
| "answer_status": { |
| "k": 44, |
| "n": 200, |
| "p": 0.22, |
| "ci95": [ |
| 0.16816547742776328, |
| 0.2823878632428503 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 318, |
| "n": 521, |
| "p": 0.6103646833013435, |
| "ci95": [ |
| 0.5678278575107883, |
| 0.651285936740072 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 4, |
| "n": 23, |
| "p": 0.17391304347826086, |
| "ci95": [ |
| 0.06978666088359675, |
| 0.37137602596083263 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 17, |
| "n": 51, |
| "p": 0.3333333333333333, |
| "ci95": [ |
| 0.21969951335330842, |
| 0.4703159383278752 |
| ] |
| }, |
| "human": { |
| "k": 3, |
| "n": 9, |
| "p": 0.3333333333333333, |
| "ci95": [ |
| 0.12058406396498972, |
| 0.6457973346907151 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 0, |
| "n": 53, |
| "p": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.06758173236340116 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 141, |
| "n": 147, |
| "p": 0.9591836734693877, |
| "ci95": [ |
| 0.9138180937286629, |
| 0.9811614101439707 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->(μμ)": 2, |
| "contradicted->contradicted": 37, |
| "contradicted->missing": 21, |
| "contradicted->supported": 73, |
| "missing->(μμ)": 7, |
| "missing->contradicted": 7, |
| "missing->missing": 42, |
| "missing->supported": 86, |
| "supported->(μμ)": 13, |
| "supported->contradicted": 2, |
| "supported->missing": 5, |
| "supported->supported": 263, |
| "unclear->contradicted": 3, |
| "unclear->missing": 8, |
| "unclear->supported": 35 |
| }, |
| "parse": "unfenced (```json code fence removed before json.loads)", |
| "strict": { |
| "criterion_acc": { |
| "mean": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "n_answers": 200 |
| }, |
| "json_invalid": 200, |
| "note": "official strict scoring (fenced JSON counts as invalid)" |
| } |
| }, |
| "paired_ft_minus_base": { |
| "all": { |
| "diff": 0.1975, |
| "ci95": [ |
| 0.1481006807023753, |
| 0.2468993192976247 |
| ], |
| "n": 200, |
| "verdict": "κ°μ " |
| }, |
| "no_majority": { |
| "diff": 0.20375, |
| "ci95": [ |
| 0.1486619394502655, |
| 0.25883806054973446 |
| ], |
| "n": 200, |
| "verdict": "κ°μ " |
| } |
| } |
| } |
