| { |
| "benchmark": "RuBench v1", |
| "split": "dev", |
| "n_answers": 200, |
| "n_criteria": 604, |
| "built_at": "2026-11-04T00:15:58", |
| "conditions": "first turn (feedback_mode=hint_1, reveal_requested=false, history=[]), no JSON schema enforcement, invalid JSON counts as wrong. RuBio: SYSTEM_PROMPT only; base and general models: SYSTEM_PROMPT+POLICY. RuBio and Qwen base: temperature 0, max_tokens 1024, thinking off. General models: vendor CLI at the listed reasoning effort, one fixed run.", |
| "panel": { |
| "round1": [ |
| "gpt", |
| "opus", |
| "grok" |
| ], |
| "round2": [ |
| "gpt", |
| "opus", |
| "gemini" |
| ], |
| "student_answer_author": "gemini" |
| }, |
| "caveats": [ |
| "GPT and Opus were round-1 gold panelists: the r1-3/3 tier (521 of 604 dev criteria) contains their own blind judgments, so their scores include self-agreement and are upper bounds.", |
| "Round-2 majority (r2-2/3) items include GPT, Opus and Gemini votes; see criterion_acc_excl_r2_2of3.", |
| "Gemini wrote the synthetic student answers with intended labels; reported as reference only (DESIGN 4-1).", |
| "Grok 4.7 (round-1 panelist) was not evaluated as a model.", |
| "RuBio-27B is the RuIso-v6 retrain (main since 2026-11-04), run on a RunPod A100 Pod with vLLM 0.32.0 under the same generation settings; RuBio-4B v2 was run on RunPod Serverless (worker-v1-vllm, vLLM 0.32.0). The decision to replace the 27B main weights used this same dev split." |
| ], |
| "models": { |
| "4b": { |
| "key": "4b", |
| "label": "RuBio-4B v2", |
| "hf": "mulkzz/RuBio-4B@ba9e00c8", |
| "role": "finetuned", |
| "size": "4B", |
| "name": "RuBio-4B v2", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.7716666666666667, |
| "ci95": [ |
| 0.7339481076683567, |
| 0.8093852256649767 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.79875, |
| "ci95": [ |
| 0.759669235815335, |
| 0.837830764184665 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 467, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 106, |
| "n": 133, |
| "p": 0.7969924812030075, |
| "ci95": [ |
| 0.7207474553277116, |
| 0.8565630300901761 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 65, |
| "n": 79, |
| "p": 0.8227848101265823, |
| "ci95": [ |
| 0.7242382665030478, |
| 0.8913956243248582 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 156, |
| "n": 206, |
| "p": 0.7572815533980582, |
| "ci95": [ |
| 0.6943735885025178, |
| 0.8107697158073134 |
| ] |
| }, |
| "II": { |
| "k": 154, |
| "n": 200, |
| "p": 0.77, |
| "ci95": [ |
| 0.7069170932155301, |
| 0.822906471137807 |
| ] |
| }, |
| "III": { |
| "k": 157, |
| "n": 198, |
| "p": 0.7929292929292929, |
| "ci95": [ |
| 0.7311760804230768, |
| 0.8435324539592243 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 153, |
| "n": 200, |
| "p": 0.765, |
| "ci95": [ |
| 0.7015864484736013, |
| 0.8184255683917109 |
| ] |
| }, |
| "answer_status": { |
| "k": 124, |
| "n": 200, |
| "p": 0.62, |
| "ci95": [ |
| 0.5510673223171103, |
| 0.684409817395484 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 431, |
| "n": 521, |
| "p": 0.8272552783109405, |
| "ci95": [ |
| 0.7924302993782009, |
| 0.8572897340091328 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 10, |
| "n": 23, |
| "p": 0.43478260869565216, |
| "ci95": [ |
| 0.2563466486442807, |
| 0.6318858887246053 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 23, |
| "n": 51, |
| "p": 0.45098039215686275, |
| "ci95": [ |
| 0.32267542174059904, |
| 0.5861526522832785 |
| ] |
| }, |
| "human": { |
| "k": 3, |
| "n": 9, |
| "p": 0.3333333333333333, |
| "ci95": [ |
| 0.12058406396498972, |
| 0.6457973346907151 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 10, |
| "n": 53, |
| "p": 0.18867924528301888, |
| "ci95": [ |
| 0.10585337627628057, |
| 0.3135843061386674 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 143, |
| "n": 147, |
| "p": 0.9727891156462585, |
| "ci95": [ |
| 0.9321289915585997, |
| 0.9893684235398896 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 121, |
| "contradicted->missing": 5, |
| "contradicted->supported": 3, |
| "contradicted->unclear": 4, |
| "missing->contradicted": 12, |
| "missing->missing": 92, |
| "missing->supported": 18, |
| "missing->unclear": 20, |
| "supported->contradicted": 14, |
| "supported->missing": 9, |
| "supported->supported": 232, |
| "supported->unclear": 28, |
| "unclear->contradicted": 8, |
| "unclear->missing": 7, |
| "unclear->supported": 9, |
| "unclear->unclear": 22 |
| }, |
| "cli_seconds_p50": null |
| }, |
| "qwen4b": { |
| "key": "qwen4b", |
| "label": "Qwen3.5-4B (base)", |
| "hf": "Qwen/Qwen3.5-4B", |
| "role": "base", |
| "size": "4B", |
| "name": "Qwen3.5-4B (기본)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.7091666666666667, |
| "ci95": [ |
| 0.673031876586196, |
| 0.7453014567471374 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.7454166666666667, |
| "ci95": [ |
| 0.7095199602308042, |
| 0.7813133731025292 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 427, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 78, |
| "n": 133, |
| "p": 0.5864661654135338, |
| "ci95": [ |
| 0.501491500687543, |
| 0.666586235573361 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 55, |
| "n": 79, |
| "p": 0.6962025316455697, |
| "ci95": [ |
| 0.5876536225817802, |
| 0.7865552130194958 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 144, |
| "n": 206, |
| "p": 0.6990291262135923, |
| "ci95": [ |
| 0.633218801872325, |
| 0.7575524335371671 |
| ] |
| }, |
| "II": { |
| "k": 142, |
| "n": 200, |
| "p": 0.71, |
| "ci95": [ |
| 0.6436253597813291, |
| 0.7684596347157107 |
| ] |
| }, |
| "III": { |
| "k": 141, |
| "n": 198, |
| "p": 0.7121212121212122, |
| "ci95": [ |
| 0.6454906562919455, |
| 0.770677592743514 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 150, |
| "n": 200, |
| "p": 0.75, |
| "ci95": [ |
| 0.6856591553575013, |
| 0.8049182190437367 |
| ] |
| }, |
| "answer_status": { |
| "k": 89, |
| "n": 200, |
| "p": 0.445, |
| "ci95": [ |
| 0.37780614202227114, |
| 0.5142668356094566 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 402, |
| "n": 521, |
| "p": 0.7715930902111324, |
| "ci95": [ |
| 0.7336347625660318, |
| 0.8055757050252159 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 4, |
| "n": 23, |
| "p": 0.17391304347826086, |
| "ci95": [ |
| 0.06978666088359675, |
| 0.37137602596083263 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 15, |
| "n": 51, |
| "p": 0.29411764705882354, |
| "ci95": [ |
| 0.1870874425221019, |
| 0.4299904683781837 |
| ] |
| }, |
| "human": { |
| "k": 6, |
| "n": 9, |
| "p": 0.6666666666666666, |
| "ci95": [ |
| 0.35420266530928485, |
| 0.8794159360350102 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 4, |
| "n": 53, |
| "p": 0.07547169811320754, |
| "ci95": [ |
| 0.029739608356547, |
| 0.17858450402747283 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 146, |
| "n": 147, |
| "p": 0.9931972789115646, |
| "ci95": [ |
| 0.9624761437110765, |
| 0.9987981382261968 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 100, |
| "contradicted->missing": 20, |
| "contradicted->supported": 10, |
| "contradicted->unclear": 3, |
| "missing->contradicted": 22, |
| "missing->missing": 79, |
| "missing->supported": 37, |
| "missing->unclear": 4, |
| "supported->contradicted": 12, |
| "supported->missing": 16, |
| "supported->supported": 244, |
| "supported->unclear": 11, |
| "unclear->contradicted": 7, |
| "unclear->missing": 9, |
| "unclear->supported": 26, |
| "unclear->unclear": 4 |
| }, |
| "cli_seconds_p50": null |
| }, |
| "27b": { |
| "key": "27b", |
| "src": "27b-v6-s3407", |
| "label": "RuBio-27B", |
| "hf": "mulkzz/RuBio-27B@c1cfefdb", |
| "role": "finetuned", |
| "size": "27B", |
| "name": "RuBio-27B RuIso-v6 (시드 3407)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.86875, |
| "ci95": [ |
| 0.8381287620959553, |
| 0.8993712379040447 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.9, |
| "ci95": [ |
| 0.8714503967862466, |
| 0.9285496032137535 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 524, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 118, |
| "n": 133, |
| "p": 0.8872180451127819, |
| "ci95": [ |
| 0.8222450524506153, |
| 0.9304508968916938 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 73, |
| "n": 79, |
| "p": 0.9240506329113924, |
| "ci95": [ |
| 0.8440460366216044, |
| 0.9647278983876056 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 185, |
| "n": 206, |
| "p": 0.8980582524271845, |
| "ci95": [ |
| 0.8491894424118119, |
| 0.9323530284071725 |
| ] |
| }, |
| "II": { |
| "k": 166, |
| "n": 200, |
| "p": 0.83, |
| "ci95": [ |
| 0.7718412936135373, |
| 0.8757208405960968 |
| ] |
| }, |
| "III": { |
| "k": 173, |
| "n": 198, |
| "p": 0.8737373737373737, |
| "ci95": [ |
| 0.8202540537968194, |
| 0.9129947659323233 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 173, |
| "n": 200, |
| "p": 0.865, |
| "ci95": [ |
| 0.810708740295804, |
| 0.9055342263300035 |
| ] |
| }, |
| "answer_status": { |
| "k": 154, |
| "n": 200, |
| "p": 0.77, |
| "ci95": [ |
| 0.7069170932155301, |
| 0.822906471137807 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 483, |
| "n": 521, |
| "p": 0.927063339731286, |
| "ci95": [ |
| 0.9014725811198147, |
| 0.9464025357639493 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 11, |
| "n": 23, |
| "p": 0.4782608695652174, |
| "ci95": [ |
| 0.29237189265791597, |
| 0.670372286465046 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 23, |
| "n": 51, |
| "p": 0.45098039215686275, |
| "ci95": [ |
| 0.32267542174059904, |
| 0.5861526522832785 |
| ] |
| }, |
| "human": { |
| "k": 7, |
| "n": 9, |
| "p": 0.7777777777777778, |
| "ci95": [ |
| 0.45258959147532657, |
| 0.9367747440984988 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 46, |
| "n": 53, |
| "p": 0.8679245283018868, |
| "ci95": [ |
| 0.7515977273649117, |
| 0.9345213752356045 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 127, |
| "n": 147, |
| "p": 0.8639455782312925, |
| "ci95": [ |
| 0.7991850746700077, |
| 0.9101690506216352 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 127, |
| "contradicted->missing": 4, |
| "contradicted->supported": 1, |
| "contradicted->unclear": 1, |
| "missing->contradicted": 8, |
| "missing->missing": 125, |
| "missing->supported": 2, |
| "missing->unclear": 7, |
| "supported->contradicted": 4, |
| "supported->missing": 20, |
| "supported->supported": 253, |
| "supported->unclear": 6, |
| "unclear->contradicted": 4, |
| "unclear->missing": 15, |
| "unclear->supported": 8, |
| "unclear->unclear": 19 |
| }, |
| "cli_seconds_p50": null |
| }, |
| "qwen27b": { |
| "key": "qwen27b", |
| "label": "Qwen3.8-27B (base)", |
| "hf": "Qwen/Qwen3.8-27B", |
| "role": "base", |
| "size": "27B", |
| "name": "Qwen3.8-27B (기본)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 5, |
| "criterion_acc": { |
| "mean": 0.8179166666666667, |
| "ci95": [ |
| 0.7816876269997497, |
| 0.8541457063335838 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.8495833333333332, |
| "ci95": [ |
| 0.8142417703596521, |
| 0.8849248963070144 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 495, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 107, |
| "n": 133, |
| "p": 0.8045112781954887, |
| "ci95": [ |
| 0.7289698915278552, |
| 0.8629560492170677 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 68, |
| "n": 79, |
| "p": 0.8607594936708861, |
| "ci95": [ |
| 0.7676269042027796, |
| 0.9204345031931153 |
| ] |
| }, |
| "json_valid": { |
| "k": 195, |
| "n": 200, |
| "p": 0.975, |
| "ci95": [ |
| 0.9428217485145032, |
| 0.989275262847849 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 165, |
| "n": 206, |
| "p": 0.8009708737864077, |
| "ci95": [ |
| 0.7411592356541727, |
| 0.8497631203309128 |
| ] |
| }, |
| "II": { |
| "k": 167, |
| "n": 200, |
| "p": 0.835, |
| "ci95": [ |
| 0.777342313757747, |
| 0.8800313679399118 |
| ] |
| }, |
| "III": { |
| "k": 163, |
| "n": 198, |
| "p": 0.8232323232323232, |
| "ci95": [ |
| 0.764095715325512, |
| 0.8700654260618547 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 156, |
| "n": 200, |
| "p": 0.78, |
| "ci95": [ |
| 0.7176121367571497, |
| 0.8318345225722368 |
| ] |
| }, |
| "answer_status": { |
| "k": 120, |
| "n": 200, |
| "p": 0.6, |
| "ci95": [ |
| 0.530836862213719, |
| 0.6653940875467761 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 461, |
| "n": 521, |
| "p": 0.8848368522072937, |
| "ci95": [ |
| 0.854565276901407, |
| 0.9094749969646364 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 5, |
| "n": 23, |
| "p": 0.21739130434782608, |
| "ci95": [ |
| 0.09663993783788177, |
| 0.4190343907606237 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 24, |
| "n": 51, |
| "p": 0.47058823529411764, |
| "ci95": [ |
| 0.3405301441444071, |
| 0.6047667002699194 |
| ] |
| }, |
| "human": { |
| "k": 5, |
| "n": 9, |
| "p": 0.5555555555555556, |
| "ci95": [ |
| 0.26665173025527866, |
| 0.8112211368594864 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 29, |
| "n": 53, |
| "p": 0.5471698113207547, |
| "ci95": [ |
| 0.41454001247278066, |
| 0.6734239750401059 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 127, |
| "n": 147, |
| "p": 0.8639455782312925, |
| "ci95": [ |
| 0.7991850746700077, |
| 0.9101690506216352 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->(없음)": 6, |
| "contradicted->contradicted": 112, |
| "contradicted->missing": 7, |
| "contradicted->supported": 5, |
| "contradicted->unclear": 3, |
| "missing->(없음)": 8, |
| "missing->contradicted": 3, |
| "missing->missing": 95, |
| "missing->supported": 20, |
| "missing->unclear": 16, |
| "supported->(없음)": 3, |
| "supported->contradicted": 4, |
| "supported->missing": 1, |
| "supported->supported": 269, |
| "supported->unclear": 6, |
| "unclear->(없음)": 1, |
| "unclear->missing": 3, |
| "unclear->supported": 23, |
| "unclear->unclear": 19 |
| }, |
| "cli_seconds_p50": 9.7 |
| }, |
| "gpt": { |
| "key": "gpt", |
| "label": "GPT-6 Astra (xhigh)", |
| "hf": null, |
| "role": "general", |
| "size": null, |
| "name": "GPT-6 Astra (xhigh)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.9116666666666667, |
| "ci95": [ |
| 0.8858637301001623, |
| 0.9374696032331712 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.9558333333333333, |
| "ci95": [ |
| 0.9362898427686828, |
| 0.9753768238979839 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 550, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 129, |
| "n": 133, |
| "p": 0.9699248120300752, |
| "ci95": [ |
| 0.9252225048958136, |
| 0.9882434530438822 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 78, |
| "n": 79, |
| "p": 0.9873417721518988, |
| "ci95": [ |
| 0.9317244628529116, |
| 0.9977619997696132 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 184, |
| "n": 206, |
| "p": 0.8932038834951457, |
| "ci95": [ |
| 0.8436020225775144, |
| 0.928409442499775 |
| ] |
| }, |
| "II": { |
| "k": 185, |
| "n": 200, |
| "p": 0.925, |
| "ci95": [ |
| 0.8799564990407998, |
| 0.9540250374413048 |
| ] |
| }, |
| "III": { |
| "k": 181, |
| "n": 198, |
| "p": 0.9141414141414141, |
| "ci95": [ |
| 0.866814758905608, |
| 0.9457042034969555 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 177, |
| "n": 200, |
| "p": 0.885, |
| "ci95": [ |
| 0.8333529508707886, |
| 0.9221362057071179 |
| ] |
| }, |
| "answer_status": { |
| "k": 176, |
| "n": 200, |
| "p": 0.88, |
| "ci95": [ |
| 0.8276575999704266, |
| 0.9180200091194551 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 510, |
| "n": 521, |
| "p": 0.9788867562380038, |
| "ci95": [ |
| 0.9625927664904859, |
| 0.9881705668241167 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 14, |
| "n": 23, |
| "p": 0.6086956521739131, |
| "ci95": [ |
| 0.4078556031551567, |
| 0.7784235012300337 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 22, |
| "n": 51, |
| "p": 0.43137254901960786, |
| "ci95": [ |
| 0.30501259121603025, |
| 0.5673467124173983 |
| ] |
| }, |
| "human": { |
| "k": 4, |
| "n": 9, |
| "p": 0.4444444444444444, |
| "ci95": [ |
| 0.18877886314051356, |
| 0.7333482697447213 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 33, |
| "n": 53, |
| "p": 0.6226415094339622, |
| "ci95": [ |
| 0.48806923125215484, |
| 0.7406371362813504 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 144, |
| "n": 147, |
| "p": 0.9795918367346939, |
| "ci95": [ |
| 0.9417208980015789, |
| 0.9930354727098384 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 129, |
| "contradicted->missing": 3, |
| "contradicted->unclear": 1, |
| "missing->contradicted": 1, |
| "missing->missing": 135, |
| "missing->supported": 2, |
| "missing->unclear": 4, |
| "supported->contradicted": 3, |
| "supported->missing": 11, |
| "supported->supported": 269, |
| "unclear->contradicted": 2, |
| "unclear->missing": 23, |
| "unclear->supported": 4, |
| "unclear->unclear": 17 |
| }, |
| "cli_seconds_p50": 575.6 |
| }, |
| "opus": { |
| "key": "opus", |
| "label": "Claude Opus 5.5 (xhigh)", |
| "hf": null, |
| "role": "general", |
| "size": null, |
| "name": "Claude Opus 5.5 (xhigh)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 0, |
| "criterion_acc": { |
| "mean": 0.9420833333333333, |
| "ci95": [ |
| 0.9203542491963614, |
| 0.9638124174703051 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.9608333333333333, |
| "ci95": [ |
| 0.9417409178385642, |
| 0.9799257488281025 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 570, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 127, |
| "n": 133, |
| "p": 0.9548872180451128, |
| "ci95": [ |
| 0.9050722874953598, |
| 0.9791627597902657 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 77, |
| "n": 79, |
| "p": 0.9746835443037974, |
| "ci95": [ |
| 0.9123143405749654, |
| 0.9930296165248962 |
| ] |
| }, |
| "json_valid": { |
| "k": 200, |
| "n": 200, |
| "p": 1.0, |
| "ci95": [ |
| 0.981154748802476, |
| 1.0 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 193, |
| "n": 206, |
| "p": 0.9368932038834952, |
| "ci95": [ |
| 0.895037892168079, |
| 0.9627526245844646 |
| ] |
| }, |
| "II": { |
| "k": 189, |
| "n": 200, |
| "p": 0.945, |
| "ci95": [ |
| 0.9042131036406227, |
| 0.9690146227935807 |
| ] |
| }, |
| "III": { |
| "k": 188, |
| "n": 198, |
| "p": 0.9494949494949495, |
| "ci95": [ |
| 0.9095419269028148, |
| 0.9723384103389922 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 195, |
| "n": 200, |
| "p": 0.975, |
| "ci95": [ |
| 0.9428217485145032, |
| 0.989275262847849 |
| ] |
| }, |
| "answer_status": { |
| "k": 184, |
| "n": 200, |
| "p": 0.92, |
| "ci95": [ |
| 0.8740106225357106, |
| 0.9501593664583693 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 510, |
| "n": 521, |
| "p": 0.9788867562380038, |
| "ci95": [ |
| 0.9625927664904859, |
| 0.9881705668241167 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 18, |
| "n": 23, |
| "p": 0.782608695652174, |
| "ci95": [ |
| 0.5809656092393763, |
| 0.9033600621621182 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 37, |
| "n": 51, |
| "p": 0.7254901960784313, |
| "ci95": [ |
| 0.5905337813556237, |
| 0.8288570781345395 |
| ] |
| }, |
| "human": { |
| "k": 5, |
| "n": 9, |
| "p": 0.5555555555555556, |
| "ci95": [ |
| 0.26665173025527866, |
| 0.8112211368594864 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 52, |
| "n": 53, |
| "p": 0.9811320754716981, |
| "ci95": [ |
| 0.9005711355345968, |
| 0.9966615370968476 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 143, |
| "n": 147, |
| "p": 0.9727891156462585, |
| "ci95": [ |
| 0.9321289915585997, |
| 0.9893684235398896 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 128, |
| "contradicted->missing": 3, |
| "contradicted->supported": 1, |
| "contradicted->unclear": 1, |
| "missing->contradicted": 1, |
| "missing->missing": 123, |
| "missing->supported": 7, |
| "missing->unclear": 11, |
| "supported->missing": 3, |
| "supported->supported": 279, |
| "supported->unclear": 1, |
| "unclear->supported": 6, |
| "unclear->unclear": 40 |
| }, |
| "cli_seconds_p50": 41.3 |
| }, |
| "gemini": { |
| "key": "gemini", |
| "label": "Gemini 4 Argon (High)", |
| "hf": null, |
| "role": "reference", |
| "size": null, |
| "name": "Gemini 4 Argon (High)", |
| "answers": 200, |
| "missing_output": 0, |
| "json_invalid": 2, |
| "criterion_acc": { |
| "mean": 0.9125, |
| "ci95": [ |
| 0.8850333281496484, |
| 0.9399666718503515 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.9358333333333333, |
| "ci95": [ |
| 0.9098731921940174, |
| 0.9617934744726492 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_micro": { |
| "hit": 551, |
| "n": 604 |
| }, |
| "misconception": { |
| "k": 122, |
| "n": 133, |
| "p": 0.9172932330827067, |
| "ci95": [ |
| 0.8579662495837602, |
| 0.9531915210666896 |
| ] |
| }, |
| "misconception_with_id": { |
| "k": 75, |
| "n": 79, |
| "p": 0.9493670886075949, |
| "ci95": [ |
| 0.8769230243999817, |
| 0.980135921654554 |
| ] |
| }, |
| "json_valid": { |
| "k": 198, |
| "n": 200, |
| "p": 0.99, |
| "ci95": [ |
| 0.9642783183248235, |
| 0.997253335501603 |
| ] |
| }, |
| "unit_criterion_micro": { |
| "I": { |
| "k": 187, |
| "n": 206, |
| "p": 0.9077669902912622, |
| "ci95": [ |
| 0.8604469235517486, |
| 0.9401575587506256 |
| ] |
| }, |
| "II": { |
| "k": 184, |
| "n": 200, |
| "p": 0.92, |
| "ci95": [ |
| 0.8740106225357106, |
| 0.9501593664583693 |
| ] |
| }, |
| "III": { |
| "k": 180, |
| "n": 198, |
| "p": 0.9090909090909091, |
| "ci95": [ |
| 0.8608883910901738, |
| 0.9417218034782121 |
| ] |
| } |
| }, |
| "other_issues": { |
| "k": 190, |
| "n": 200, |
| "p": 0.95, |
| "ci95": [ |
| 0.9104219523342212, |
| 0.9726173215880072 |
| ] |
| }, |
| "answer_status": { |
| "k": 179, |
| "n": 200, |
| "p": 0.895, |
| "ci95": [ |
| 0.8448197976153835, |
| 0.9302924539385726 |
| ] |
| }, |
| "gold_tier_criterion_acc": { |
| "r1-3/3": { |
| "k": 496, |
| "n": 521, |
| "p": 0.9520153550863724, |
| "ci95": [ |
| 0.9301245809465594, |
| 0.9672893068113121 |
| ] |
| }, |
| "r2-3/3": { |
| "k": 19, |
| "n": 23, |
| "p": 0.8260869565217391, |
| "ci95": [ |
| 0.6286239740391674, |
| 0.9302133391164032 |
| ] |
| }, |
| "r2-2/3": { |
| "k": 32, |
| "n": 51, |
| "p": 0.6274509803921569, |
| "ci95": [ |
| 0.49025241212438464, |
| 0.7467945954135337 |
| ] |
| }, |
| "human": { |
| "k": 4, |
| "n": 9, |
| "p": 0.4444444444444444, |
| "ci95": [ |
| 0.18877886314051356, |
| 0.7333482697447213 |
| ] |
| } |
| }, |
| "other_issues_split": { |
| "has_error_recall": { |
| "k": 45, |
| "n": 53, |
| "p": 0.8490566037735849, |
| "ci95": [ |
| 0.7294577896504801, |
| 0.9214757179448814 |
| ] |
| }, |
| "no_error_specificity": { |
| "k": 145, |
| "n": 147, |
| "p": 0.9863945578231292, |
| "ci95": [ |
| 0.9517544022691328, |
| 0.9962609240552124 |
| ] |
| } |
| }, |
| "confusion": { |
| "contradicted->contradicted": 125, |
| "contradicted->missing": 5, |
| "contradicted->supported": 1, |
| "contradicted->unclear": 2, |
| "missing->(없음)": 2, |
| "missing->missing": 125, |
| "missing->supported": 10, |
| "missing->unclear": 5, |
| "supported->(없음)": 3, |
| "supported->contradicted": 2, |
| "supported->missing": 6, |
| "supported->supported": 267, |
| "supported->unclear": 5, |
| "unclear->(없음)": 1, |
| "unclear->contradicted": 1, |
| "unclear->missing": 9, |
| "unclear->supported": 1, |
| "unclear->unclear": 34 |
| }, |
| "cli_seconds_p50": 87.9 |
| } |
| }, |
| "paired": { |
| "qwen4b-minus-4b": { |
| "all": { |
| "diff": -0.06250000000000001, |
| "ci95": [ |
| -0.09929215430834135, |
| -0.02570784569165868 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.05333333333333334, |
| "ci95": [ |
| -0.0949195943790324, |
| -0.011747072287634275 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "27b-minus-4b": { |
| "all": { |
| "diff": 0.09708333333333334, |
| "ci95": [ |
| 0.06123184381518337, |
| 0.1329348228514833 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.10125, |
| "ci95": [ |
| 0.06521017094468781, |
| 0.1372898290553122 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "qwen27b-minus-4b": { |
| "all": { |
| "diff": 0.04625, |
| "ci95": [ |
| 0.007906503441078273, |
| 0.08459349655892173 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.05083333333333334, |
| "ci95": [ |
| 0.008389559707535299, |
| 0.09327710695913138 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gpt-minus-4b": { |
| "all": { |
| "diff": 0.14, |
| "ci95": [ |
| 0.1028811816212637, |
| 0.17711881837873633 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.15708333333333335, |
| "ci95": [ |
| 0.11841813107448726, |
| 0.19574853559217945 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "opus-minus-4b": { |
| "all": { |
| "diff": 0.1704166666666667, |
| "ci95": [ |
| 0.1349501475307221, |
| 0.20588318580261128 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.16208333333333336, |
| "ci95": [ |
| 0.12777146352786023, |
| 0.19639520313880648 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gemini-minus-4b": { |
| "all": { |
| "diff": 0.14083333333333334, |
| "ci95": [ |
| 0.10129028178518129, |
| 0.18037638488148539 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.13708333333333333, |
| "ci95": [ |
| 0.0986490412756628, |
| 0.17551762539100385 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "4b-minus-qwen4b": { |
| "all": { |
| "diff": 0.06250000000000001, |
| "ci95": [ |
| 0.02570784569165868, |
| 0.09929215430834135 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.05333333333333334, |
| "ci95": [ |
| 0.011747072287634275, |
| 0.0949195943790324 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "27b-minus-qwen4b": { |
| "all": { |
| "diff": 0.15958333333333333, |
| "ci95": [ |
| 0.12437326122557398, |
| 0.1947934054410927 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.15458333333333335, |
| "ci95": [ |
| 0.11312043866157052, |
| 0.1960462280050962 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "qwen27b-minus-qwen4b": { |
| "all": { |
| "diff": 0.10875, |
| "ci95": [ |
| 0.06948238748995589, |
| 0.1480176125100441 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.10416666666666669, |
| "ci95": [ |
| 0.06221595810731039, |
| 0.14611737522602297 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gpt-minus-qwen4b": { |
| "all": { |
| "diff": 0.2025, |
| "ci95": [ |
| 0.16509784766297997, |
| 0.23990215233702006 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.21041666666666667, |
| "ci95": [ |
| 0.1727924563143645, |
| 0.24804087701896885 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "opus-minus-qwen4b": { |
| "all": { |
| "diff": 0.2329166666666667, |
| "ci95": [ |
| 0.1962971271528449, |
| 0.26953620618048846 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.21541666666666667, |
| "ci95": [ |
| 0.17708051297608593, |
| 0.25375282035724744 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gemini-minus-qwen4b": { |
| "all": { |
| "diff": 0.20333333333333337, |
| "ci95": [ |
| 0.16215851002079762, |
| 0.2445081566458691 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.19041666666666668, |
| "ci95": [ |
| 0.14755506490751477, |
| 0.23327826842581859 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "4b-minus-27b": { |
| "all": { |
| "diff": -0.09708333333333334, |
| "ci95": [ |
| -0.1329348228514833, |
| -0.06123184381518337 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.10125, |
| "ci95": [ |
| -0.1372898290553122, |
| -0.06521017094468781 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen4b-minus-27b": { |
| "all": { |
| "diff": -0.15958333333333333, |
| "ci95": [ |
| -0.1947934054410927, |
| -0.12437326122557398 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.15458333333333335, |
| "ci95": [ |
| -0.1960462280050962, |
| -0.11312043866157052 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen27b-minus-27b": { |
| "all": { |
| "diff": -0.05083333333333334, |
| "ci95": [ |
| -0.09092183669301501, |
| -0.010744829973651678 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.05041666666666667, |
| "ci95": [ |
| -0.0903058270121205, |
| -0.01052750632121284 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "gpt-minus-27b": { |
| "all": { |
| "diff": 0.04291666666666667, |
| "ci95": [ |
| 0.01078839936599886, |
| 0.07504493396733448 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.05583333333333334, |
| "ci95": [ |
| 0.02437694687489072, |
| 0.08728971979177597 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "opus-minus-27b": { |
| "all": { |
| "diff": 0.07333333333333333, |
| "ci95": [ |
| 0.04152834968415308, |
| 0.10513831698251358 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.060833333333333336, |
| "ci95": [ |
| 0.030981954682252844, |
| 0.09068471198441383 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gemini-minus-27b": { |
| "all": { |
| "diff": 0.04375, |
| "ci95": [ |
| 0.009717845060980758, |
| 0.07778215493901924 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.035833333333333335, |
| "ci95": [ |
| 0.001730887172231424, |
| 0.06993577949443525 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "4b-minus-qwen27b": { |
| "all": { |
| "diff": -0.04625, |
| "ci95": [ |
| -0.08459349655892173, |
| -0.007906503441078273 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.05083333333333334, |
| "ci95": [ |
| -0.09327710695913138, |
| -0.008389559707535299 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen4b-minus-qwen27b": { |
| "all": { |
| "diff": -0.10875, |
| "ci95": [ |
| -0.1480176125100441, |
| -0.06948238748995589 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.10416666666666669, |
| "ci95": [ |
| -0.14611737522602297, |
| -0.06221595810731039 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "27b-minus-qwen27b": { |
| "all": { |
| "diff": 0.05083333333333334, |
| "ci95": [ |
| 0.010744829973651678, |
| 0.09092183669301501 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.05041666666666667, |
| "ci95": [ |
| 0.01052750632121284, |
| 0.0903058270121205 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gpt-minus-qwen27b": { |
| "all": { |
| "diff": 0.09375, |
| "ci95": [ |
| 0.05730219163645278, |
| 0.13019780836354722 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.10625, |
| "ci95": [ |
| 0.07279390330302032, |
| 0.13970609669697967 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "opus-minus-qwen27b": { |
| "all": { |
| "diff": 0.12416666666666668, |
| "ci95": [ |
| 0.09197561056896073, |
| 0.15635772276437263 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.11125, |
| "ci95": [ |
| 0.07880409709964073, |
| 0.14369590290035927 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "gemini-minus-qwen27b": { |
| "all": { |
| "diff": 0.09458333333333334, |
| "ci95": [ |
| 0.0551882445624855, |
| 0.13397842210418118 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.08625, |
| "ci95": [ |
| 0.04819286835030037, |
| 0.12430713164969961 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| }, |
| "4b-minus-gpt": { |
| "all": { |
| "diff": -0.14, |
| "ci95": [ |
| -0.17711881837873633, |
| -0.1028811816212637 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.15708333333333335, |
| "ci95": [ |
| -0.19574853559217945, |
| -0.11841813107448726 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen4b-minus-gpt": { |
| "all": { |
| "diff": -0.2025, |
| "ci95": [ |
| -0.23990215233702006, |
| -0.16509784766297997 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.21041666666666667, |
| "ci95": [ |
| -0.24804087701896885, |
| -0.1727924563143645 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "27b-minus-gpt": { |
| "all": { |
| "diff": -0.04291666666666667, |
| "ci95": [ |
| -0.07504493396733448, |
| -0.01078839936599886 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.05583333333333334, |
| "ci95": [ |
| -0.08728971979177597, |
| -0.02437694687489072 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen27b-minus-gpt": { |
| "all": { |
| "diff": -0.09375, |
| "ci95": [ |
| -0.13019780836354722, |
| -0.05730219163645278 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.10625, |
| "ci95": [ |
| -0.13970609669697967, |
| -0.07279390330302032 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "opus-minus-gpt": { |
| "all": { |
| "diff": 0.030416666666666668, |
| "ci95": [ |
| 0.004444457390938891, |
| 0.056388875942394445 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.005000000000000001, |
| "ci95": [ |
| -0.015406897160101511, |
| 0.025406897160101513 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| } |
| }, |
| "gemini-minus-gpt": { |
| "all": { |
| "diff": 0.0008333333333333328, |
| "ci95": [ |
| -0.029734282082786483, |
| 0.03140094874945315 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| }, |
| "no_majority": { |
| "diff": -0.02, |
| "ci95": [ |
| -0.048200934523245254, |
| 0.008200934523245256 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| } |
| }, |
| "4b-minus-opus": { |
| "all": { |
| "diff": -0.1704166666666667, |
| "ci95": [ |
| -0.20588318580261128, |
| -0.1349501475307221 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.16208333333333336, |
| "ci95": [ |
| -0.19639520313880648, |
| -0.12777146352786023 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen4b-minus-opus": { |
| "all": { |
| "diff": -0.2329166666666667, |
| "ci95": [ |
| -0.26953620618048846, |
| -0.1962971271528449 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.21541666666666667, |
| "ci95": [ |
| -0.25375282035724744, |
| -0.17708051297608593 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "27b-minus-opus": { |
| "all": { |
| "diff": -0.07333333333333333, |
| "ci95": [ |
| -0.10513831698251358, |
| -0.04152834968415308 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.060833333333333336, |
| "ci95": [ |
| -0.09068471198441383, |
| -0.030981954682252844 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen27b-minus-opus": { |
| "all": { |
| "diff": -0.12416666666666668, |
| "ci95": [ |
| -0.15635772276437263, |
| -0.09197561056896073 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.11125, |
| "ci95": [ |
| -0.14369590290035927, |
| -0.07880409709964073 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "gpt-minus-opus": { |
| "all": { |
| "diff": -0.030416666666666668, |
| "ci95": [ |
| -0.056388875942394445, |
| -0.004444457390938891 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.005000000000000001, |
| "ci95": [ |
| -0.025406897160101513, |
| 0.015406897160101511 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| } |
| }, |
| "gemini-minus-opus": { |
| "all": { |
| "diff": -0.029583333333333336, |
| "ci95": [ |
| -0.056084893999743986, |
| -0.003081772666922687 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.025, |
| "ci95": [ |
| -0.04693792318061361, |
| -0.003062076819386396 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "4b-minus-gemini": { |
| "all": { |
| "diff": -0.14083333333333334, |
| "ci95": [ |
| -0.18037638488148539, |
| -0.10129028178518129 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.13708333333333333, |
| "ci95": [ |
| -0.17551762539100385, |
| -0.0986490412756628 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen4b-minus-gemini": { |
| "all": { |
| "diff": -0.20333333333333337, |
| "ci95": [ |
| -0.2445081566458691, |
| -0.16215851002079762 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.19041666666666668, |
| "ci95": [ |
| -0.23327826842581859, |
| -0.14755506490751477 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "27b-minus-gemini": { |
| "all": { |
| "diff": -0.04375, |
| "ci95": [ |
| -0.07778215493901924, |
| -0.009717845060980758 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.035833333333333335, |
| "ci95": [ |
| -0.06993577949443525, |
| -0.001730887172231424 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "qwen27b-minus-gemini": { |
| "all": { |
| "diff": -0.09458333333333334, |
| "ci95": [ |
| -0.13397842210418118, |
| -0.0551882445624855 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| }, |
| "no_majority": { |
| "diff": -0.08625, |
| "ci95": [ |
| -0.12430713164969961, |
| -0.04819286835030037 |
| ], |
| "n": 200, |
| "verdict": "악화" |
| } |
| }, |
| "gpt-minus-gemini": { |
| "all": { |
| "diff": -0.0008333333333333328, |
| "ci95": [ |
| -0.03140094874945315, |
| 0.029734282082786483 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| }, |
| "no_majority": { |
| "diff": 0.02, |
| "ci95": [ |
| -0.008200934523245256, |
| 0.048200934523245254 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| } |
| }, |
| "opus-minus-gemini": { |
| "all": { |
| "diff": 0.029583333333333336, |
| "ci95": [ |
| 0.003081772666922687, |
| 0.056084893999743986 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| }, |
| "no_majority": { |
| "diff": 0.025, |
| "ci95": [ |
| 0.003062076819386396, |
| 0.04693792318061361 |
| ], |
| "n": 200, |
| "verdict": "개선" |
| } |
| } |
| }, |
| "supplementary": { |
| "qwen27b_system_prompt_only": { |
| "label": "Qwen3.8-27B (base, SYSTEM_PROMPT only)", |
| "criterion_acc": { |
| "mean": 0.8433333333333333, |
| "ci95": [ |
| 0.8119954335270959, |
| 0.8746712331395706 |
| ], |
| "n_answers": 200 |
| }, |
| "criterion_acc_excl_r2_2of3": { |
| "mean": 0.8729166666666667, |
| "ci95": [ |
| 0.8426809286496472, |
| 0.9031524046836862 |
| ], |
| "n_answers": 200 |
| }, |
| "json_invalid": 0, |
| "paired_27b_minus_this": { |
| "all": { |
| "diff": 0.02541666666666667, |
| "ci95": [ |
| -0.009341613239777467, |
| 0.06017494657311081 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| }, |
| "no_majority": { |
| "diff": 0.027083333333333334, |
| "ci95": [ |
| -0.008110744018988085, |
| 0.062277410685654754 |
| ], |
| "n": 200, |
| "verdict": "차이 없음" |
| } |
| } |
| } |
| } |
| } |
