| { |
| "cost_weights_assumed": { |
| "small": 1, |
| "medium": 3, |
| "large": 10, |
| "reasoning": 15 |
| }, |
| "sets": { |
| "heldout_combo": { |
| "n": 1200, |
| "accuracy": 0.9841666666666666, |
| "under_routing_rate": 0.0016666666666666668, |
| "over_routing_rate": 0.014166666666666666, |
| "cost_ratio_vs_gold": 1.0072413793103447, |
| "overspend_rate": 0.009655172413793104, |
| "confusion_matrix": { |
| "rows_gold_cols_pred": [ |
| "small", |
| "medium", |
| "large", |
| "reasoning" |
| ], |
| "matrix": [ |
| [ |
| 293, |
| 7, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 290, |
| 10, |
| 0 |
| ], |
| [ |
| 0, |
| 1, |
| 299, |
| 0 |
| ], |
| [ |
| 1, |
| 0, |
| 0, |
| 299 |
| ] |
| ] |
| }, |
| "per_tier_accuracy": { |
| "small": 0.9766666666666667, |
| "medium": 0.9666666666666667, |
| "large": 0.9966666666666667, |
| "reasoning": 0.9966666666666667 |
| }, |
| "per_language_accuracy": { |
| "en": 0.9916666666666667, |
| "ko": 0.9966666666666667, |
| "vi": 0.9833333333333333, |
| "vi_nodiac": 0.9666666666666667 |
| }, |
| "per_family": { |
| "bash_script": { |
| "accuracy": 0.9803921568627451, |
| "n": 51, |
| "predicted": { |
| "medium": 50, |
| "large": 1 |
| } |
| }, |
| "cicd_monorepo": { |
| "accuracy": 1.0, |
| "n": 52, |
| "predicted": { |
| "large": 52 |
| } |
| }, |
| "cipher": { |
| "accuracy": 1.0, |
| "n": 64, |
| "predicted": { |
| "reasoning": 64 |
| } |
| }, |
| "constraint_puzzle": { |
| "accuracy": 1.0, |
| "n": 74, |
| "predicted": { |
| "reasoning": 74 |
| } |
| }, |
| "csv_to_json": { |
| "accuracy": 0.9857142857142858, |
| "n": 70, |
| "predicted": { |
| "small": 69, |
| "medium": 1 |
| } |
| }, |
| "dockerfile": { |
| "accuracy": 0.9310344827586207, |
| "n": 58, |
| "predicted": { |
| "medium": 54, |
| "large": 4 |
| } |
| }, |
| "explain_snippet": { |
| "accuracy": 1.0, |
| "n": 78, |
| "predicted": { |
| "medium": 78 |
| } |
| }, |
| "graph_proof": { |
| "accuracy": 1.0, |
| "n": 53, |
| "predicted": { |
| "reasoning": 53 |
| } |
| }, |
| "multi_transcript": { |
| "accuracy": 1.0, |
| "n": 61, |
| "predicted": { |
| "large": 61 |
| } |
| }, |
| "paradox": { |
| "accuracy": 0.9830508474576272, |
| "n": 59, |
| "predicted": { |
| "reasoning": 58, |
| "small": 1 |
| } |
| }, |
| "physics_derive": { |
| "accuracy": 1.0, |
| "n": 50, |
| "predicted": { |
| "reasoning": 50 |
| } |
| }, |
| "prd": { |
| "accuracy": 0.984375, |
| "n": 64, |
| "predicted": { |
| "large": 63, |
| "medium": 1 |
| } |
| }, |
| "regex": { |
| "accuracy": 1.0, |
| "n": 47, |
| "predicted": { |
| "medium": 47 |
| } |
| }, |
| "repo_qa": { |
| "accuracy": 1.0, |
| "n": 65, |
| "predicted": { |
| "large": 65 |
| } |
| }, |
| "study_plan": { |
| "accuracy": 0.9242424242424242, |
| "n": 66, |
| "predicted": { |
| "medium": 61, |
| "large": 5 |
| } |
| }, |
| "synonym": { |
| "accuracy": 1.0, |
| "n": 54, |
| "predicted": { |
| "small": 54 |
| } |
| }, |
| "timezone": { |
| "accuracy": 1.0, |
| "n": 63, |
| "predicted": { |
| "small": 63 |
| } |
| }, |
| "vendor_compare": { |
| "accuracy": 1.0, |
| "n": 58, |
| "predicted": { |
| "large": 58 |
| } |
| }, |
| "word_count": { |
| "accuracy": 0.9629629629629629, |
| "n": 54, |
| "predicted": { |
| "small": 52, |
| "medium": 2 |
| } |
| }, |
| "yes_no_fact": { |
| "accuracy": 0.9322033898305084, |
| "n": 59, |
| "predicted": { |
| "small": 55, |
| "medium": 4 |
| } |
| } |
| }, |
| "pass": true |
| }, |
| "heldout_style": { |
| "n": 1200, |
| "accuracy": 0.96, |
| "under_routing_rate": 0.0016666666666666668, |
| "over_routing_rate": 0.03833333333333333, |
| "cost_ratio_vs_gold": 1.0241379310344827, |
| "overspend_rate": 0.02517241379310345, |
| "confusion_matrix": { |
| "rows_gold_cols_pred": [ |
| "small", |
| "medium", |
| "large", |
| "reasoning" |
| ], |
| "matrix": [ |
| [ |
| 261, |
| 32, |
| 0, |
| 7 |
| ], |
| [ |
| 1, |
| 293, |
| 4, |
| 2 |
| ], |
| [ |
| 0, |
| 1, |
| 298, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 300 |
| ] |
| ] |
| }, |
| "per_tier_accuracy": { |
| "small": 0.87, |
| "medium": 0.9766666666666667, |
| "large": 0.9933333333333333, |
| "reasoning": 1.0 |
| }, |
| "per_language_accuracy": { |
| "en": 0.9791666666666666, |
| "ko": 0.95, |
| "vi": 0.9611111111111111, |
| "vi_nodiac": 0.9533333333333334 |
| }, |
| "per_family": { |
| "bash_script": { |
| "accuracy": 1.0, |
| "n": 58, |
| "predicted": { |
| "medium": 58 |
| } |
| }, |
| "cicd_monorepo": { |
| "accuracy": 0.9583333333333334, |
| "n": 48, |
| "predicted": { |
| "large": 46, |
| "reasoning": 1, |
| "medium": 1 |
| } |
| }, |
| "cipher": { |
| "accuracy": 1.0, |
| "n": 62, |
| "predicted": { |
| "reasoning": 62 |
| } |
| }, |
| "constraint_puzzle": { |
| "accuracy": 1.0, |
| "n": 72, |
| "predicted": { |
| "reasoning": 72 |
| } |
| }, |
| "csv_to_json": { |
| "accuracy": 0.9661016949152542, |
| "n": 59, |
| "predicted": { |
| "small": 57, |
| "medium": 2 |
| } |
| }, |
| "dockerfile": { |
| "accuracy": 0.9636363636363636, |
| "n": 55, |
| "predicted": { |
| "medium": 53, |
| "reasoning": 1, |
| "large": 1 |
| } |
| }, |
| "explain_snippet": { |
| "accuracy": 0.9714285714285714, |
| "n": 70, |
| "predicted": { |
| "medium": 68, |
| "small": 1, |
| "reasoning": 1 |
| } |
| }, |
| "graph_proof": { |
| "accuracy": 1.0, |
| "n": 47, |
| "predicted": { |
| "reasoning": 47 |
| } |
| }, |
| "multi_transcript": { |
| "accuracy": 1.0, |
| "n": 78, |
| "predicted": { |
| "large": 78 |
| } |
| }, |
| "paradox": { |
| "accuracy": 1.0, |
| "n": 60, |
| "predicted": { |
| "reasoning": 60 |
| } |
| }, |
| "physics_derive": { |
| "accuracy": 1.0, |
| "n": 59, |
| "predicted": { |
| "reasoning": 59 |
| } |
| }, |
| "prd": { |
| "accuracy": 1.0, |
| "n": 57, |
| "predicted": { |
| "large": 57 |
| } |
| }, |
| "regex": { |
| "accuracy": 1.0, |
| "n": 56, |
| "predicted": { |
| "medium": 56 |
| } |
| }, |
| "repo_qa": { |
| "accuracy": 1.0, |
| "n": 60, |
| "predicted": { |
| "large": 60 |
| } |
| }, |
| "study_plan": { |
| "accuracy": 0.9508196721311475, |
| "n": 61, |
| "predicted": { |
| "medium": 58, |
| "large": 3 |
| } |
| }, |
| "synonym": { |
| "accuracy": 0.86, |
| "n": 50, |
| "predicted": { |
| "medium": 6, |
| "small": 43, |
| "reasoning": 1 |
| } |
| }, |
| "timezone": { |
| "accuracy": 0.9298245614035088, |
| "n": 57, |
| "predicted": { |
| "small": 53, |
| "medium": 1, |
| "reasoning": 3 |
| } |
| }, |
| "vendor_compare": { |
| "accuracy": 1.0, |
| "n": 57, |
| "predicted": { |
| "large": 57 |
| } |
| }, |
| "word_count": { |
| "accuracy": 0.8333333333333334, |
| "n": 66, |
| "predicted": { |
| "small": 55, |
| "medium": 10, |
| "reasoning": 1 |
| } |
| }, |
| "yes_no_fact": { |
| "accuracy": 0.7794117647058824, |
| "n": 68, |
| "predicted": { |
| "small": 53, |
| "medium": 13, |
| "reasoning": 2 |
| } |
| } |
| }, |
| "pass": true |
| } |
| } |
| } |
