{
  "cost_weights_assumed": {
    "small": 1,
    "medium": 3,
    "large": 10,
    "reasoning": 15
  },
  "sets": {
    "heldout_combo": {
      "n": 1200,
      "accuracy": 0.9841666666666666,
      "under_routing_rate": 0.0016666666666666668,
      "over_routing_rate": 0.014166666666666666,
      "cost_ratio_vs_gold": 1.0072413793103447,
      "overspend_rate": 0.009655172413793104,
      "confusion_matrix": {
        "rows_gold_cols_pred": [
          "small",
          "medium",
          "large",
          "reasoning"
        ],
        "matrix": [
          [
            293,
            7,
            0,
            0
          ],
          [
            0,
            290,
            10,
            0
          ],
          [
            0,
            1,
            299,
            0
          ],
          [
            1,
            0,
            0,
            299
          ]
        ]
      },
      "per_tier_accuracy": {
        "small": 0.9766666666666667,
        "medium": 0.9666666666666667,
        "large": 0.9966666666666667,
        "reasoning": 0.9966666666666667
      },
      "per_language_accuracy": {
        "en": 0.9916666666666667,
        "ko": 0.9966666666666667,
        "vi": 0.9833333333333333,
        "vi_nodiac": 0.9666666666666667
      },
      "per_family": {
        "bash_script": {
          "accuracy": 0.9803921568627451,
          "n": 51,
          "predicted": {
            "medium": 50,
            "large": 1
          }
        },
        "cicd_monorepo": {
          "accuracy": 1.0,
          "n": 52,
          "predicted": {
            "large": 52
          }
        },
        "cipher": {
          "accuracy": 1.0,
          "n": 64,
          "predicted": {
            "reasoning": 64
          }
        },
        "constraint_puzzle": {
          "accuracy": 1.0,
          "n": 74,
          "predicted": {
            "reasoning": 74
          }
        },
        "csv_to_json": {
          "accuracy": 0.9857142857142858,
          "n": 70,
          "predicted": {
            "small": 69,
            "medium": 1
          }
        },
        "dockerfile": {
          "accuracy": 0.9310344827586207,
          "n": 58,
          "predicted": {
            "medium": 54,
            "large": 4
          }
        },
        "explain_snippet": {
          "accuracy": 1.0,
          "n": 78,
          "predicted": {
            "medium": 78
          }
        },
        "graph_proof": {
          "accuracy": 1.0,
          "n": 53,
          "predicted": {
            "reasoning": 53
          }
        },
        "multi_transcript": {
          "accuracy": 1.0,
          "n": 61,
          "predicted": {
            "large": 61
          }
        },
        "paradox": {
          "accuracy": 0.9830508474576272,
          "n": 59,
          "predicted": {
            "reasoning": 58,
            "small": 1
          }
        },
        "physics_derive": {
          "accuracy": 1.0,
          "n": 50,
          "predicted": {
            "reasoning": 50
          }
        },
        "prd": {
          "accuracy": 0.984375,
          "n": 64,
          "predicted": {
            "large": 63,
            "medium": 1
          }
        },
        "regex": {
          "accuracy": 1.0,
          "n": 47,
          "predicted": {
            "medium": 47
          }
        },
        "repo_qa": {
          "accuracy": 1.0,
          "n": 65,
          "predicted": {
            "large": 65
          }
        },
        "study_plan": {
          "accuracy": 0.9242424242424242,
          "n": 66,
          "predicted": {
            "medium": 61,
            "large": 5
          }
        },
        "synonym": {
          "accuracy": 1.0,
          "n": 54,
          "predicted": {
            "small": 54
          }
        },
        "timezone": {
          "accuracy": 1.0,
          "n": 63,
          "predicted": {
            "small": 63
          }
        },
        "vendor_compare": {
          "accuracy": 1.0,
          "n": 58,
          "predicted": {
            "large": 58
          }
        },
        "word_count": {
          "accuracy": 0.9629629629629629,
          "n": 54,
          "predicted": {
            "small": 52,
            "medium": 2
          }
        },
        "yes_no_fact": {
          "accuracy": 0.9322033898305084,
          "n": 59,
          "predicted": {
            "small": 55,
            "medium": 4
          }
        }
      },
      "pass": true
    },
    "heldout_style": {
      "n": 1200,
      "accuracy": 0.96,
      "under_routing_rate": 0.0016666666666666668,
      "over_routing_rate": 0.03833333333333333,
      "cost_ratio_vs_gold": 1.0241379310344827,
      "overspend_rate": 0.02517241379310345,
      "confusion_matrix": {
        "rows_gold_cols_pred": [
          "small",
          "medium",
          "large",
          "reasoning"
        ],
        "matrix": [
          [
            261,
            32,
            0,
            7
          ],
          [
            1,
            293,
            4,
            2
          ],
          [
            0,
            1,
            298,
            1
          ],
          [
            0,
            0,
            0,
            300
          ]
        ]
      },
      "per_tier_accuracy": {
        "small": 0.87,
        "medium": 0.9766666666666667,
        "large": 0.9933333333333333,
        "reasoning": 1.0
      },
      "per_language_accuracy": {
        "en": 0.9791666666666666,
        "ko": 0.95,
        "vi": 0.9611111111111111,
        "vi_nodiac": 0.9533333333333334
      },
      "per_family": {
        "bash_script": {
          "accuracy": 1.0,
          "n": 58,
          "predicted": {
            "medium": 58
          }
        },
        "cicd_monorepo": {
          "accuracy": 0.9583333333333334,
          "n": 48,
          "predicted": {
            "large": 46,
            "reasoning": 1,
            "medium": 1
          }
        },
        "cipher": {
          "accuracy": 1.0,
          "n": 62,
          "predicted": {
            "reasoning": 62
          }
        },
        "constraint_puzzle": {
          "accuracy": 1.0,
          "n": 72,
          "predicted": {
            "reasoning": 72
          }
        },
        "csv_to_json": {
          "accuracy": 0.9661016949152542,
          "n": 59,
          "predicted": {
            "small": 57,
            "medium": 2
          }
        },
        "dockerfile": {
          "accuracy": 0.9636363636363636,
          "n": 55,
          "predicted": {
            "medium": 53,
            "reasoning": 1,
            "large": 1
          }
        },
        "explain_snippet": {
          "accuracy": 0.9714285714285714,
          "n": 70,
          "predicted": {
            "medium": 68,
            "small": 1,
            "reasoning": 1
          }
        },
        "graph_proof": {
          "accuracy": 1.0,
          "n": 47,
          "predicted": {
            "reasoning": 47
          }
        },
        "multi_transcript": {
          "accuracy": 1.0,
          "n": 78,
          "predicted": {
            "large": 78
          }
        },
        "paradox": {
          "accuracy": 1.0,
          "n": 60,
          "predicted": {
            "reasoning": 60
          }
        },
        "physics_derive": {
          "accuracy": 1.0,
          "n": 59,
          "predicted": {
            "reasoning": 59
          }
        },
        "prd": {
          "accuracy": 1.0,
          "n": 57,
          "predicted": {
            "large": 57
          }
        },
        "regex": {
          "accuracy": 1.0,
          "n": 56,
          "predicted": {
            "medium": 56
          }
        },
        "repo_qa": {
          "accuracy": 1.0,
          "n": 60,
          "predicted": {
            "large": 60
          }
        },
        "study_plan": {
          "accuracy": 0.9508196721311475,
          "n": 61,
          "predicted": {
            "medium": 58,
            "large": 3
          }
        },
        "synonym": {
          "accuracy": 0.86,
          "n": 50,
          "predicted": {
            "medium": 6,
            "small": 43,
            "reasoning": 1
          }
        },
        "timezone": {
          "accuracy": 0.9298245614035088,
          "n": 57,
          "predicted": {
            "small": 53,
            "medium": 1,
            "reasoning": 3
          }
        },
        "vendor_compare": {
          "accuracy": 1.0,
          "n": 57,
          "predicted": {
            "large": 57
          }
        },
        "word_count": {
          "accuracy": 0.8333333333333334,
          "n": 66,
          "predicted": {
            "small": 55,
            "medium": 10,
            "reasoning": 1
          }
        },
        "yes_no_fact": {
          "accuracy": 0.7794117647058824,
          "n": 68,
          "predicted": {
            "small": 53,
            "medium": 13,
            "reasoning": 2
          }
        }
      },
      "pass": true
    }
  }
}