{
  "n_test": 554,
  "valid_json": 1.0,
  "route_match": 0.5306859205776173,
  "live_ensembles": 31,
  "route_counts": {
    "nemotron-lightning": 218,
    "glm-5.3-flash": 192,
    "kimi-vl": 80,
    "glm-5.3-flash,qwen3.8-27b,nemotron-ultra": 32,
    "nemotron-ultra": 31,
    "qwen3.8-27b": 1
  },
  "policies": {
    "shaktimaan": {
      "accuracy": 0.8339,
      "relative_cost": 0.424
    },
    "oracle": {
      "accuracy": 0.8971,
      "relative_cost": 0.49
    },
    "always:gemma-3-1b": {
      "accuracy": 0.2563,
      "relative_cost": 0.0024
    },
    "always:kimi-vl": {
      "accuracy": 0.6137,
      "relative_cost": 0.0736
    },
    "always:nemotron-lightning": {
      "accuracy": 0.7563,
      "relative_cost": 0.1089
    },
    "always:glm-5.3-flash": {
      "accuracy": 0.7924,
      "relative_cost": 0.0944
    },
    "always:qwen3.8-27b": {
      "accuracy": 0.7888,
      "relative_cost": 0.3641
    },
    "always:nemotron-ultra": {
      "accuracy": 0.7852,
      "relative_cost": 1.0
    },
    "rule:glm-5.3-flash": {
      "accuracy": 0.8574,
      "relative_cost": 0.1331
    },
    "rule:qwen3.8-27b": {
      "accuracy": 0.8538,
      "relative_cost": 0.4028
    },
    "rule:nemotron-ultra": {
      "accuracy": 0.8267,
      "relative_cost": 1.0009
    }
  },
  "accuracy_by_source": {
    "arc": {
      "n": 75,
      "shaktimaan": 0.987,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.987,
      "rule:glm-5.3-flash": 0.987,
      "rule:qwen3.8-27b": 0.987
    },
    "chartqa": {
      "n": 31,
      "shaktimaan": 0.742,
      "oracle": 0.742,
      "always:nemotron-ultra": 0.0,
      "rule:glm-5.3-flash": 0.742,
      "rule:qwen3.8-27b": 0.742
    },
    "gsm8k": {
      "n": 105,
      "shaktimaan": 0.895,
      "oracle": 0.981,
      "always:nemotron-ultra": 0.971,
      "rule:glm-5.3-flash": 0.962,
      "rule:qwen3.8-27b": 0.981
    },
    "humaneval": {
      "n": 18,
      "shaktimaan": 0.889,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.944,
      "rule:glm-5.3-flash": 0.944,
      "rule:qwen3.8-27b": 0.944
    },
    "longbench": {
      "n": 32,
      "shaktimaan": 0.594,
      "oracle": 0.594,
      "always:nemotron-ultra": 0.594,
      "rule:glm-5.3-flash": 0.469,
      "rule:qwen3.8-27b": 0.5
    },
    "math": {
      "n": 106,
      "shaktimaan": 0.755,
      "oracle": 0.821,
      "always:nemotron-ultra": 0.651,
      "rule:glm-5.3-flash": 0.745,
      "rule:qwen3.8-27b": 0.755
    },
    "mbpp": {
      "n": 67,
      "shaktimaan": 0.94,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.97,
      "rule:glm-5.3-flash": 0.985,
      "rule:qwen3.8-27b": 0.925
    },
    "mmlu_pro": {
      "n": 120,
      "shaktimaan": 0.775,
      "oracle": 0.875,
      "always:nemotron-ultra": 0.742,
      "rule:glm-5.3-flash": 0.833,
      "rule:qwen3.8-27b": 0.817
    }
  },
  "live_ensembles_solved": 11
}