| { |
| "n_test": 554, |
| "valid_json": 1.0, |
| "route_match": 0.5306859205776173, |
| "live_ensembles": 31, |
| "route_counts": { |
| "nemotron-lightning": 218, |
| "glm-5.3-flash": 192, |
| "kimi-vl": 80, |
| "glm-5.3-flash,qwen3.8-27b,nemotron-ultra": 32, |
| "nemotron-ultra": 31, |
| "qwen3.8-27b": 1 |
| }, |
| "policies": { |
| "shaktimaan": { |
| "accuracy": 0.8339, |
| "relative_cost": 0.424 |
| }, |
| "oracle": { |
| "accuracy": 0.8971, |
| "relative_cost": 0.49 |
| }, |
| "always:gemma-3-1b": { |
| "accuracy": 0.2563, |
| "relative_cost": 0.0024 |
| }, |
| "always:kimi-vl": { |
| "accuracy": 0.6137, |
| "relative_cost": 0.0736 |
| }, |
| "always:nemotron-lightning": { |
| "accuracy": 0.7563, |
| "relative_cost": 0.1089 |
| }, |
| "always:glm-5.3-flash": { |
| "accuracy": 0.7924, |
| "relative_cost": 0.0944 |
| }, |
| "always:qwen3.8-27b": { |
| "accuracy": 0.7888, |
| "relative_cost": 0.3641 |
| }, |
| "always:nemotron-ultra": { |
| "accuracy": 0.7852, |
| "relative_cost": 1.0 |
| }, |
| "rule:glm-5.3-flash": { |
| "accuracy": 0.8574, |
| "relative_cost": 0.1331 |
| }, |
| "rule:qwen3.8-27b": { |
| "accuracy": 0.8538, |
| "relative_cost": 0.4028 |
| }, |
| "rule:nemotron-ultra": { |
| "accuracy": 0.8267, |
| "relative_cost": 1.0009 |
| } |
| }, |
| "accuracy_by_source": { |
| "arc": { |
| "n": 75, |
| "shaktimaan": 0.987, |
| "oracle": 1.0, |
| "always:nemotron-ultra": 0.987, |
| "rule:glm-5.3-flash": 0.987, |
| "rule:qwen3.8-27b": 0.987 |
| }, |
| "chartqa": { |
| "n": 31, |
| "shaktimaan": 0.742, |
| "oracle": 0.742, |
| "always:nemotron-ultra": 0.0, |
| "rule:glm-5.3-flash": 0.742, |
| "rule:qwen3.8-27b": 0.742 |
| }, |
| "gsm8k": { |
| "n": 105, |
| "shaktimaan": 0.895, |
| "oracle": 0.981, |
| "always:nemotron-ultra": 0.971, |
| "rule:glm-5.3-flash": 0.962, |
| "rule:qwen3.8-27b": 0.981 |
| }, |
| "humaneval": { |
| "n": 18, |
| "shaktimaan": 0.889, |
| "oracle": 1.0, |
| "always:nemotron-ultra": 0.944, |
| "rule:glm-5.3-flash": 0.944, |
| "rule:qwen3.8-27b": 0.944 |
| }, |
| "longbench": { |
| "n": 32, |
| "shaktimaan": 0.594, |
| "oracle": 0.594, |
| "always:nemotron-ultra": 0.594, |
| "rule:glm-5.3-flash": 0.469, |
| "rule:qwen3.8-27b": 0.5 |
| }, |
| "math": { |
| "n": 106, |
| "shaktimaan": 0.755, |
| "oracle": 0.821, |
| "always:nemotron-ultra": 0.651, |
| "rule:glm-5.3-flash": 0.745, |
| "rule:qwen3.8-27b": 0.755 |
| }, |
| "mbpp": { |
| "n": 67, |
| "shaktimaan": 0.94, |
| "oracle": 1.0, |
| "always:nemotron-ultra": 0.97, |
| "rule:glm-5.3-flash": 0.985, |
| "rule:qwen3.8-27b": 0.925 |
| }, |
| "mmlu_pro": { |
| "n": 120, |
| "shaktimaan": 0.775, |
| "oracle": 0.875, |
| "always:nemotron-ultra": 0.742, |
| "rule:glm-5.3-flash": 0.833, |
| "rule:qwen3.8-27b": 0.817 |
| } |
| }, |
| "live_ensembles_solved": 11 |
| } |
