Hugging Face's logo

podstack
/
Shaktimaan

Text Generation
Transformers
Safetensors
English
qwen2
router
llm-router
orchestration
multi-agent
model-selection
fugu
conversational
Eval Results (legacy)
text-generation-inference
Model card Files
xet
 Community
Shaktimaan
File size: 2,899 Bytes
c1f5d12
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
{
  "n_test": 554,
  "valid_json": 1.0,
  "route_match": 0.5306859205776173,
  "live_ensembles": 31,
  "route_counts": {
    "nemotron-lightning": 218,
    "glm-5.3-flash": 192,
    "kimi-vl": 80,
    "glm-5.3-flash,qwen3.8-27b,nemotron-ultra": 32,
    "nemotron-ultra": 31,
    "qwen3.8-27b": 1
  },
  "policies": {
    "shaktimaan": {
      "accuracy": 0.8339,
      "relative_cost": 0.424
    },
    "oracle": {
      "accuracy": 0.8971,
      "relative_cost": 0.49
    },
    "always:gemma-3-1b": {
      "accuracy": 0.2563,
      "relative_cost": 0.0024
    },
    "always:kimi-vl": {
      "accuracy": 0.6137,
      "relative_cost": 0.0736
    },
    "always:nemotron-lightning": {
      "accuracy": 0.7563,
      "relative_cost": 0.1089
    },
    "always:glm-5.3-flash": {
      "accuracy": 0.7924,
      "relative_cost": 0.0944
    },
    "always:qwen3.8-27b": {
      "accuracy": 0.7888,
      "relative_cost": 0.3641
    },
    "always:nemotron-ultra": {
      "accuracy": 0.7852,
      "relative_cost": 1.0
    },
    "rule:glm-5.3-flash": {
      "accuracy": 0.8574,
      "relative_cost": 0.1331
    },
    "rule:qwen3.8-27b": {
      "accuracy": 0.8538,
      "relative_cost": 0.4028
    },
    "rule:nemotron-ultra": {
      "accuracy": 0.8267,
      "relative_cost": 1.0009
    }
  },
  "accuracy_by_source": {
    "arc": {
      "n": 75,
      "shaktimaan": 0.987,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.987,
      "rule:glm-5.3-flash": 0.987,
      "rule:qwen3.8-27b": 0.987
    },
    "chartqa": {
      "n": 31,
      "shaktimaan": 0.742,
      "oracle": 0.742,
      "always:nemotron-ultra": 0.0,
      "rule:glm-5.3-flash": 0.742,
      "rule:qwen3.8-27b": 0.742
    },
    "gsm8k": {
      "n": 105,
      "shaktimaan": 0.895,
      "oracle": 0.981,
      "always:nemotron-ultra": 0.971,
      "rule:glm-5.3-flash": 0.962,
      "rule:qwen3.8-27b": 0.981
    },
    "humaneval": {
      "n": 18,
      "shaktimaan": 0.889,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.944,
      "rule:glm-5.3-flash": 0.944,
      "rule:qwen3.8-27b": 0.944
    },
    "longbench": {
      "n": 32,
      "shaktimaan": 0.594,
      "oracle": 0.594,
      "always:nemotron-ultra": 0.594,
      "rule:glm-5.3-flash": 0.469,
      "rule:qwen3.8-27b": 0.5
    },
    "math": {
      "n": 106,
      "shaktimaan": 0.755,
      "oracle": 0.821,
      "always:nemotron-ultra": 0.651,
      "rule:glm-5.3-flash": 0.745,
      "rule:qwen3.8-27b": 0.755
    },
    "mbpp": {
      "n": 67,
      "shaktimaan": 0.94,
      "oracle": 1.0,
      "always:nemotron-ultra": 0.97,
      "rule:glm-5.3-flash": 0.985,
      "rule:qwen3.8-27b": 0.925
    },
    "mmlu_pro": {
      "n": 120,
      "shaktimaan": 0.775,
      "oracle": 0.875,
      "always:nemotron-ultra": 0.742,
      "rule:glm-5.3-flash": 0.833,
      "rule:qwen3.8-27b": 0.817
    }
  },
  "live_ensembles_solved": 11
}