File size: 4,785 Bytes
7bcd331
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
{
  "chat": {
    "answerability": {
      "False": {
        "answer_match": 0,
        "answer_match_with_neutral_wrapper": 0,
        "clean_response": 15,
        "count": 15,
        "empty": 0,
        "natural_stop": 15,
        "repeated": 0,
        "strict_task_success": 0,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "True": {
        "answer_match": 12,
        "answer_match_with_neutral_wrapper": 12,
        "clean_response": 15,
        "count": 15,
        "empty": 0,
        "natural_stop": 15,
        "repeated": 0,
        "strict_task_success": 12,
        "token_limit": 0,
        "unexpected_structure": 0
      }
    },
    "categories": {
      "answerability": {
        "answer_match": 12,
        "answer_match_with_neutral_wrapper": 12,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 12,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "conversation": {
        "answer_match": 25,
        "answer_match_with_neutral_wrapper": 25,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 25,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "instructions": {
        "answer_match": 11,
        "answer_match_with_neutral_wrapper": 11,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 11,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "knowledge": {
        "answer_match": 15,
        "answer_match_with_neutral_wrapper": 15,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 15,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "reading": {
        "answer_match": 27,
        "answer_match_with_neutral_wrapper": 27,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 27,
        "token_limit": 0,
        "unexpected_structure": 0
      },
      "reasoning": {
        "answer_match": 11,
        "answer_match_with_neutral_wrapper": 11,
        "clean_response": 30,
        "count": 30,
        "empty": 0,
        "natural_stop": 30,
        "repeated": 0,
        "strict_task_success": 11,
        "token_limit": 0,
        "unexpected_structure": 0
      }
    },
    "caveat": "Answer match is conservative automatic scoring, not a complete semantic correctness judgment. Non-matching answers and open chat require review. Repetition detection is heuristic.",
    "objective": {
      "answer_match": 101,
      "answer_match_with_neutral_wrapper": 101,
      "clean_response": 180,
      "count": 180,
      "empty": 0,
      "natural_stop": 180,
      "repeated": 0,
      "strict_task_success": 101,
      "token_limit": 0,
      "unexpected_structure": 0
    },
    "response_control_all": {
      "answer_match": 101,
      "answer_match_with_neutral_wrapper": 101,
      "clean_response": 190,
      "count": 192,
      "empty": 0,
      "natural_stop": 191,
      "repeated": 2,
      "strict_task_success": 101,
      "token_limit": 1,
      "unexpected_structure": 0
    }
  },
  "choices": {
    "arc_challenge": {
      "acc": 0.2901023890784983,
      "acc_norm": 0.3122866894197952,
      "complete": true,
      "context_truncated_choices": 0,
      "evaluated": 1172,
      "expected": 1172
    },
    "arc_easy": {
      "acc": 0.6300505050505051,
      "acc_norm": 0.6233164983164983,
      "complete": true,
      "context_truncated_choices": 0,
      "evaluated": 2376,
      "expected": 2376
    },
    "hellaswag": {
      "acc": 0.329416450906194,
      "acc_norm": 0.394443337980482,
      "complete": true,
      "context_truncated_choices": 0,
      "evaluated": 10042,
      "expected": 10042
    },
    "piqa": {
      "acc": 0.6632208922742111,
      "acc_norm": 0.6643090315560392,
      "complete": true,
      "context_truncated_choices": 0,
      "evaluated": 1838,
      "expected": 1838
    }
  },
  "complete": true,
  "ifeval": {
    "loose": {
      "instruction_accuracy": 0.30335731414868106,
      "instructions": 834,
      "prompt_accuracy": 0.1645101663585952,
      "prompts": 541
    },
    "strict": {
      "instruction_accuracy": 0.28177458033573144,
      "instructions": 834,
      "prompt_accuracy": 0.15526802218114602,
      "prompts": 541
    }
  },
  "model_sha256": "2cba542e887527a294bf8435d59015318dbeb3f58503d075abe14c90775b5ab0"
}