Hugging Face's logo

DatPySci
/
AR-Diff

Safetensors
Model card Files
xet
 Community
DatPySci commited on
Commit
0df005c
·
verified ·
1 Parent(s): 79d7e97

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +103 -0
  2. results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl +3 -0
  3. results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json +1897 -0
  4. results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl +3 -0
  5. results/composition/SDAR-100M-block16-epoch1-op11-14/summary.csv +2 -0
  6. results/composition/SDAR-100M-block16-epoch1-op11-14/summary.json +645 -0
  7. results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl +3 -0
  8. results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json +1897 -0
  9. results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl +3 -0
  10. results/composition/SDAR-100M-block16-epoch1-op17-20/summary.csv +2 -0
  11. results/composition/SDAR-100M-block16-epoch1-op17-20/summary.json +645 -0
  12. results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl +3 -0
  13. results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json +1897 -0
  14. results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl +3 -0
  15. results/composition/SDAR-100M-block16-epoch1-op7-10/summary.csv +2 -0
  16. results/composition/SDAR-100M-block16-epoch1-op7-10/summary.json +645 -0
  17. results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl +3 -0
  18. results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json +1897 -0
  19. results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl +3 -0
  20. results/composition/SDAR-100M-block16-epoch1-op9-12/summary.csv +2 -0
  21. results/composition/SDAR-100M-block16-epoch1-op9-12/summary.json +645 -0
  22. results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl +3 -0
  23. results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json +1897 -0
  24. results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl +3 -0
  25. results/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv +2 -0
  26. results/composition/SDAR-100M-block16-epoch4-op11-14/summary.json +645 -0
  27. results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl +3 -0
  28. results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json +1897 -0
  29. results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl +3 -0
  30. results/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv +2 -0
  31. results/composition/SDAR-100M-block16-epoch4-op17-20/summary.json +645 -0
  32. results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl +3 -0
  33. results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json +1897 -0
  34. results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl +3 -0
  35. results/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv +2 -0
  36. results/composition/SDAR-100M-block16-epoch4-op7-10/summary.json +645 -0
  37. results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl +3 -0
  38. results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json +1897 -0
  39. results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl +3 -0
  40. results/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv +2 -0
  41. results/composition/SDAR-100M-block16-epoch4-op9-12/summary.json +645 -0
  42. results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl +3 -0
  43. results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json +1897 -0
  44. results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl +3 -0
  45. results/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv +2 -0
  46. results/composition/SDAR-100M-block4-epoch1-op11-14/summary.json +645 -0
  47. results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl +3 -0
  48. results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json +1897 -0
  49. results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl +3 -0
  50. results/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv +2 -0
.gitattributes CHANGED
@@ -92,3 +92,106 @@ data/context/teachers_in_school/heldout/op6_50k.jsonl filter=lfs diff=lfs merge=
92
  data/context/teachers_in_school/heldout/op7_50k.jsonl filter=lfs diff=lfs merge=lfs -text
93
  data/context/teachers_in_school/heldout/op8_50k.jsonl filter=lfs diff=lfs merge=lfs -text
94
  data/context/teachers_in_school/heldout/op9_50k.jsonl filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  data/context/teachers_in_school/heldout/op7_50k.jsonl filter=lfs diff=lfs merge=lfs -text
93
  data/context/teachers_in_school/heldout/op8_50k.jsonl filter=lfs diff=lfs merge=lfs -text
94
  data/context/teachers_in_school/heldout/op9_50k.jsonl filter=lfs diff=lfs merge=lfs -text
95
+ results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
96
+ results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
97
+ results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
98
+ results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
99
+ results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
100
+ results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
101
+ results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
102
+ results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
103
+ results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
104
+ results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
105
+ results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
106
+ results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
107
+ results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
108
+ results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
109
+ results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
110
+ results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
111
+ results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
112
+ results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
113
+ results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
114
+ results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
115
+ results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
116
+ results/composition/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
117
+ results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
118
+ results/composition/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
119
+ results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
120
+ results/composition/SDAR-100M-block4-epoch4-op11-14/SDAR-100M-block4-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
121
+ results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
122
+ results/composition/SDAR-100M-block4-epoch4-op17-20/SDAR-100M-block4-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
123
+ results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
124
+ results/composition/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
125
+ results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
126
+ results/composition/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
127
+ results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
128
+ results/composition/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
129
+ results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
130
+ results/composition/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
131
+ results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
132
+ results/composition/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
133
+ results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
134
+ results/composition/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
135
+ results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
136
+ results/composition/SDAR-100M-block8-epoch4-op11-14/SDAR-100M-block8-epoch4-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
137
+ results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
138
+ results/composition/SDAR-100M-block8-epoch4-op17-20/SDAR-100M-block8-epoch4-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
139
+ results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
140
+ results/composition/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
141
+ results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
142
+ results/composition/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
143
+ results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
144
+ results/composition/block4/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
145
+ results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
146
+ results/composition/block4/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
147
+ results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
148
+ results/composition/block4/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
149
+ results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
150
+ results/composition/block4/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
151
+ results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
152
+ results/composition/block4/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
153
+ results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
154
+ results/composition/block4/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
155
+ results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
156
+ results/composition/block4/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
157
+ results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
158
+ results/composition/block4/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
159
+ results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
160
+ results/composition/block4/SDAR-100M-block4-epoch1-op7-10/SDAR-100M-block4-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
161
+ results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
162
+ results/composition/block4/SDAR-100M-block4-epoch1-op9-12/SDAR-100M-block4-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
163
+ results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
164
+ results/composition/block4/SDAR-100M-block4-epoch4-op7-10/SDAR-100M-block4-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
165
+ results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
166
+ results/composition/block4/SDAR-100M-block4-epoch4-op9-12/SDAR-100M-block4-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
167
+ results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
168
+ results/composition/block4/SDAR-100M-block8-epoch1-op11-14/SDAR-100M-block8-epoch1-op11-14_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
169
+ results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
170
+ results/composition/block4/SDAR-100M-block8-epoch1-op17-20/SDAR-100M-block8-epoch1-op17-20_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
171
+ results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
172
+ results/composition/block4/SDAR-100M-block8-epoch1-op7-10/SDAR-100M-block8-epoch1-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
173
+ results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
174
+ results/composition/block4/SDAR-100M-block8-epoch1-op9-12/SDAR-100M-block8-epoch1-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
175
+ results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
176
+ results/composition/block4/SDAR-100M-block8-epoch4-op7-10/SDAR-100M-block8-epoch4-op7-10_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
177
+ results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
178
+ results/composition/block4/SDAR-100M-block8-epoch4-op9-12/SDAR-100M-block8-epoch4-op9-12_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
179
+ results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
180
+ results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
181
+ results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
182
+ results/context/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2/graphs/Qwen3-100M-AR-0.999zoo_op2-20+0.001teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
183
+ results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
184
+ results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
185
+ results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
186
+ results/context/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2/graphs/Qwen3-100M-AR-0.99zoo_op2-20+0.01teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
187
+ results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
188
+ results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
189
+ results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
190
+ results/context/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2/graphs/Qwen3-100M-AR-0.9zoo_op2-20+0.1teacher_op2_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
191
+ results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
192
+ results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
193
+ results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
194
+ results/context/block4/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/graphs/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2-dlm_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
195
+ results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
196
+ results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
197
+ results/context/block4/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/graphs/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2-dlm_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e31eecdcea7aae730ad9cf8d0ab92c3308fedd3855b97e3e9ece59ccb9e73220
3
+ size 1795735165
results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch1-op11-14",
3
+ "id": {
4
+ "answer_accuracy": 0.9566944444444444,
5
+ "avg_response_len": 201.07631163194444,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9566944444444444,
11
+ "pass@2": 0.9703619996719052,
12
+ "pass@4": 0.9787109671603543,
13
+ "pass@8": 0.9843092719873776,
14
+ "pass@16": 0.9881272957042198,
15
+ "pass@32": 0.9908108707272433,
16
+ "pass@64": 0.9929228603396479,
17
+ "pass@128": 0.9947777777777778
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9013359375,
22
+ "pass@2": 0.9259847440944887,
23
+ "pass@4": 0.9431592971503563,
24
+ "pass@8": 0.956216013428469,
25
+ "pass@16": 0.9655633252300617,
26
+ "pass@32": 0.9717840648944569,
27
+ "pass@64": 0.9764011519484884,
28
+ "pass@128": 0.981
29
+ },
30
+ "2": {
31
+ "pass@1": 0.9834765625,
32
+ "pass@2": 0.9886993110236222,
33
+ "pass@4": 0.9916434472253468,
34
+ "pass@8": 0.9931709501255696,
35
+ "pass@16": 0.9940232177790858,
36
+ "pass@32": 0.9945486103085506,
37
+ "pass@64": 0.9948778631860671,
38
+ "pass@128": 0.995
39
+ },
40
+ "3": {
41
+ "pass@1": 0.9914375,
42
+ "pass@2": 0.9955533956692916,
43
+ "pass@4": 0.9974521435133106,
44
+ "pass@8": 0.9985974908269989,
45
+ "pass@16": 0.9994160208486209,
46
+ "pass@32": 0.9998813614910528,
47
+ "pass@64": 0.99999788750106,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9889921875,
52
+ "pass@2": 0.9921766732283467,
53
+ "pass@4": 0.9943459424446944,
54
+ "pass@8": 0.9959193357486759,
55
+ "pass@16": 0.9970008520200708,
56
+ "pass@32": 0.9978515643805362,
57
+ "pass@64": 0.9987504617142802,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.9543515625,
62
+ "pass@2": 0.9634336860236224,
63
+ "pass@4": 0.9701805443382078,
64
+ "pass@8": 0.9754922702366896,
65
+ "pass@16": 0.9795036618309861,
66
+ "pass@32": 0.9827616662620356,
67
+ "pass@64": 0.9859211476022041,
68
+ "pass@128": 0.989
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9538828125,
72
+ "pass@2": 0.9683150836614177,
73
+ "pass@4": 0.9770699007311582,
74
+ "pass@8": 0.9835945887414447,
75
+ "pass@16": 0.988634619084515,
76
+ "pass@32": 0.9923047369821443,
77
+ "pass@64": 0.9946799358205762,
78
+ "pass@128": 0.996
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9616796875,
82
+ "pass@2": 0.9749260580708665,
83
+ "pass@4": 0.9833017036932883,
84
+ "pass@8": 0.9889375836603118,
85
+ "pass@16": 0.992596462343972,
86
+ "pass@32": 0.9949202064490399,
87
+ "pass@64": 0.9966019468994491,
88
+ "pass@128": 0.998
89
+ },
90
+ "8": {
91
+ "pass@1": 0.940296875,
92
+ "pass@2": 0.9630469980314962,
93
+ "pass@4": 0.9771136480127484,
94
+ "pass@8": 0.9858646091378575,
95
+ "pass@16": 0.9910784862737938,
96
+ "pass@32": 0.9945541301929133,
97
+ "pass@64": 0.9975505786482517,
98
+ "pass@128": 1.0
99
+ },
100
+ "9": {
101
+ "pass@1": 0.934796875,
102
+ "pass@2": 0.961122047244095,
103
+ "pass@4": 0.9741320773340832,
104
+ "pass@8": 0.9809906059803875,
105
+ "pass@16": 0.9853290159268713,
106
+ "pass@32": 0.9886914955844506,
107
+ "pass@64": 0.9915247697364523,
108
+ "pass@128": 0.994
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9013359375,
113
+ "2": 0.9834765625,
114
+ "3": 0.9914375,
115
+ "4": 0.9889921875,
116
+ "5": 0.9543515625,
117
+ "6": 0.9538828125,
118
+ "7": 0.9616796875,
119
+ "8": 0.940296875,
120
+ "9": 0.934796875
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 279.94903125,
124
+ "2": 117.7111875,
125
+ "3": 145.45546875,
126
+ "4": 160.1310546875,
127
+ "5": 175.9252734375,
128
+ "6": 201.25878125,
129
+ "7": 225.22690625,
130
+ "8": 244.521140625,
131
+ "9": 259.5079609375
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 279.94903125,
136
+ "2": 117.7111875,
137
+ "3": 145.45546875,
138
+ "4": 160.1310546875,
139
+ "5": 175.9252734375,
140
+ "6": 201.25878125,
141
+ "7": 225.22690625,
142
+ "8": 244.521140625,
143
+ "9": 259.5079609375
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 360710,
150
+ "answer_accuracy": 0.9517213356973995,
151
+ "avg_response_len": 191.63888625042216,
152
+ "resp_tokens_sum": 72632671,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9517213356973995,
155
+ "pass@2": 0.9669526016694736,
156
+ "pass@4": 0.9763565958130618,
157
+ "pass@8": 0.9828898231964195,
158
+ "pass@16": 0.987507739394257,
159
+ "pass@32": 0.9905947455484967,
160
+ "pass@64": 0.9928967227104302,
161
+ "pass@128": 0.9952718676122931
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9076327433628318,
166
+ "pass@2": 0.9269853436462037,
167
+ "pass@4": 0.9398492909625238,
168
+ "pass@8": 0.9506059829868715,
169
+ "pass@16": 0.9596484055654059,
170
+ "pass@32": 0.9658798721555774,
171
+ "pass@64": 0.9688322453458257,
172
+ "pass@128": 0.9705014749262537
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9780957943925234,
176
+ "pass@2": 0.9850905757107465,
177
+ "pass@4": 0.9894686271692673,
178
+ "pass@8": 0.9920504616490534,
179
+ "pass@16": 0.9933079829828495,
180
+ "pass@32": 0.9937265331983628,
181
+ "pass@64": 0.9937693846777408,
182
+ "pass@128": 0.9937694704049844
183
+ },
184
+ "3": {
185
+ "pass@1": 0.991829128440367,
186
+ "pass@2": 0.9972839010089335,
187
+ "pass@4": 0.9991199451559382,
188
+ "pass@8": 0.9998000718183061,
189
+ "pass@16": 0.9999896701598865,
190
+ "pass@32": 0.999999988244179,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.9865234375,
196
+ "pass@2": 0.9907341904527559,
197
+ "pass@4": 0.9941022271630107,
198
+ "pass@8": 0.9962775829156062,
199
+ "pass@16": 0.997180906859181,
200
+ "pass@32": 0.9976553064004617,
201
+ "pass@64": 0.9984374999950079,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9469990079365079,
206
+ "pass@2": 0.9564476315460567,
207
+ "pass@4": 0.9630626984126983,
208
+ "pass@8": 0.9694378790828586,
209
+ "pass@16": 0.9759160456546958,
210
+ "pass@32": 0.9818998343552849,
211
+ "pass@64": 0.9875177250605657,
212
+ "pass@128": 0.9936507936507937
213
+ },
214
+ "6": {
215
+ "pass@1": 0.926529752994012,
216
+ "pass@2": 0.9463030140506391,
217
+ "pass@4": 0.9610347533279894,
218
+ "pass@8": 0.9741423767959982,
219
+ "pass@16": 0.9846537870946782,
220
+ "pass@32": 0.9919116519382953,
221
+ "pass@64": 0.996595146234816,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9607063609467456,
226
+ "pass@2": 0.9766032678330153,
227
+ "pass@4": 0.9867770337879364,
228
+ "pass@8": 0.9932539965998115,
229
+ "pass@16": 0.9966691243987713,
230
+ "pass@32": 0.997758364804311,
231
+ "pass@64": 0.9985207077144822,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9363231169871795,
236
+ "pass@2": 0.9618858677821527,
237
+ "pass@4": 0.9778153304755176,
238
+ "pass@8": 0.9872242972247056,
239
+ "pass@16": 0.9919063397806129,
240
+ "pass@32": 0.9944326739175762,
241
+ "pass@64": 0.9972112783705197,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9345290492957746,
246
+ "pass@2": 0.9633556753909281,
247
+ "pass@4": 0.977406277032272,
248
+ "pass@8": 0.9842799230966626,
249
+ "pass@16": 0.9890320300163855,
250
+ "pass@32": 0.9926185701753283,
251
+ "pass@64": 0.9956966535148944,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 375668,
259
+ "answer_accuracy": 0.9616337647444299,
260
+ "avg_response_len": 198.80759542922672,
261
+ "resp_tokens_sum": 77665380,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9616337647444299,
264
+ "pass@2": 0.9739375996506754,
265
+ "pass@4": 0.9813179619442245,
266
+ "pass@8": 0.9862698702130008,
267
+ "pass@16": 0.9896383230988169,
268
+ "pass@32": 0.9920310335769109,
269
+ "pass@64": 0.9939622377244645,
270
+ "pass@128": 0.9954128440366973
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.8978144171779141,
275
+ "pass@2": 0.9213515500458915,
276
+ "pass@4": 0.9390322137646904,
277
+ "pass@8": 0.9526189121876671,
278
+ "pass@16": 0.9621797529616819,
279
+ "pass@32": 0.9684335022171842,
280
+ "pass@64": 0.9742357566082435,
281
+ "pass@128": 0.9815950920245399
282
+ },
283
+ "2": {
284
+ "pass@1": 0.991950037993921,
285
+ "pass@2": 0.9956385521623626,
286
+ "pass@4": 0.996789173769692,
287
+ "pass@8": 0.9969556274713829,
288
+ "pass@16": 0.996960482749072,
289
+ "pass@32": 0.9969604863221885,
290
+ "pass@64": 0.9969604863221885,
291
+ "pass@128": 0.9969604863221885
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9910748106060606,
295
+ "pass@2": 0.9938342579336673,
296
+ "pass@4": 0.9954610878185681,
297
+ "pass@8": 0.9967861359234476,
298
+ "pass@16": 0.9983705508051556,
299
+ "pass@32": 0.999643213120596,
300
+ "pass@64": 0.99999359856941,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9862934541420119,
305
+ "pass@2": 0.9914085431906072,
306
+ "pass@4": 0.9944976862174774,
307
+ "pass@8": 0.9968238286910404,
308
+ "pass@16": 0.9986471171735518,
309
+ "pass@32": 0.9997422779717037,
310
+ "pass@64": 0.9999955420471618,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9700860507246377,
315
+ "pass@2": 0.9768297814675341,
316
+ "pass@4": 0.9813448272770252,
317
+ "pass@8": 0.9845774240321683,
318
+ "pass@16": 0.9867515567319303,
319
+ "pass@32": 0.9884426560783856,
320
+ "pass@64": 0.9902316406247298,
321
+ "pass@128": 0.991304347826087
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9678545845272206,
325
+ "pass@2": 0.9800999199061439,
326
+ "pass@4": 0.9855557597742974,
327
+ "pass@8": 0.988306274970953,
328
+ "pass@16": 0.9904143515720015,
329
+ "pass@32": 0.9923533345945439,
330
+ "pass@64": 0.9938368819680918,
331
+ "pass@128": 0.994269340974212
332
+ },
333
+ "7": {
334
+ "pass@1": 0.965904985549133,
335
+ "pass@2": 0.9778386850848846,
336
+ "pass@4": 0.9852525098018816,
337
+ "pass@8": 0.9905732375817846,
338
+ "pass@16": 0.9943721417100441,
339
+ "pass@32": 0.997168595303731,
340
+ "pass@64": 0.9991059501069279,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9451804577464789,
345
+ "pass@2": 0.9669578989686152,
346
+ "pass@4": 0.9803207029579047,
347
+ "pass@8": 0.9881750952049374,
348
+ "pass@16": 0.9925613357660423,
349
+ "pass@32": 0.9958566064759846,
350
+ "pass@64": 0.9987507974877009,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9378508607784432,
355
+ "pass@2": 0.960500259323872,
356
+ "pass@4": 0.9725681423991157,
357
+ "pass@8": 0.9806556822439841,
358
+ "pass@16": 0.9856425911302683,
359
+ "pass@32": 0.988864619985313,
360
+ "pass@64": 0.9917644684832534,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 365734,
368
+ "answer_accuracy": 0.9565774606628724,
369
+ "avg_response_len": 212.74967567793772,
370
+ "resp_tokens_sum": 81341860,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9565774606628724,
373
+ "pass@2": 0.9700883124774305,
374
+ "pass@4": 0.9783811196474563,
375
+ "pass@8": 0.9837131026152431,
376
+ "pass@16": 0.9871975503495145,
377
+ "pass@32": 0.989778399899347,
378
+ "pass@64": 0.9918867752180041,
379
+ "pass@128": 0.9936391027787077
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.8983908582089553,
384
+ "pass@2": 0.9294809172640736,
385
+ "pass@4": 0.950525032318721,
386
+ "pass@8": 0.9653934920081807,
387
+ "pass@16": 0.9748415411280041,
388
+ "pass@32": 0.9810193030176554,
389
+ "pass@64": 0.9861676540954208,
390
+ "pass@128": 0.991044776119403
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9804464285714286,
394
+ "pass@2": 0.9854861501687289,
395
+ "pass@4": 0.9888010849536667,
396
+ "pass@8": 0.9906410014232532,
397
+ "pass@16": 0.9919181611918471,
398
+ "pass@32": 0.99303540900536,
399
+ "pass@64": 0.9939368305843209,
400
+ "pass@128": 0.9942857142857143
401
+ },
402
+ "3": {
403
+ "pass@1": 0.991413083090379,
404
+ "pass@2": 0.9955575945226233,
405
+ "pass@4": 0.9977777331405003,
406
+ "pass@8": 0.9991937098766038,
407
+ "pass@16": 0.9998749765033144,
408
+ "pass@32": 0.9999973906863244,
409
+ "pass@64": 0.9999999999217336,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9939692982456141,
414
+ "pass@2": 0.9942855107749691,
415
+ "pass@4": 0.994424011143344,
416
+ "pass@8": 0.9946902196435972,
417
+ "pass@16": 0.9952053690657088,
418
+ "pass@32": 0.9961665975963532,
419
+ "pass@64": 0.9978127734033246,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9451976102941176,
424
+ "pass@2": 0.9563129631773971,
425
+ "pass@4": 0.9654465556695121,
426
+ "pass@8": 0.9718827265720327,
427
+ "pass@16": 0.9754730011095328,
428
+ "pass@32": 0.9777955943914058,
429
+ "pass@64": 0.9800680829193935,
430
+ "pass@128": 0.9823529411764705
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9673205835962145,
434
+ "pass@2": 0.9785331385528704,
435
+ "pass@4": 0.9846225014459944,
436
+ "pass@8": 0.9883663877814471,
437
+ "pass@16": 0.9908695441522511,
438
+ "pass@32": 0.9926653989314137,
439
+ "pass@64": 0.9935901740419045,
440
+ "pass@128": 0.9936908517350158
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9580943433544303,
444
+ "pass@2": 0.9699429383035981,
445
+ "pass@4": 0.9774484110174518,
446
+ "pass@8": 0.9825297234375893,
447
+ "pass@16": 0.9862960040047856,
448
+ "pass@32": 0.9894226112977601,
449
+ "pass@64": 0.9918078764397377,
450
+ "pass@128": 0.9936708860759493
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9388138138138138,
454
+ "pass@2": 0.9599656250738929,
455
+ "pass@4": 0.9730372833463381,
456
+ "pass@8": 0.982127536354344,
457
+ "pass@16": 0.9887220242201132,
458
+ "pass@32": 0.9932794012962612,
459
+ "pass@64": 0.9965889690345809,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9318227491961415,
464
+ "pass@2": 0.9592401796338966,
465
+ "pass@4": 0.9720742425283658,
466
+ "pass@8": 0.9775956122558883,
467
+ "pass@16": 0.9807653370853542,
468
+ "pass@32": 0.9840228942029388,
469
+ "pass@64": 0.9865052259333055,
470
+ "pass@128": 0.9871382636655949
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.510975,
478
+ "avg_response_len": 362.007928125,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.510975,
484
+ "pass@2": 0.5936387180118026,
485
+ "pass@4": 0.6617797073396955,
486
+ "pass@8": 0.7167137733109136,
487
+ "pass@16": 0.7611447510468944,
488
+ "pass@32": 0.7983214509893242,
489
+ "pass@64": 0.8300479591802541,
490
+ "pass@128": 0.8567
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.89196875,
495
+ "pass@2": 0.9299221210629928,
496
+ "pass@4": 0.9529178682977126,
497
+ "pass@8": 0.9672675175171029,
498
+ "pass@16": 0.9756364231803202,
499
+ "pass@32": 0.9807627127916235,
500
+ "pass@64": 0.9844788117192562,
501
+ "pass@128": 0.987
502
+ },
503
+ "12": {
504
+ "pass@1": 0.841203125,
505
+ "pass@2": 0.8876090059055125,
506
+ "pass@4": 0.9141195879265096,
507
+ "pass@8": 0.9306535756844154,
508
+ "pass@16": 0.9413871883991257,
509
+ "pass@32": 0.9492624080901529,
510
+ "pass@64": 0.9559608145833538,
511
+ "pass@128": 0.962
512
+ },
513
+ "13": {
514
+ "pass@1": 0.6938671875,
515
+ "pass@2": 0.7763882874015748,
516
+ "pass@4": 0.8326519359767524,
517
+ "pass@8": 0.8707372795349085,
518
+ "pass@16": 0.8970861149243851,
519
+ "pass@32": 0.9176561586813041,
520
+ "pass@64": 0.9349613423561156,
521
+ "pass@128": 0.949
522
+ },
523
+ "14": {
524
+ "pass@1": 0.62446875,
525
+ "pass@2": 0.7159948326771655,
526
+ "pass@4": 0.7863533314585672,
527
+ "pass@8": 0.8389012067186266,
528
+ "pass@16": 0.8768195969158379,
529
+ "pass@32": 0.9040171453745361,
530
+ "pass@64": 0.9236577047523269,
531
+ "pass@128": 0.937
532
+ },
533
+ "15": {
534
+ "pass@1": 0.5383828125,
535
+ "pass@2": 0.6369622293307086,
536
+ "pass@4": 0.7131546289838765,
537
+ "pass@8": 0.769510564426078,
538
+ "pass@16": 0.8106159119539578,
539
+ "pass@32": 0.8417999582459403,
540
+ "pass@64": 0.8667511771745826,
541
+ "pass@128": 0.886
542
+ },
543
+ "16": {
544
+ "pass@1": 0.4368359375,
545
+ "pass@2": 0.5438690944881892,
546
+ "pass@4": 0.6312509509748778,
547
+ "pass@8": 0.6990218214156836,
548
+ "pass@16": 0.7515598142013872,
549
+ "pass@32": 0.7941632145309236,
550
+ "pass@64": 0.8296922098460493,
551
+ "pass@128": 0.859
552
+ },
553
+ "17": {
554
+ "pass@1": 0.352109375,
555
+ "pass@2": 0.4523837352362206,
556
+ "pass@4": 0.5387510978627673,
557
+ "pass@8": 0.6097353386899184,
558
+ "pass@16": 0.6696671824626427,
559
+ "pass@32": 0.7233398987358384,
560
+ "pass@64": 0.7717775833444259,
561
+ "pass@128": 0.817
562
+ },
563
+ "18": {
564
+ "pass@1": 0.3154453125,
565
+ "pass@2": 0.41220497047244115,
566
+ "pass@4": 0.49942514810648647,
567
+ "pass@8": 0.572338038929963,
568
+ "pass@16": 0.6328966641461382,
569
+ "pass@32": 0.6842474572023839,
570
+ "pass@64": 0.7286842008450987,
571
+ "pass@128": 0.767
572
+ },
573
+ "19": {
574
+ "pass@1": 0.211578125,
575
+ "pass@2": 0.294009350393701,
576
+ "pass@4": 0.3782810793963256,
577
+ "pass@8": 0.45912572627706577,
578
+ "pass@16": 0.5336634516261636,
579
+ "pass@32": 0.5996903294576137,
580
+ "pass@64": 0.6561316526827659,
581
+ "pass@128": 0.704
582
+ },
583
+ "20": {
584
+ "pass@1": 0.203890625,
585
+ "pass@2": 0.2870435531496063,
586
+ "pass@4": 0.3708914444131982,
587
+ "pass@8": 0.44984666391540407,
588
+ "pass@16": 0.5221151626588344,
589
+ "pass@32": 0.588275226782918,
590
+ "pass@64": 0.6483840944984783,
591
+ "pass@128": 0.699
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.89196875,
596
+ "12": 0.841203125,
597
+ "13": 0.6938671875,
598
+ "14": 0.62446875,
599
+ "15": 0.5383828125,
600
+ "16": 0.4368359375,
601
+ "17": 0.352109375,
602
+ "18": 0.3154453125,
603
+ "19": 0.211578125,
604
+ "20": 0.203890625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 298.1007421875,
608
+ "12": 308.0588671875,
609
+ "13": 326.875390625,
610
+ "14": 349.399546875,
611
+ "15": 365.40840625,
612
+ "16": 372.7167734375,
613
+ "17": 389.5476953125,
614
+ "18": 398.404875,
615
+ "19": 399.2373515625,
616
+ "20": 412.3296328125
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 298.1007421875,
621
+ "12": 308.0588671875,
622
+ "13": 326.875390625,
623
+ "14": 349.399546875,
624
+ "15": 365.40840625,
625
+ "16": 372.7167734375,
626
+ "17": 389.5476953125,
627
+ "18": 398.404875,
628
+ "19": 399.2373515625,
629
+ "20": 412.3296328125
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 223546,
636
+ "answer_accuracy": 0.5190053863298663,
637
+ "avg_response_len": 343.9361325222883,
638
+ "resp_tokens_sum": 148140171,
639
+ "pass_at_k": {
640
+ "pass@1": 0.5190053863298663,
641
+ "pass@2": 0.6017590761778867,
642
+ "pass@4": 0.6688807888334177,
643
+ "pass@8": 0.7220798095471659,
644
+ "pass@16": 0.7645909661017662,
645
+ "pass@32": 0.8002128845619237,
646
+ "pass@64": 0.8311640113734907,
647
+ "pass@128": 0.8564635958395245
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.8755975566572238,
652
+ "pass@2": 0.9161817436595214,
653
+ "pass@4": 0.9432045066604631,
654
+ "pass@8": 0.9603687852296585,
655
+ "pass@16": 0.9706278871438179,
656
+ "pass@32": 0.9775053297819941,
657
+ "pass@64": 0.981852619948499,
658
+ "pass@128": 0.9830028328611898
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8450300141242938,
662
+ "pass@2": 0.8938623965701327,
663
+ "pass@4": 0.9229093023859299,
664
+ "pass@8": 0.9409758597723392,
665
+ "pass@16": 0.9516659432562773,
666
+ "pass@32": 0.9588886011978869,
667
+ "pass@64": 0.965495662578892,
668
+ "pass@128": 0.9717514124293786
669
+ },
670
+ "13": {
671
+ "pass@1": 0.7161295062695925,
672
+ "pass@2": 0.7977161651815466,
673
+ "pass@4": 0.8520592728612683,
674
+ "pass@8": 0.8873050572913995,
675
+ "pass@16": 0.9106471139096689,
676
+ "pass@32": 0.9292828710771102,
677
+ "pass@64": 0.9443766484455557,
678
+ "pass@128": 0.9561128526645768
679
+ },
680
+ "14": {
681
+ "pass@1": 0.6268983644859814,
682
+ "pass@2": 0.7150230272033751,
683
+ "pass@4": 0.7842801770105835,
684
+ "pass@8": 0.837397655361588,
685
+ "pass@16": 0.8762786423304784,
686
+ "pass@32": 0.9046818450843452,
687
+ "pass@64": 0.9289289086935143,
688
+ "pass@128": 0.9470404984423676
689
+ },
690
+ "15": {
691
+ "pass@1": 0.5480717124277457,
692
+ "pass@2": 0.6457133835510445,
693
+ "pass@4": 0.7187044204373292,
694
+ "pass@8": 0.7727291108200902,
695
+ "pass@16": 0.8139621475751042,
696
+ "pass@32": 0.8458336523771858,
697
+ "pass@64": 0.8698378477573592,
698
+ "pass@128": 0.8872832369942196
699
+ },
700
+ "16": {
701
+ "pass@1": 0.4441280241935484,
702
+ "pass@2": 0.5569766795783586,
703
+ "pass@4": 0.6460751194407147,
704
+ "pass@8": 0.7137075598181689,
705
+ "pass@16": 0.7681949863235685,
706
+ "pass@32": 0.8142011454810046,
707
+ "pass@64": 0.8517005897273575,
708
+ "pass@128": 0.8774193548387097
709
+ },
710
+ "17": {
711
+ "pass@1": 0.36806809971098264,
712
+ "pass@2": 0.4666509973373991,
713
+ "pass@4": 0.5472581679638306,
714
+ "pass@8": 0.6127021215261312,
715
+ "pass@16": 0.6691159679443502,
716
+ "pass@32": 0.7203100001054069,
717
+ "pass@64": 0.7654746436146028,
718
+ "pass@128": 0.8063583815028902
719
+ },
720
+ "18": {
721
+ "pass@1": 0.304375,
722
+ "pass@2": 0.3967902840269967,
723
+ "pass@4": 0.47884594809577347,
724
+ "pass@8": 0.5467893711314302,
725
+ "pass@16": 0.6037676808016317,
726
+ "pass@32": 0.6536113740422012,
727
+ "pass@64": 0.6999131573501903,
728
+ "pass@128": 0.74
729
+ },
730
+ "19": {
731
+ "pass@1": 0.21579391891891891,
732
+ "pass@2": 0.3006612665815421,
733
+ "pass@4": 0.38616035720760156,
734
+ "pass@8": 0.46550902965231605,
735
+ "pass@16": 0.5350209172548729,
736
+ "pass@32": 0.595347618281271,
737
+ "pass@64": 0.6482623365836448,
738
+ "pass@128": 0.6936936936936937
739
+ },
740
+ "20": {
741
+ "pass@1": 0.22670326576576577,
742
+ "pass@2": 0.3139590131470051,
743
+ "pass@4": 0.39960391071611545,
744
+ "pass@8": 0.4769628415256585,
745
+ "pass@16": 0.5428805161447351,
746
+ "pass@32": 0.6007913078306004,
747
+ "pass@64": 0.6557323133549934,
748
+ "pass@128": 0.7027027027027027
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 218386,
755
+ "answer_accuracy": 0.50732697740113,
756
+ "avg_response_len": 357.6412940454951,
757
+ "resp_tokens_sum": 153951702,
758
+ "pass_at_k": {
759
+ "pass@1": 0.50732697740113,
760
+ "pass@2": 0.5890962047033392,
761
+ "pass@4": 0.6570096634051469,
762
+ "pass@8": 0.7120303717421669,
763
+ "pass@16": 0.7565837178460383,
764
+ "pass@32": 0.794083787266784,
765
+ "pass@64": 0.8263549990088304,
766
+ "pass@128": 0.8542967588462682
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9046733810240963,
771
+ "pass@2": 0.9394221818375864,
772
+ "pass@4": 0.9586932646220422,
773
+ "pass@8": 0.9705318114541525,
774
+ "pass@16": 0.9775325668360785,
775
+ "pass@32": 0.9814679960471665,
776
+ "pass@64": 0.9848638566163492,
777
+ "pass@128": 0.9879518072289156
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8320190047021944,
781
+ "pass@2": 0.8726735862562636,
782
+ "pass@4": 0.8965225433223666,
783
+ "pass@8": 0.9119176475126256,
784
+ "pass@16": 0.9216786817337147,
785
+ "pass@32": 0.9290268632604375,
786
+ "pass@64": 0.9362206325610609,
787
+ "pass@128": 0.9435736677115988
788
+ },
789
+ "13": {
790
+ "pass@1": 0.6824010854341737,
791
+ "pass@2": 0.7620663915172362,
792
+ "pass@4": 0.816710596574587,
793
+ "pass@8": 0.8568656338449239,
794
+ "pass@16": 0.8863689471035519,
795
+ "pass@32": 0.9093439500882927,
796
+ "pass@64": 0.929124181128099,
797
+ "pass@128": 0.9467787114845938
798
+ },
799
+ "14": {
800
+ "pass@1": 0.6183982090643275,
801
+ "pass@2": 0.7113501231753926,
802
+ "pass@4": 0.7827317758525795,
803
+ "pass@8": 0.8349723554689547,
804
+ "pass@16": 0.8735588597859756,
805
+ "pass@32": 0.9025529986048424,
806
+ "pass@64": 0.9217735735634356,
807
+ "pass@128": 0.9327485380116959
808
+ },
809
+ "15": {
810
+ "pass@1": 0.5301846590909091,
811
+ "pass@2": 0.6305125566690525,
812
+ "pass@4": 0.7065275675767798,
813
+ "pass@8": 0.7606926843299037,
814
+ "pass@16": 0.7990498123628644,
815
+ "pass@32": 0.8293580860368167,
816
+ "pass@64": 0.8559740595796936,
817
+ "pass@128": 0.8787878787878788
818
+ },
819
+ "16": {
820
+ "pass@1": 0.4334045584045584,
821
+ "pass@2": 0.5336198459968144,
822
+ "pass@4": 0.6156393698650913,
823
+ "pass@8": 0.6793719515351224,
824
+ "pass@16": 0.7282840734236551,
825
+ "pass@32": 0.7671109052665265,
826
+ "pass@64": 0.800987680746752,
827
+ "pass@128": 0.8319088319088319
828
+ },
829
+ "17": {
830
+ "pass@1": 0.3566818711180124,
831
+ "pass@2": 0.46245017606494854,
832
+ "pass@4": 0.5551400752700942,
833
+ "pass@8": 0.6291890200102845,
834
+ "pass@16": 0.6878346867297733,
835
+ "pass@32": 0.7384604652449943,
836
+ "pass@64": 0.7881347414072649,
837
+ "pass@128": 0.8385093167701864
838
+ },
839
+ "18": {
840
+ "pass@1": 0.32173713235294116,
841
+ "pass@2": 0.42347802802223233,
842
+ "pass@4": 0.5183494841637442,
843
+ "pass@8": 0.5990558759706359,
844
+ "pass@16": 0.6651894072773802,
845
+ "pass@32": 0.7191764944894445,
846
+ "pass@64": 0.7606179403198643,
847
+ "pass@128": 0.7911764705882353
848
+ },
849
+ "19": {
850
+ "pass@1": 0.20986793154761904,
851
+ "pass@2": 0.2867501669713159,
852
+ "pass@4": 0.3652424758289142,
853
+ "pass@8": 0.4410855714382008,
854
+ "pass@16": 0.5130674911401856,
855
+ "pass@32": 0.5803165592539755,
856
+ "pass@64": 0.6393250538858533,
857
+ "pass@128": 0.6934523809523809
858
+ },
859
+ "20": {
860
+ "pass@1": 0.18988585329341318,
861
+ "pass@2": 0.2721880304351926,
862
+ "pass@4": 0.35637058588110615,
863
+ "pass@8": 0.4371482552763727,
864
+ "pass@16": 0.512849984475913,
865
+ "pass@32": 0.5830933497627412,
866
+ "pass@64": 0.645622648235715,
867
+ "pass@128": 0.6976047904191617
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 212116,
874
+ "answer_accuracy": 0.5064658465770171,
875
+ "avg_response_len": 385.08145581830684,
876
+ "resp_tokens_sum": 161278275,
877
+ "pass_at_k": {
878
+ "pass@1": 0.5064658465770171,
879
+ "pass@2": 0.589956403533684,
880
+ "pass@4": 0.6593794990651408,
881
+ "pass@8": 0.7160088734150627,
882
+ "pass@16": 0.7622884677323257,
883
+ "pass@32": 0.8007317777396525,
884
+ "pass@64": 0.8326958532591422,
885
+ "pass@128": 0.8594132029339854
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.8969246031746032,
890
+ "pass@2": 0.9353073053368325,
891
+ "pass@4": 0.9577159161652413,
892
+ "pass@8": 0.9715580156452523,
893
+ "pass@16": 0.9792506883459516,
894
+ "pass@32": 0.9836697037806984,
895
+ "pass@64": 0.9870159888279616,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8460196865443425,
900
+ "pass@2": 0.8954093073514888,
901
+ "pass@4": 0.921770653094968,
902
+ "pass@8": 0.9377565497507031,
903
+ "pass@16": 0.9494860703772144,
904
+ "pass@32": 0.9585818773272825,
905
+ "pass@64": 0.9648959273499919,
906
+ "pass@128": 0.9694189602446484
907
+ },
908
+ "13": {
909
+ "pass@1": 0.6845823688271605,
910
+ "pass@2": 0.7711701510401477,
911
+ "pass@4": 0.831109089373087,
912
+ "pass@8": 0.8697096758528202,
913
+ "pass@16": 0.8955431218247937,
914
+ "pass@32": 0.9153676932906923,
915
+ "pass@64": 0.9321230211088019,
916
+ "pass@128": 0.9444444444444444
917
+ },
918
+ "14": {
919
+ "pass@1": 0.6283150964391692,
920
+ "pass@2": 0.721634121124325,
921
+ "pass@4": 0.7920033450937324,
922
+ "pass@8": 0.844320515659269,
923
+ "pass@16": 0.8806439842164706,
924
+ "pass@32": 0.9048698741234554,
925
+ "pass@64": 0.9205488513442519,
926
+ "pass@128": 0.9317507418397626
927
+ },
928
+ "15": {
929
+ "pass@1": 0.5363859953703703,
930
+ "pass@2": 0.6341859719791972,
931
+ "pass@4": 0.713977784605628,
932
+ "pass@8": 0.7750546489304276,
933
+ "pass@16": 0.8188227494235403,
934
+ "pass@32": 0.8501646485533466,
935
+ "pass@64": 0.8744316116951758,
936
+ "pass@128": 0.8919753086419753
937
+ },
938
+ "16": {
939
+ "pass@1": 0.43372050147492625,
940
+ "pass@2": 0.5424948609852972,
941
+ "pass@4": 0.6338591301640388,
942
+ "pass@8": 0.7059378256142278,
943
+ "pass@16": 0.760447370706132,
944
+ "pass@32": 0.8038493559978188,
945
+ "pass@64": 0.8392871713523845,
946
+ "pass@128": 0.8702064896755162
947
+ },
948
+ "17": {
949
+ "pass@1": 0.331042921686747,
950
+ "pass@2": 0.42775160682098473,
951
+ "pass@4": 0.5139899623804555,
952
+ "pass@8": 0.5877757234896518,
953
+ "pass@16": 0.6526213506834679,
954
+ "pass@32": 0.7118324364171099,
955
+ "pass@64": 0.7624818371103406,
956
+ "pass@128": 0.8072289156626506
957
+ },
958
+ "18": {
959
+ "pass@1": 0.32104334677419355,
960
+ "pass@2": 0.41724465011430023,
961
+ "pass@4": 0.5019040053461057,
962
+ "pass@8": 0.5718798748514401,
963
+ "pass@16": 0.6303663786814787,
964
+ "pass@32": 0.6805273166490414,
965
+ "pass@64": 0.7261435356896053,
966
+ "pass@128": 0.7709677419354839
967
+ },
968
+ "19": {
969
+ "pass@1": 0.20907288519637462,
970
+ "pass@2": 0.2946860801198942,
971
+ "pass@4": 0.3835897539809034,
972
+ "pass@8": 0.4710165178235919,
973
+ "pass@16": 0.5532048615020811,
974
+ "pass@32": 0.623725705923306,
975
+ "pass@64": 0.6811089924313154,
976
+ "pass@128": 0.7250755287009063
977
+ },
978
+ "20": {
979
+ "pass@1": 0.19512481231231232,
980
+ "pass@2": 0.2750282270459435,
981
+ "pass@4": 0.35674344282415144,
982
+ "pass@8": 0.435467028303459,
983
+ "pass@16": 0.510642810713764,
984
+ "pass@32": 0.5809565839476649,
985
+ "pass@64": 0.6438056145361447,
986
+ "pass@128": 0.6966966966966966
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.7221052631578947,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.7221052631578947,
1002
+ "pass@2": 0.7720865882718362,
1003
+ "pass@4": 0.8119050409389371,
1004
+ "pass@8": 0.8434695358418849,
1005
+ "pass@16": 0.8686627985161548,
1006
+ "pass@32": 0.8895006498125616,
1007
+ "pass@64": 0.9071992281504938,
1008
+ "pass@128": 0.9221052631578948
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9013359375,
1013
+ "pass@2": 0.9259847440944887,
1014
+ "pass@4": 0.9431592971503563,
1015
+ "pass@8": 0.956216013428469,
1016
+ "pass@16": 0.9655633252300617,
1017
+ "pass@32": 0.9717840648944569,
1018
+ "pass@64": 0.9764011519484884,
1019
+ "pass@128": 0.981
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.9834765625,
1023
+ "pass@2": 0.9886993110236222,
1024
+ "pass@4": 0.9916434472253468,
1025
+ "pass@8": 0.9931709501255696,
1026
+ "pass@16": 0.9940232177790858,
1027
+ "pass@32": 0.9945486103085506,
1028
+ "pass@64": 0.9948778631860671,
1029
+ "pass@128": 0.995
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.9914375,
1033
+ "pass@2": 0.9955533956692916,
1034
+ "pass@4": 0.9974521435133106,
1035
+ "pass@8": 0.9985974908269989,
1036
+ "pass@16": 0.9994160208486209,
1037
+ "pass@32": 0.9998813614910528,
1038
+ "pass@64": 0.99999788750106,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9889921875,
1043
+ "pass@2": 0.9921766732283467,
1044
+ "pass@4": 0.9943459424446944,
1045
+ "pass@8": 0.9959193357486759,
1046
+ "pass@16": 0.9970008520200708,
1047
+ "pass@32": 0.9978515643805362,
1048
+ "pass@64": 0.9987504617142802,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.9543515625,
1053
+ "pass@2": 0.9634336860236224,
1054
+ "pass@4": 0.9701805443382078,
1055
+ "pass@8": 0.9754922702366896,
1056
+ "pass@16": 0.9795036618309861,
1057
+ "pass@32": 0.9827616662620356,
1058
+ "pass@64": 0.9859211476022041,
1059
+ "pass@128": 0.989
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9538828125,
1063
+ "pass@2": 0.9683150836614177,
1064
+ "pass@4": 0.9770699007311582,
1065
+ "pass@8": 0.9835945887414447,
1066
+ "pass@16": 0.988634619084515,
1067
+ "pass@32": 0.9923047369821443,
1068
+ "pass@64": 0.9946799358205762,
1069
+ "pass@128": 0.996
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9616796875,
1073
+ "pass@2": 0.9749260580708665,
1074
+ "pass@4": 0.9833017036932883,
1075
+ "pass@8": 0.9889375836603118,
1076
+ "pass@16": 0.992596462343972,
1077
+ "pass@32": 0.9949202064490399,
1078
+ "pass@64": 0.9966019468994491,
1079
+ "pass@128": 0.998
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.940296875,
1083
+ "pass@2": 0.9630469980314962,
1084
+ "pass@4": 0.9771136480127484,
1085
+ "pass@8": 0.9858646091378575,
1086
+ "pass@16": 0.9910784862737938,
1087
+ "pass@32": 0.9945541301929133,
1088
+ "pass@64": 0.9975505786482517,
1089
+ "pass@128": 1.0
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.934796875,
1093
+ "pass@2": 0.961122047244095,
1094
+ "pass@4": 0.9741320773340832,
1095
+ "pass@8": 0.9809906059803875,
1096
+ "pass@16": 0.9853290159268713,
1097
+ "pass@32": 0.9886914955844506,
1098
+ "pass@64": 0.9915247697364523,
1099
+ "pass@128": 0.994
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.89196875,
1103
+ "pass@2": 0.9299221210629928,
1104
+ "pass@4": 0.9529178682977126,
1105
+ "pass@8": 0.9672675175171029,
1106
+ "pass@16": 0.9756364231803202,
1107
+ "pass@32": 0.9807627127916235,
1108
+ "pass@64": 0.9844788117192562,
1109
+ "pass@128": 0.987
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.841203125,
1113
+ "pass@2": 0.8876090059055125,
1114
+ "pass@4": 0.9141195879265096,
1115
+ "pass@8": 0.9306535756844154,
1116
+ "pass@16": 0.9413871883991257,
1117
+ "pass@32": 0.9492624080901529,
1118
+ "pass@64": 0.9559608145833538,
1119
+ "pass@128": 0.962
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.6938671875,
1123
+ "pass@2": 0.7763882874015748,
1124
+ "pass@4": 0.8326519359767524,
1125
+ "pass@8": 0.8707372795349085,
1126
+ "pass@16": 0.8970861149243851,
1127
+ "pass@32": 0.9176561586813041,
1128
+ "pass@64": 0.9349613423561156,
1129
+ "pass@128": 0.949
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.62446875,
1133
+ "pass@2": 0.7159948326771655,
1134
+ "pass@4": 0.7863533314585672,
1135
+ "pass@8": 0.8389012067186266,
1136
+ "pass@16": 0.8768195969158379,
1137
+ "pass@32": 0.9040171453745361,
1138
+ "pass@64": 0.9236577047523269,
1139
+ "pass@128": 0.937
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.5383828125,
1143
+ "pass@2": 0.6369622293307086,
1144
+ "pass@4": 0.7131546289838765,
1145
+ "pass@8": 0.769510564426078,
1146
+ "pass@16": 0.8106159119539578,
1147
+ "pass@32": 0.8417999582459403,
1148
+ "pass@64": 0.8667511771745826,
1149
+ "pass@128": 0.886
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.4368359375,
1153
+ "pass@2": 0.5438690944881892,
1154
+ "pass@4": 0.6312509509748778,
1155
+ "pass@8": 0.6990218214156836,
1156
+ "pass@16": 0.7515598142013872,
1157
+ "pass@32": 0.7941632145309236,
1158
+ "pass@64": 0.8296922098460493,
1159
+ "pass@128": 0.859
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.352109375,
1163
+ "pass@2": 0.4523837352362206,
1164
+ "pass@4": 0.5387510978627673,
1165
+ "pass@8": 0.6097353386899184,
1166
+ "pass@16": 0.6696671824626427,
1167
+ "pass@32": 0.7233398987358384,
1168
+ "pass@64": 0.7717775833444259,
1169
+ "pass@128": 0.817
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.3154453125,
1173
+ "pass@2": 0.41220497047244115,
1174
+ "pass@4": 0.49942514810648647,
1175
+ "pass@8": 0.572338038929963,
1176
+ "pass@16": 0.6328966641461382,
1177
+ "pass@32": 0.6842474572023839,
1178
+ "pass@64": 0.7286842008450987,
1179
+ "pass@128": 0.767
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.211578125,
1183
+ "pass@2": 0.294009350393701,
1184
+ "pass@4": 0.3782810793963256,
1185
+ "pass@8": 0.45912572627706577,
1186
+ "pass@16": 0.5336634516261636,
1187
+ "pass@32": 0.5996903294576137,
1188
+ "pass@64": 0.6561316526827659,
1189
+ "pass@128": 0.704
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.203890625,
1193
+ "pass@2": 0.2870435531496063,
1194
+ "pass@4": 0.3708914444131982,
1195
+ "pass@8": 0.44984666391540407,
1196
+ "pass@16": 0.5221151626588344,
1197
+ "pass@32": 0.588275226782918,
1198
+ "pass@64": 0.6483840944984783,
1199
+ "pass@128": 0.699
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9013359375,
1204
+ "2": 0.9834765625,
1205
+ "3": 0.9914375,
1206
+ "4": 0.9889921875,
1207
+ "5": 0.9543515625,
1208
+ "6": 0.9538828125,
1209
+ "7": 0.9616796875,
1210
+ "8": 0.940296875,
1211
+ "9": 0.934796875,
1212
+ "11": 0.89196875,
1213
+ "12": 0.841203125,
1214
+ "13": 0.6938671875,
1215
+ "14": 0.62446875,
1216
+ "15": 0.5383828125,
1217
+ "16": 0.4368359375,
1218
+ "17": 0.352109375,
1219
+ "18": 0.3154453125,
1220
+ "19": 0.211578125,
1221
+ "20": 0.203890625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 279.94903125,
1225
+ "2": 117.7111875,
1226
+ "3": 145.45546875,
1227
+ "4": 160.1310546875,
1228
+ "5": 175.9252734375,
1229
+ "6": 201.25878125,
1230
+ "7": 225.22690625,
1231
+ "8": 244.521140625,
1232
+ "9": 259.5079609375,
1233
+ "11": 298.1007421875,
1234
+ "12": 308.0588671875,
1235
+ "13": 326.875390625,
1236
+ "14": 349.399546875,
1237
+ "15": 365.40840625,
1238
+ "16": 372.7167734375,
1239
+ "17": 389.5476953125,
1240
+ "18": 398.404875,
1241
+ "19": 399.2373515625,
1242
+ "20": 412.3296328125
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 279.94903125,
1247
+ "2": 117.7111875,
1248
+ "3": 145.45546875,
1249
+ "4": 160.1310546875,
1250
+ "5": 175.9252734375,
1251
+ "6": 201.25878125,
1252
+ "7": 225.22690625,
1253
+ "8": 244.521140625,
1254
+ "9": 259.5079609375,
1255
+ "11": 298.1007421875,
1256
+ "12": 308.0588671875,
1257
+ "13": 326.875390625,
1258
+ "14": 349.399546875,
1259
+ "15": 365.40840625,
1260
+ "16": 372.7167734375,
1261
+ "17": 389.5476953125,
1262
+ "18": 398.404875,
1263
+ "19": 399.2373515625,
1264
+ "20": 412.3296328125
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 584256,
1271
+ "answer_accuracy": 0.7215460006323111,
1272
+ "avg_response_len": 272.650620949257,
1273
+ "resp_tokens_sum": 220772842,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.7215460006323111,
1276
+ "pass@2": 0.7726945850271698,
1277
+ "pass@4": 0.8128004638992861,
1278
+ "pass@8": 0.844156706546122,
1279
+ "pass@16": 0.8689312388996039,
1280
+ "pass@32": 0.8893245966044871,
1281
+ "pass@64": 0.9068659649411043,
1282
+ "pass@128": 0.9214353461903256
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9076327433628318,
1287
+ "pass@2": 0.9269853436462037,
1288
+ "pass@4": 0.9398492909625238,
1289
+ "pass@8": 0.9506059829868715,
1290
+ "pass@16": 0.9596484055654059,
1291
+ "pass@32": 0.9658798721555774,
1292
+ "pass@64": 0.9688322453458257,
1293
+ "pass@128": 0.9705014749262537
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9780957943925234,
1297
+ "pass@2": 0.9850905757107465,
1298
+ "pass@4": 0.9894686271692673,
1299
+ "pass@8": 0.9920504616490534,
1300
+ "pass@16": 0.9933079829828495,
1301
+ "pass@32": 0.9937265331983628,
1302
+ "pass@64": 0.9937693846777408,
1303
+ "pass@128": 0.9937694704049844
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.991829128440367,
1307
+ "pass@2": 0.9972839010089335,
1308
+ "pass@4": 0.9991199451559382,
1309
+ "pass@8": 0.9998000718183061,
1310
+ "pass@16": 0.9999896701598865,
1311
+ "pass@32": 0.999999988244179,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.9865234375,
1317
+ "pass@2": 0.9907341904527559,
1318
+ "pass@4": 0.9941022271630107,
1319
+ "pass@8": 0.9962775829156062,
1320
+ "pass@16": 0.997180906859181,
1321
+ "pass@32": 0.9976553064004617,
1322
+ "pass@64": 0.9984374999950079,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9469990079365079,
1327
+ "pass@2": 0.9564476315460567,
1328
+ "pass@4": 0.9630626984126983,
1329
+ "pass@8": 0.9694378790828586,
1330
+ "pass@16": 0.9759160456546958,
1331
+ "pass@32": 0.9818998343552849,
1332
+ "pass@64": 0.9875177250605657,
1333
+ "pass@128": 0.9936507936507937
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.926529752994012,
1337
+ "pass@2": 0.9463030140506391,
1338
+ "pass@4": 0.9610347533279894,
1339
+ "pass@8": 0.9741423767959982,
1340
+ "pass@16": 0.9846537870946782,
1341
+ "pass@32": 0.9919116519382953,
1342
+ "pass@64": 0.996595146234816,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9607063609467456,
1347
+ "pass@2": 0.9766032678330153,
1348
+ "pass@4": 0.9867770337879364,
1349
+ "pass@8": 0.9932539965998115,
1350
+ "pass@16": 0.9966691243987713,
1351
+ "pass@32": 0.997758364804311,
1352
+ "pass@64": 0.9985207077144822,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9363231169871795,
1357
+ "pass@2": 0.9618858677821527,
1358
+ "pass@4": 0.9778153304755176,
1359
+ "pass@8": 0.9872242972247056,
1360
+ "pass@16": 0.9919063397806129,
1361
+ "pass@32": 0.9944326739175762,
1362
+ "pass@64": 0.9972112783705197,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9345290492957746,
1367
+ "pass@2": 0.9633556753909281,
1368
+ "pass@4": 0.977406277032272,
1369
+ "pass@8": 0.9842799230966626,
1370
+ "pass@16": 0.9890320300163855,
1371
+ "pass@32": 0.9926185701753283,
1372
+ "pass@64": 0.9956966535148944,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.8755975566572238,
1377
+ "pass@2": 0.9161817436595214,
1378
+ "pass@4": 0.9432045066604631,
1379
+ "pass@8": 0.9603687852296585,
1380
+ "pass@16": 0.9706278871438179,
1381
+ "pass@32": 0.9775053297819941,
1382
+ "pass@64": 0.981852619948499,
1383
+ "pass@128": 0.9830028328611898
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8450300141242938,
1387
+ "pass@2": 0.8938623965701327,
1388
+ "pass@4": 0.9229093023859299,
1389
+ "pass@8": 0.9409758597723392,
1390
+ "pass@16": 0.9516659432562773,
1391
+ "pass@32": 0.9588886011978869,
1392
+ "pass@64": 0.965495662578892,
1393
+ "pass@128": 0.9717514124293786
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.7161295062695925,
1397
+ "pass@2": 0.7977161651815466,
1398
+ "pass@4": 0.8520592728612683,
1399
+ "pass@8": 0.8873050572913995,
1400
+ "pass@16": 0.9106471139096689,
1401
+ "pass@32": 0.9292828710771102,
1402
+ "pass@64": 0.9443766484455557,
1403
+ "pass@128": 0.9561128526645768
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.6268983644859814,
1407
+ "pass@2": 0.7150230272033751,
1408
+ "pass@4": 0.7842801770105835,
1409
+ "pass@8": 0.837397655361588,
1410
+ "pass@16": 0.8762786423304784,
1411
+ "pass@32": 0.9046818450843452,
1412
+ "pass@64": 0.9289289086935143,
1413
+ "pass@128": 0.9470404984423676
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.5480717124277457,
1417
+ "pass@2": 0.6457133835510445,
1418
+ "pass@4": 0.7187044204373292,
1419
+ "pass@8": 0.7727291108200902,
1420
+ "pass@16": 0.8139621475751042,
1421
+ "pass@32": 0.8458336523771858,
1422
+ "pass@64": 0.8698378477573592,
1423
+ "pass@128": 0.8872832369942196
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.4441280241935484,
1427
+ "pass@2": 0.5569766795783586,
1428
+ "pass@4": 0.6460751194407147,
1429
+ "pass@8": 0.7137075598181689,
1430
+ "pass@16": 0.7681949863235685,
1431
+ "pass@32": 0.8142011454810046,
1432
+ "pass@64": 0.8517005897273575,
1433
+ "pass@128": 0.8774193548387097
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.36806809971098264,
1437
+ "pass@2": 0.4666509973373991,
1438
+ "pass@4": 0.5472581679638306,
1439
+ "pass@8": 0.6127021215261312,
1440
+ "pass@16": 0.6691159679443502,
1441
+ "pass@32": 0.7203100001054069,
1442
+ "pass@64": 0.7654746436146028,
1443
+ "pass@128": 0.8063583815028902
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.304375,
1447
+ "pass@2": 0.3967902840269967,
1448
+ "pass@4": 0.47884594809577347,
1449
+ "pass@8": 0.5467893711314302,
1450
+ "pass@16": 0.6037676808016317,
1451
+ "pass@32": 0.6536113740422012,
1452
+ "pass@64": 0.6999131573501903,
1453
+ "pass@128": 0.74
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.21579391891891891,
1457
+ "pass@2": 0.3006612665815421,
1458
+ "pass@4": 0.38616035720760156,
1459
+ "pass@8": 0.46550902965231605,
1460
+ "pass@16": 0.5350209172548729,
1461
+ "pass@32": 0.595347618281271,
1462
+ "pass@64": 0.6482623365836448,
1463
+ "pass@128": 0.6936936936936937
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.22670326576576577,
1467
+ "pass@2": 0.3139590131470051,
1468
+ "pass@4": 0.39960391071611545,
1469
+ "pass@8": 0.4769628415256585,
1470
+ "pass@16": 0.5428805161447351,
1471
+ "pass@32": 0.6007913078306004,
1472
+ "pass@64": 0.6557323133549934,
1473
+ "pass@128": 0.7027027027027027
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 594054,
1480
+ "answer_accuracy": 0.7234679462197974,
1481
+ "avg_response_len": 282.07458349571317,
1482
+ "resp_tokens_sum": 231617082,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.7234679462197974,
1485
+ "pass@2": 0.7721883227671293,
1486
+ "pass@4": 0.8113025592962161,
1487
+ "pass@8": 0.8425025384347558,
1488
+ "pass@16": 0.8674617623092497,
1489
+ "pass@32": 0.8882591568285177,
1490
+ "pass@64": 0.9060958084492297,
1491
+ "pass@128": 0.9214341387373344
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.8978144171779141,
1496
+ "pass@2": 0.9213515500458915,
1497
+ "pass@4": 0.9390322137646904,
1498
+ "pass@8": 0.9526189121876671,
1499
+ "pass@16": 0.9621797529616819,
1500
+ "pass@32": 0.9684335022171842,
1501
+ "pass@64": 0.9742357566082435,
1502
+ "pass@128": 0.9815950920245399
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.991950037993921,
1506
+ "pass@2": 0.9956385521623626,
1507
+ "pass@4": 0.996789173769692,
1508
+ "pass@8": 0.9969556274713829,
1509
+ "pass@16": 0.996960482749072,
1510
+ "pass@32": 0.9969604863221885,
1511
+ "pass@64": 0.9969604863221885,
1512
+ "pass@128": 0.9969604863221885
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9910748106060606,
1516
+ "pass@2": 0.9938342579336673,
1517
+ "pass@4": 0.9954610878185681,
1518
+ "pass@8": 0.9967861359234476,
1519
+ "pass@16": 0.9983705508051556,
1520
+ "pass@32": 0.999643213120596,
1521
+ "pass@64": 0.99999359856941,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9862934541420119,
1526
+ "pass@2": 0.9914085431906072,
1527
+ "pass@4": 0.9944976862174774,
1528
+ "pass@8": 0.9968238286910404,
1529
+ "pass@16": 0.9986471171735518,
1530
+ "pass@32": 0.9997422779717037,
1531
+ "pass@64": 0.9999955420471618,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9700860507246377,
1536
+ "pass@2": 0.9768297814675341,
1537
+ "pass@4": 0.9813448272770252,
1538
+ "pass@8": 0.9845774240321683,
1539
+ "pass@16": 0.9867515567319303,
1540
+ "pass@32": 0.9884426560783856,
1541
+ "pass@64": 0.9902316406247298,
1542
+ "pass@128": 0.991304347826087
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9678545845272206,
1546
+ "pass@2": 0.9800999199061439,
1547
+ "pass@4": 0.9855557597742974,
1548
+ "pass@8": 0.988306274970953,
1549
+ "pass@16": 0.9904143515720015,
1550
+ "pass@32": 0.9923533345945439,
1551
+ "pass@64": 0.9938368819680918,
1552
+ "pass@128": 0.994269340974212
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.965904985549133,
1556
+ "pass@2": 0.9778386850848846,
1557
+ "pass@4": 0.9852525098018816,
1558
+ "pass@8": 0.9905732375817846,
1559
+ "pass@16": 0.9943721417100441,
1560
+ "pass@32": 0.997168595303731,
1561
+ "pass@64": 0.9991059501069279,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9451804577464789,
1566
+ "pass@2": 0.9669578989686152,
1567
+ "pass@4": 0.9803207029579047,
1568
+ "pass@8": 0.9881750952049374,
1569
+ "pass@16": 0.9925613357660423,
1570
+ "pass@32": 0.9958566064759846,
1571
+ "pass@64": 0.9987507974877009,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9378508607784432,
1576
+ "pass@2": 0.960500259323872,
1577
+ "pass@4": 0.9725681423991157,
1578
+ "pass@8": 0.9806556822439841,
1579
+ "pass@16": 0.9856425911302683,
1580
+ "pass@32": 0.988864619985313,
1581
+ "pass@64": 0.9917644684832534,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9046733810240963,
1586
+ "pass@2": 0.9394221818375864,
1587
+ "pass@4": 0.9586932646220422,
1588
+ "pass@8": 0.9705318114541525,
1589
+ "pass@16": 0.9775325668360785,
1590
+ "pass@32": 0.9814679960471665,
1591
+ "pass@64": 0.9848638566163492,
1592
+ "pass@128": 0.9879518072289156
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8320190047021944,
1596
+ "pass@2": 0.8726735862562636,
1597
+ "pass@4": 0.8965225433223666,
1598
+ "pass@8": 0.9119176475126256,
1599
+ "pass@16": 0.9216786817337147,
1600
+ "pass@32": 0.9290268632604375,
1601
+ "pass@64": 0.9362206325610609,
1602
+ "pass@128": 0.9435736677115988
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.6824010854341737,
1606
+ "pass@2": 0.7620663915172362,
1607
+ "pass@4": 0.816710596574587,
1608
+ "pass@8": 0.8568656338449239,
1609
+ "pass@16": 0.8863689471035519,
1610
+ "pass@32": 0.9093439500882927,
1611
+ "pass@64": 0.929124181128099,
1612
+ "pass@128": 0.9467787114845938
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.6183982090643275,
1616
+ "pass@2": 0.7113501231753926,
1617
+ "pass@4": 0.7827317758525795,
1618
+ "pass@8": 0.8349723554689547,
1619
+ "pass@16": 0.8735588597859756,
1620
+ "pass@32": 0.9025529986048424,
1621
+ "pass@64": 0.9217735735634356,
1622
+ "pass@128": 0.9327485380116959
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.5301846590909091,
1626
+ "pass@2": 0.6305125566690525,
1627
+ "pass@4": 0.7065275675767798,
1628
+ "pass@8": 0.7606926843299037,
1629
+ "pass@16": 0.7990498123628644,
1630
+ "pass@32": 0.8293580860368167,
1631
+ "pass@64": 0.8559740595796936,
1632
+ "pass@128": 0.8787878787878788
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.4334045584045584,
1636
+ "pass@2": 0.5336198459968144,
1637
+ "pass@4": 0.6156393698650913,
1638
+ "pass@8": 0.6793719515351224,
1639
+ "pass@16": 0.7282840734236551,
1640
+ "pass@32": 0.7671109052665265,
1641
+ "pass@64": 0.800987680746752,
1642
+ "pass@128": 0.8319088319088319
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.3566818711180124,
1646
+ "pass@2": 0.46245017606494854,
1647
+ "pass@4": 0.5551400752700942,
1648
+ "pass@8": 0.6291890200102845,
1649
+ "pass@16": 0.6878346867297733,
1650
+ "pass@32": 0.7384604652449943,
1651
+ "pass@64": 0.7881347414072649,
1652
+ "pass@128": 0.8385093167701864
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.32173713235294116,
1656
+ "pass@2": 0.42347802802223233,
1657
+ "pass@4": 0.5183494841637442,
1658
+ "pass@8": 0.5990558759706359,
1659
+ "pass@16": 0.6651894072773802,
1660
+ "pass@32": 0.7191764944894445,
1661
+ "pass@64": 0.7606179403198643,
1662
+ "pass@128": 0.7911764705882353
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.20986793154761904,
1666
+ "pass@2": 0.2867501669713159,
1667
+ "pass@4": 0.3652424758289142,
1668
+ "pass@8": 0.4410855714382008,
1669
+ "pass@16": 0.5130674911401856,
1670
+ "pass@32": 0.5803165592539755,
1671
+ "pass@64": 0.6393250538858533,
1672
+ "pass@128": 0.6934523809523809
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.18988585329341318,
1676
+ "pass@2": 0.2721880304351926,
1677
+ "pass@4": 0.35637058588110615,
1678
+ "pass@8": 0.4371482552763727,
1679
+ "pass@16": 0.512849984475913,
1680
+ "pass@32": 0.5830933497627412,
1681
+ "pass@64": 0.645622648235715,
1682
+ "pass@128": 0.6976047904191617
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 577850,
1689
+ "answer_accuracy": 0.7212738656334878,
1690
+ "avg_response_len": 302.8390804741173,
1691
+ "resp_tokens_sum": 242620135,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.7212738656334878,
1694
+ "pass@2": 0.7713678130264008,
1695
+ "pass@4": 0.8116175308081232,
1696
+ "pass@8": 0.8437661082162949,
1697
+ "pass@16": 0.8696224555542752,
1698
+ "pass@32": 0.8909510236880488,
1699
+ "pass@64": 0.9086669802588497,
1700
+ "pass@128": 0.9234702029078128
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.8983908582089553,
1705
+ "pass@2": 0.9294809172640736,
1706
+ "pass@4": 0.950525032318721,
1707
+ "pass@8": 0.9653934920081807,
1708
+ "pass@16": 0.9748415411280041,
1709
+ "pass@32": 0.9810193030176554,
1710
+ "pass@64": 0.9861676540954208,
1711
+ "pass@128": 0.991044776119403
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9804464285714286,
1715
+ "pass@2": 0.9854861501687289,
1716
+ "pass@4": 0.9888010849536667,
1717
+ "pass@8": 0.9906410014232532,
1718
+ "pass@16": 0.9919181611918471,
1719
+ "pass@32": 0.99303540900536,
1720
+ "pass@64": 0.9939368305843209,
1721
+ "pass@128": 0.9942857142857143
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.991413083090379,
1725
+ "pass@2": 0.9955575945226233,
1726
+ "pass@4": 0.9977777331405003,
1727
+ "pass@8": 0.9991937098766038,
1728
+ "pass@16": 0.9998749765033144,
1729
+ "pass@32": 0.9999973906863244,
1730
+ "pass@64": 0.9999999999217336,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9939692982456141,
1735
+ "pass@2": 0.9942855107749691,
1736
+ "pass@4": 0.994424011143344,
1737
+ "pass@8": 0.9946902196435972,
1738
+ "pass@16": 0.9952053690657088,
1739
+ "pass@32": 0.9961665975963532,
1740
+ "pass@64": 0.9978127734033246,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9451976102941176,
1745
+ "pass@2": 0.9563129631773971,
1746
+ "pass@4": 0.9654465556695121,
1747
+ "pass@8": 0.9718827265720327,
1748
+ "pass@16": 0.9754730011095328,
1749
+ "pass@32": 0.9777955943914058,
1750
+ "pass@64": 0.9800680829193935,
1751
+ "pass@128": 0.9823529411764705
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9673205835962145,
1755
+ "pass@2": 0.9785331385528704,
1756
+ "pass@4": 0.9846225014459944,
1757
+ "pass@8": 0.9883663877814471,
1758
+ "pass@16": 0.9908695441522511,
1759
+ "pass@32": 0.9926653989314137,
1760
+ "pass@64": 0.9935901740419045,
1761
+ "pass@128": 0.9936908517350158
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9580943433544303,
1765
+ "pass@2": 0.9699429383035981,
1766
+ "pass@4": 0.9774484110174518,
1767
+ "pass@8": 0.9825297234375893,
1768
+ "pass@16": 0.9862960040047856,
1769
+ "pass@32": 0.9894226112977601,
1770
+ "pass@64": 0.9918078764397377,
1771
+ "pass@128": 0.9936708860759493
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9388138138138138,
1775
+ "pass@2": 0.9599656250738929,
1776
+ "pass@4": 0.9730372833463381,
1777
+ "pass@8": 0.982127536354344,
1778
+ "pass@16": 0.9887220242201132,
1779
+ "pass@32": 0.9932794012962612,
1780
+ "pass@64": 0.9965889690345809,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9318227491961415,
1785
+ "pass@2": 0.9592401796338966,
1786
+ "pass@4": 0.9720742425283658,
1787
+ "pass@8": 0.9775956122558883,
1788
+ "pass@16": 0.9807653370853542,
1789
+ "pass@32": 0.9840228942029388,
1790
+ "pass@64": 0.9865052259333055,
1791
+ "pass@128": 0.9871382636655949
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.8969246031746032,
1795
+ "pass@2": 0.9353073053368325,
1796
+ "pass@4": 0.9577159161652413,
1797
+ "pass@8": 0.9715580156452523,
1798
+ "pass@16": 0.9792506883459516,
1799
+ "pass@32": 0.9836697037806984,
1800
+ "pass@64": 0.9870159888279616,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8460196865443425,
1805
+ "pass@2": 0.8954093073514888,
1806
+ "pass@4": 0.921770653094968,
1807
+ "pass@8": 0.9377565497507031,
1808
+ "pass@16": 0.9494860703772144,
1809
+ "pass@32": 0.9585818773272825,
1810
+ "pass@64": 0.9648959273499919,
1811
+ "pass@128": 0.9694189602446484
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.6845823688271605,
1815
+ "pass@2": 0.7711701510401477,
1816
+ "pass@4": 0.831109089373087,
1817
+ "pass@8": 0.8697096758528202,
1818
+ "pass@16": 0.8955431218247937,
1819
+ "pass@32": 0.9153676932906923,
1820
+ "pass@64": 0.9321230211088019,
1821
+ "pass@128": 0.9444444444444444
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.6283150964391692,
1825
+ "pass@2": 0.721634121124325,
1826
+ "pass@4": 0.7920033450937324,
1827
+ "pass@8": 0.844320515659269,
1828
+ "pass@16": 0.8806439842164706,
1829
+ "pass@32": 0.9048698741234554,
1830
+ "pass@64": 0.9205488513442519,
1831
+ "pass@128": 0.9317507418397626
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.5363859953703703,
1835
+ "pass@2": 0.6341859719791972,
1836
+ "pass@4": 0.713977784605628,
1837
+ "pass@8": 0.7750546489304276,
1838
+ "pass@16": 0.8188227494235403,
1839
+ "pass@32": 0.8501646485533466,
1840
+ "pass@64": 0.8744316116951758,
1841
+ "pass@128": 0.8919753086419753
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.43372050147492625,
1845
+ "pass@2": 0.5424948609852972,
1846
+ "pass@4": 0.6338591301640388,
1847
+ "pass@8": 0.7059378256142278,
1848
+ "pass@16": 0.760447370706132,
1849
+ "pass@32": 0.8038493559978188,
1850
+ "pass@64": 0.8392871713523845,
1851
+ "pass@128": 0.8702064896755162
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.331042921686747,
1855
+ "pass@2": 0.42775160682098473,
1856
+ "pass@4": 0.5139899623804555,
1857
+ "pass@8": 0.5877757234896518,
1858
+ "pass@16": 0.6526213506834679,
1859
+ "pass@32": 0.7118324364171099,
1860
+ "pass@64": 0.7624818371103406,
1861
+ "pass@128": 0.8072289156626506
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.32104334677419355,
1865
+ "pass@2": 0.41724465011430023,
1866
+ "pass@4": 0.5019040053461057,
1867
+ "pass@8": 0.5718798748514401,
1868
+ "pass@16": 0.6303663786814787,
1869
+ "pass@32": 0.6805273166490414,
1870
+ "pass@64": 0.7261435356896053,
1871
+ "pass@128": 0.7709677419354839
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.20907288519637462,
1875
+ "pass@2": 0.2946860801198942,
1876
+ "pass@4": 0.3835897539809034,
1877
+ "pass@8": 0.4710165178235919,
1878
+ "pass@16": 0.5532048615020811,
1879
+ "pass@32": 0.623725705923306,
1880
+ "pass@64": 0.6811089924313154,
1881
+ "pass@128": 0.7250755287009063
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.19512481231231232,
1885
+ "pass@2": 0.2750282270459435,
1886
+ "pass@4": 0.35674344282415144,
1887
+ "pass@8": 0.435467028303459,
1888
+ "pass@16": 0.510642810713764,
1889
+ "pass@32": 0.5809565839476649,
1890
+ "pass@64": 0.6438056145361447,
1891
+ "pass@128": 0.6966966966966966
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch1-op11-14/SDAR-100M-block16-epoch1-op11-14_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3fa207a7e3b7bd86eaea8958003a4efd4ad64f3f3d59a4ff1330432f98d00d8
3
+ size 2676542954
results/composition/SDAR-100M-block16-epoch1-op11-14/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch1-op11-14,0.9566944444444444,0.510975,0.7221052631578947,0.0,0.0,0.0,201.07631163194444,362.007928125,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch1-op11-14/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch1-op11-14",
4
+ "id_acc": 0.9566944444444444,
5
+ "ood_acc": 0.510975,
6
+ "total_acc": 0.7221052631578947,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 201.07631163194444,
11
+ "ood_avg_resp_len": 362.007928125,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 584256,
19
+ "answer_accuracy": 0.7215460006323111,
20
+ "avg_response_len": 272.650620949257,
21
+ "resp_tokens_sum": 220772842,
22
+ "pass_at_k": {
23
+ "pass@1": 0.7215460006323111,
24
+ "pass@2": 0.7726945850271698,
25
+ "pass@4": 0.8128004638992861,
26
+ "pass@8": 0.844156706546122,
27
+ "pass@16": 0.8689312388996039,
28
+ "pass@32": 0.8893245966044871,
29
+ "pass@64": 0.9068659649411043,
30
+ "pass@128": 0.9214353461903256
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9076327433628318,
35
+ "pass@2": 0.9269853436462037,
36
+ "pass@4": 0.9398492909625238,
37
+ "pass@8": 0.9506059829868715,
38
+ "pass@16": 0.9596484055654059,
39
+ "pass@32": 0.9658798721555774,
40
+ "pass@64": 0.9688322453458257,
41
+ "pass@128": 0.9705014749262537
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9780957943925234,
45
+ "pass@2": 0.9850905757107465,
46
+ "pass@4": 0.9894686271692673,
47
+ "pass@8": 0.9920504616490534,
48
+ "pass@16": 0.9933079829828495,
49
+ "pass@32": 0.9937265331983628,
50
+ "pass@64": 0.9937693846777408,
51
+ "pass@128": 0.9937694704049844
52
+ },
53
+ "3": {
54
+ "pass@1": 0.991829128440367,
55
+ "pass@2": 0.9972839010089335,
56
+ "pass@4": 0.9991199451559382,
57
+ "pass@8": 0.9998000718183061,
58
+ "pass@16": 0.9999896701598865,
59
+ "pass@32": 0.999999988244179,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.9865234375,
65
+ "pass@2": 0.9907341904527559,
66
+ "pass@4": 0.9941022271630107,
67
+ "pass@8": 0.9962775829156062,
68
+ "pass@16": 0.997180906859181,
69
+ "pass@32": 0.9976553064004617,
70
+ "pass@64": 0.9984374999950079,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9469990079365079,
75
+ "pass@2": 0.9564476315460567,
76
+ "pass@4": 0.9630626984126983,
77
+ "pass@8": 0.9694378790828586,
78
+ "pass@16": 0.9759160456546958,
79
+ "pass@32": 0.9818998343552849,
80
+ "pass@64": 0.9875177250605657,
81
+ "pass@128": 0.9936507936507937
82
+ },
83
+ "6": {
84
+ "pass@1": 0.926529752994012,
85
+ "pass@2": 0.9463030140506391,
86
+ "pass@4": 0.9610347533279894,
87
+ "pass@8": 0.9741423767959982,
88
+ "pass@16": 0.9846537870946782,
89
+ "pass@32": 0.9919116519382953,
90
+ "pass@64": 0.996595146234816,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9607063609467456,
95
+ "pass@2": 0.9766032678330153,
96
+ "pass@4": 0.9867770337879364,
97
+ "pass@8": 0.9932539965998115,
98
+ "pass@16": 0.9966691243987713,
99
+ "pass@32": 0.997758364804311,
100
+ "pass@64": 0.9985207077144822,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9363231169871795,
105
+ "pass@2": 0.9618858677821527,
106
+ "pass@4": 0.9778153304755176,
107
+ "pass@8": 0.9872242972247056,
108
+ "pass@16": 0.9919063397806129,
109
+ "pass@32": 0.9944326739175762,
110
+ "pass@64": 0.9972112783705197,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9345290492957746,
115
+ "pass@2": 0.9633556753909281,
116
+ "pass@4": 0.977406277032272,
117
+ "pass@8": 0.9842799230966626,
118
+ "pass@16": 0.9890320300163855,
119
+ "pass@32": 0.9926185701753283,
120
+ "pass@64": 0.9956966535148944,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.8755975566572238,
125
+ "pass@2": 0.9161817436595214,
126
+ "pass@4": 0.9432045066604631,
127
+ "pass@8": 0.9603687852296585,
128
+ "pass@16": 0.9706278871438179,
129
+ "pass@32": 0.9775053297819941,
130
+ "pass@64": 0.981852619948499,
131
+ "pass@128": 0.9830028328611898
132
+ },
133
+ "12": {
134
+ "pass@1": 0.8450300141242938,
135
+ "pass@2": 0.8938623965701327,
136
+ "pass@4": 0.9229093023859299,
137
+ "pass@8": 0.9409758597723392,
138
+ "pass@16": 0.9516659432562773,
139
+ "pass@32": 0.9588886011978869,
140
+ "pass@64": 0.965495662578892,
141
+ "pass@128": 0.9717514124293786
142
+ },
143
+ "13": {
144
+ "pass@1": 0.7161295062695925,
145
+ "pass@2": 0.7977161651815466,
146
+ "pass@4": 0.8520592728612683,
147
+ "pass@8": 0.8873050572913995,
148
+ "pass@16": 0.9106471139096689,
149
+ "pass@32": 0.9292828710771102,
150
+ "pass@64": 0.9443766484455557,
151
+ "pass@128": 0.9561128526645768
152
+ },
153
+ "14": {
154
+ "pass@1": 0.6268983644859814,
155
+ "pass@2": 0.7150230272033751,
156
+ "pass@4": 0.7842801770105835,
157
+ "pass@8": 0.837397655361588,
158
+ "pass@16": 0.8762786423304784,
159
+ "pass@32": 0.9046818450843452,
160
+ "pass@64": 0.9289289086935143,
161
+ "pass@128": 0.9470404984423676
162
+ },
163
+ "15": {
164
+ "pass@1": 0.5480717124277457,
165
+ "pass@2": 0.6457133835510445,
166
+ "pass@4": 0.7187044204373292,
167
+ "pass@8": 0.7727291108200902,
168
+ "pass@16": 0.8139621475751042,
169
+ "pass@32": 0.8458336523771858,
170
+ "pass@64": 0.8698378477573592,
171
+ "pass@128": 0.8872832369942196
172
+ },
173
+ "16": {
174
+ "pass@1": 0.4441280241935484,
175
+ "pass@2": 0.5569766795783586,
176
+ "pass@4": 0.6460751194407147,
177
+ "pass@8": 0.7137075598181689,
178
+ "pass@16": 0.7681949863235685,
179
+ "pass@32": 0.8142011454810046,
180
+ "pass@64": 0.8517005897273575,
181
+ "pass@128": 0.8774193548387097
182
+ },
183
+ "17": {
184
+ "pass@1": 0.36806809971098264,
185
+ "pass@2": 0.4666509973373991,
186
+ "pass@4": 0.5472581679638306,
187
+ "pass@8": 0.6127021215261312,
188
+ "pass@16": 0.6691159679443502,
189
+ "pass@32": 0.7203100001054069,
190
+ "pass@64": 0.7654746436146028,
191
+ "pass@128": 0.8063583815028902
192
+ },
193
+ "18": {
194
+ "pass@1": 0.304375,
195
+ "pass@2": 0.3967902840269967,
196
+ "pass@4": 0.47884594809577347,
197
+ "pass@8": 0.5467893711314302,
198
+ "pass@16": 0.6037676808016317,
199
+ "pass@32": 0.6536113740422012,
200
+ "pass@64": 0.6999131573501903,
201
+ "pass@128": 0.74
202
+ },
203
+ "19": {
204
+ "pass@1": 0.21579391891891891,
205
+ "pass@2": 0.3006612665815421,
206
+ "pass@4": 0.38616035720760156,
207
+ "pass@8": 0.46550902965231605,
208
+ "pass@16": 0.5350209172548729,
209
+ "pass@32": 0.595347618281271,
210
+ "pass@64": 0.6482623365836448,
211
+ "pass@128": 0.6936936936936937
212
+ },
213
+ "20": {
214
+ "pass@1": 0.22670326576576577,
215
+ "pass@2": 0.3139590131470051,
216
+ "pass@4": 0.39960391071611545,
217
+ "pass@8": 0.4769628415256585,
218
+ "pass@16": 0.5428805161447351,
219
+ "pass@32": 0.6007913078306004,
220
+ "pass@64": 0.6557323133549934,
221
+ "pass@128": 0.7027027027027027
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 594054,
228
+ "answer_accuracy": 0.7234679462197974,
229
+ "avg_response_len": 282.07458349571317,
230
+ "resp_tokens_sum": 231617082,
231
+ "pass_at_k": {
232
+ "pass@1": 0.7234679462197974,
233
+ "pass@2": 0.7721883227671293,
234
+ "pass@4": 0.8113025592962161,
235
+ "pass@8": 0.8425025384347558,
236
+ "pass@16": 0.8674617623092497,
237
+ "pass@32": 0.8882591568285177,
238
+ "pass@64": 0.9060958084492297,
239
+ "pass@128": 0.9214341387373344
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.8978144171779141,
244
+ "pass@2": 0.9213515500458915,
245
+ "pass@4": 0.9390322137646904,
246
+ "pass@8": 0.9526189121876671,
247
+ "pass@16": 0.9621797529616819,
248
+ "pass@32": 0.9684335022171842,
249
+ "pass@64": 0.9742357566082435,
250
+ "pass@128": 0.9815950920245399
251
+ },
252
+ "2": {
253
+ "pass@1": 0.991950037993921,
254
+ "pass@2": 0.9956385521623626,
255
+ "pass@4": 0.996789173769692,
256
+ "pass@8": 0.9969556274713829,
257
+ "pass@16": 0.996960482749072,
258
+ "pass@32": 0.9969604863221885,
259
+ "pass@64": 0.9969604863221885,
260
+ "pass@128": 0.9969604863221885
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9910748106060606,
264
+ "pass@2": 0.9938342579336673,
265
+ "pass@4": 0.9954610878185681,
266
+ "pass@8": 0.9967861359234476,
267
+ "pass@16": 0.9983705508051556,
268
+ "pass@32": 0.999643213120596,
269
+ "pass@64": 0.99999359856941,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9862934541420119,
274
+ "pass@2": 0.9914085431906072,
275
+ "pass@4": 0.9944976862174774,
276
+ "pass@8": 0.9968238286910404,
277
+ "pass@16": 0.9986471171735518,
278
+ "pass@32": 0.9997422779717037,
279
+ "pass@64": 0.9999955420471618,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9700860507246377,
284
+ "pass@2": 0.9768297814675341,
285
+ "pass@4": 0.9813448272770252,
286
+ "pass@8": 0.9845774240321683,
287
+ "pass@16": 0.9867515567319303,
288
+ "pass@32": 0.9884426560783856,
289
+ "pass@64": 0.9902316406247298,
290
+ "pass@128": 0.991304347826087
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9678545845272206,
294
+ "pass@2": 0.9800999199061439,
295
+ "pass@4": 0.9855557597742974,
296
+ "pass@8": 0.988306274970953,
297
+ "pass@16": 0.9904143515720015,
298
+ "pass@32": 0.9923533345945439,
299
+ "pass@64": 0.9938368819680918,
300
+ "pass@128": 0.994269340974212
301
+ },
302
+ "7": {
303
+ "pass@1": 0.965904985549133,
304
+ "pass@2": 0.9778386850848846,
305
+ "pass@4": 0.9852525098018816,
306
+ "pass@8": 0.9905732375817846,
307
+ "pass@16": 0.9943721417100441,
308
+ "pass@32": 0.997168595303731,
309
+ "pass@64": 0.9991059501069279,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9451804577464789,
314
+ "pass@2": 0.9669578989686152,
315
+ "pass@4": 0.9803207029579047,
316
+ "pass@8": 0.9881750952049374,
317
+ "pass@16": 0.9925613357660423,
318
+ "pass@32": 0.9958566064759846,
319
+ "pass@64": 0.9987507974877009,
320
+ "pass@128": 1.0
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9378508607784432,
324
+ "pass@2": 0.960500259323872,
325
+ "pass@4": 0.9725681423991157,
326
+ "pass@8": 0.9806556822439841,
327
+ "pass@16": 0.9856425911302683,
328
+ "pass@32": 0.988864619985313,
329
+ "pass@64": 0.9917644684832534,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9046733810240963,
334
+ "pass@2": 0.9394221818375864,
335
+ "pass@4": 0.9586932646220422,
336
+ "pass@8": 0.9705318114541525,
337
+ "pass@16": 0.9775325668360785,
338
+ "pass@32": 0.9814679960471665,
339
+ "pass@64": 0.9848638566163492,
340
+ "pass@128": 0.9879518072289156
341
+ },
342
+ "12": {
343
+ "pass@1": 0.8320190047021944,
344
+ "pass@2": 0.8726735862562636,
345
+ "pass@4": 0.8965225433223666,
346
+ "pass@8": 0.9119176475126256,
347
+ "pass@16": 0.9216786817337147,
348
+ "pass@32": 0.9290268632604375,
349
+ "pass@64": 0.9362206325610609,
350
+ "pass@128": 0.9435736677115988
351
+ },
352
+ "13": {
353
+ "pass@1": 0.6824010854341737,
354
+ "pass@2": 0.7620663915172362,
355
+ "pass@4": 0.816710596574587,
356
+ "pass@8": 0.8568656338449239,
357
+ "pass@16": 0.8863689471035519,
358
+ "pass@32": 0.9093439500882927,
359
+ "pass@64": 0.929124181128099,
360
+ "pass@128": 0.9467787114845938
361
+ },
362
+ "14": {
363
+ "pass@1": 0.6183982090643275,
364
+ "pass@2": 0.7113501231753926,
365
+ "pass@4": 0.7827317758525795,
366
+ "pass@8": 0.8349723554689547,
367
+ "pass@16": 0.8735588597859756,
368
+ "pass@32": 0.9025529986048424,
369
+ "pass@64": 0.9217735735634356,
370
+ "pass@128": 0.9327485380116959
371
+ },
372
+ "15": {
373
+ "pass@1": 0.5301846590909091,
374
+ "pass@2": 0.6305125566690525,
375
+ "pass@4": 0.7065275675767798,
376
+ "pass@8": 0.7606926843299037,
377
+ "pass@16": 0.7990498123628644,
378
+ "pass@32": 0.8293580860368167,
379
+ "pass@64": 0.8559740595796936,
380
+ "pass@128": 0.8787878787878788
381
+ },
382
+ "16": {
383
+ "pass@1": 0.4334045584045584,
384
+ "pass@2": 0.5336198459968144,
385
+ "pass@4": 0.6156393698650913,
386
+ "pass@8": 0.6793719515351224,
387
+ "pass@16": 0.7282840734236551,
388
+ "pass@32": 0.7671109052665265,
389
+ "pass@64": 0.800987680746752,
390
+ "pass@128": 0.8319088319088319
391
+ },
392
+ "17": {
393
+ "pass@1": 0.3566818711180124,
394
+ "pass@2": 0.46245017606494854,
395
+ "pass@4": 0.5551400752700942,
396
+ "pass@8": 0.6291890200102845,
397
+ "pass@16": 0.6878346867297733,
398
+ "pass@32": 0.7384604652449943,
399
+ "pass@64": 0.7881347414072649,
400
+ "pass@128": 0.8385093167701864
401
+ },
402
+ "18": {
403
+ "pass@1": 0.32173713235294116,
404
+ "pass@2": 0.42347802802223233,
405
+ "pass@4": 0.5183494841637442,
406
+ "pass@8": 0.5990558759706359,
407
+ "pass@16": 0.6651894072773802,
408
+ "pass@32": 0.7191764944894445,
409
+ "pass@64": 0.7606179403198643,
410
+ "pass@128": 0.7911764705882353
411
+ },
412
+ "19": {
413
+ "pass@1": 0.20986793154761904,
414
+ "pass@2": 0.2867501669713159,
415
+ "pass@4": 0.3652424758289142,
416
+ "pass@8": 0.4410855714382008,
417
+ "pass@16": 0.5130674911401856,
418
+ "pass@32": 0.5803165592539755,
419
+ "pass@64": 0.6393250538858533,
420
+ "pass@128": 0.6934523809523809
421
+ },
422
+ "20": {
423
+ "pass@1": 0.18988585329341318,
424
+ "pass@2": 0.2721880304351926,
425
+ "pass@4": 0.35637058588110615,
426
+ "pass@8": 0.4371482552763727,
427
+ "pass@16": 0.512849984475913,
428
+ "pass@32": 0.5830933497627412,
429
+ "pass@64": 0.645622648235715,
430
+ "pass@128": 0.6976047904191617
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 577850,
437
+ "answer_accuracy": 0.7212738656334878,
438
+ "avg_response_len": 302.8390804741173,
439
+ "resp_tokens_sum": 242620135,
440
+ "pass_at_k": {
441
+ "pass@1": 0.7212738656334878,
442
+ "pass@2": 0.7713678130264008,
443
+ "pass@4": 0.8116175308081232,
444
+ "pass@8": 0.8437661082162949,
445
+ "pass@16": 0.8696224555542752,
446
+ "pass@32": 0.8909510236880488,
447
+ "pass@64": 0.9086669802588497,
448
+ "pass@128": 0.9234702029078128
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.8983908582089553,
453
+ "pass@2": 0.9294809172640736,
454
+ "pass@4": 0.950525032318721,
455
+ "pass@8": 0.9653934920081807,
456
+ "pass@16": 0.9748415411280041,
457
+ "pass@32": 0.9810193030176554,
458
+ "pass@64": 0.9861676540954208,
459
+ "pass@128": 0.991044776119403
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9804464285714286,
463
+ "pass@2": 0.9854861501687289,
464
+ "pass@4": 0.9888010849536667,
465
+ "pass@8": 0.9906410014232532,
466
+ "pass@16": 0.9919181611918471,
467
+ "pass@32": 0.99303540900536,
468
+ "pass@64": 0.9939368305843209,
469
+ "pass@128": 0.9942857142857143
470
+ },
471
+ "3": {
472
+ "pass@1": 0.991413083090379,
473
+ "pass@2": 0.9955575945226233,
474
+ "pass@4": 0.9977777331405003,
475
+ "pass@8": 0.9991937098766038,
476
+ "pass@16": 0.9998749765033144,
477
+ "pass@32": 0.9999973906863244,
478
+ "pass@64": 0.9999999999217336,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9939692982456141,
483
+ "pass@2": 0.9942855107749691,
484
+ "pass@4": 0.994424011143344,
485
+ "pass@8": 0.9946902196435972,
486
+ "pass@16": 0.9952053690657088,
487
+ "pass@32": 0.9961665975963532,
488
+ "pass@64": 0.9978127734033246,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9451976102941176,
493
+ "pass@2": 0.9563129631773971,
494
+ "pass@4": 0.9654465556695121,
495
+ "pass@8": 0.9718827265720327,
496
+ "pass@16": 0.9754730011095328,
497
+ "pass@32": 0.9777955943914058,
498
+ "pass@64": 0.9800680829193935,
499
+ "pass@128": 0.9823529411764705
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9673205835962145,
503
+ "pass@2": 0.9785331385528704,
504
+ "pass@4": 0.9846225014459944,
505
+ "pass@8": 0.9883663877814471,
506
+ "pass@16": 0.9908695441522511,
507
+ "pass@32": 0.9926653989314137,
508
+ "pass@64": 0.9935901740419045,
509
+ "pass@128": 0.9936908517350158
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9580943433544303,
513
+ "pass@2": 0.9699429383035981,
514
+ "pass@4": 0.9774484110174518,
515
+ "pass@8": 0.9825297234375893,
516
+ "pass@16": 0.9862960040047856,
517
+ "pass@32": 0.9894226112977601,
518
+ "pass@64": 0.9918078764397377,
519
+ "pass@128": 0.9936708860759493
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9388138138138138,
523
+ "pass@2": 0.9599656250738929,
524
+ "pass@4": 0.9730372833463381,
525
+ "pass@8": 0.982127536354344,
526
+ "pass@16": 0.9887220242201132,
527
+ "pass@32": 0.9932794012962612,
528
+ "pass@64": 0.9965889690345809,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9318227491961415,
533
+ "pass@2": 0.9592401796338966,
534
+ "pass@4": 0.9720742425283658,
535
+ "pass@8": 0.9775956122558883,
536
+ "pass@16": 0.9807653370853542,
537
+ "pass@32": 0.9840228942029388,
538
+ "pass@64": 0.9865052259333055,
539
+ "pass@128": 0.9871382636655949
540
+ },
541
+ "11": {
542
+ "pass@1": 0.8969246031746032,
543
+ "pass@2": 0.9353073053368325,
544
+ "pass@4": 0.9577159161652413,
545
+ "pass@8": 0.9715580156452523,
546
+ "pass@16": 0.9792506883459516,
547
+ "pass@32": 0.9836697037806984,
548
+ "pass@64": 0.9870159888279616,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.8460196865443425,
553
+ "pass@2": 0.8954093073514888,
554
+ "pass@4": 0.921770653094968,
555
+ "pass@8": 0.9377565497507031,
556
+ "pass@16": 0.9494860703772144,
557
+ "pass@32": 0.9585818773272825,
558
+ "pass@64": 0.9648959273499919,
559
+ "pass@128": 0.9694189602446484
560
+ },
561
+ "13": {
562
+ "pass@1": 0.6845823688271605,
563
+ "pass@2": 0.7711701510401477,
564
+ "pass@4": 0.831109089373087,
565
+ "pass@8": 0.8697096758528202,
566
+ "pass@16": 0.8955431218247937,
567
+ "pass@32": 0.9153676932906923,
568
+ "pass@64": 0.9321230211088019,
569
+ "pass@128": 0.9444444444444444
570
+ },
571
+ "14": {
572
+ "pass@1": 0.6283150964391692,
573
+ "pass@2": 0.721634121124325,
574
+ "pass@4": 0.7920033450937324,
575
+ "pass@8": 0.844320515659269,
576
+ "pass@16": 0.8806439842164706,
577
+ "pass@32": 0.9048698741234554,
578
+ "pass@64": 0.9205488513442519,
579
+ "pass@128": 0.9317507418397626
580
+ },
581
+ "15": {
582
+ "pass@1": 0.5363859953703703,
583
+ "pass@2": 0.6341859719791972,
584
+ "pass@4": 0.713977784605628,
585
+ "pass@8": 0.7750546489304276,
586
+ "pass@16": 0.8188227494235403,
587
+ "pass@32": 0.8501646485533466,
588
+ "pass@64": 0.8744316116951758,
589
+ "pass@128": 0.8919753086419753
590
+ },
591
+ "16": {
592
+ "pass@1": 0.43372050147492625,
593
+ "pass@2": 0.5424948609852972,
594
+ "pass@4": 0.6338591301640388,
595
+ "pass@8": 0.7059378256142278,
596
+ "pass@16": 0.760447370706132,
597
+ "pass@32": 0.8038493559978188,
598
+ "pass@64": 0.8392871713523845,
599
+ "pass@128": 0.8702064896755162
600
+ },
601
+ "17": {
602
+ "pass@1": 0.331042921686747,
603
+ "pass@2": 0.42775160682098473,
604
+ "pass@4": 0.5139899623804555,
605
+ "pass@8": 0.5877757234896518,
606
+ "pass@16": 0.6526213506834679,
607
+ "pass@32": 0.7118324364171099,
608
+ "pass@64": 0.7624818371103406,
609
+ "pass@128": 0.8072289156626506
610
+ },
611
+ "18": {
612
+ "pass@1": 0.32104334677419355,
613
+ "pass@2": 0.41724465011430023,
614
+ "pass@4": 0.5019040053461057,
615
+ "pass@8": 0.5718798748514401,
616
+ "pass@16": 0.6303663786814787,
617
+ "pass@32": 0.6805273166490414,
618
+ "pass@64": 0.7261435356896053,
619
+ "pass@128": 0.7709677419354839
620
+ },
621
+ "19": {
622
+ "pass@1": 0.20907288519637462,
623
+ "pass@2": 0.2946860801198942,
624
+ "pass@4": 0.3835897539809034,
625
+ "pass@8": 0.4710165178235919,
626
+ "pass@16": 0.5532048615020811,
627
+ "pass@32": 0.623725705923306,
628
+ "pass@64": 0.6811089924313154,
629
+ "pass@128": 0.7250755287009063
630
+ },
631
+ "20": {
632
+ "pass@1": 0.19512481231231232,
633
+ "pass@2": 0.2750282270459435,
634
+ "pass@4": 0.35674344282415144,
635
+ "pass@8": 0.435467028303459,
636
+ "pass@16": 0.510642810713764,
637
+ "pass@32": 0.5809565839476649,
638
+ "pass@64": 0.6438056145361447,
639
+ "pass@128": 0.6966966966966966
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2fa6ee4b9228a31951910b169eae81bfbda21ef687b8869e2da069a264d0bef
3
+ size 1796825349
results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch1-op17-20",
3
+ "id": {
4
+ "answer_accuracy": 0.9134383680555556,
5
+ "avg_response_len": 202.60357899305555,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9134383680555556,
11
+ "pass@2": 0.9481420466972658,
12
+ "pass@4": 0.966789149585467,
13
+ "pass@8": 0.9779535209475929,
14
+ "pass@16": 0.9851560966687626,
15
+ "pass@32": 0.989967281322493,
16
+ "pass@64": 0.9933028645399439,
17
+ "pass@128": 0.9957777777777778
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.8259609375,
22
+ "pass@2": 0.893026451771654,
23
+ "pass@4": 0.9322231326396698,
24
+ "pass@8": 0.9565009551603775,
25
+ "pass@16": 0.971507385154414,
26
+ "pass@32": 0.9808318511743037,
27
+ "pass@64": 0.9876695059302211,
28
+ "pass@128": 0.993
29
+ },
30
+ "2": {
31
+ "pass@1": 0.935625,
32
+ "pass@2": 0.9512100147637794,
33
+ "pass@4": 0.9636213038995125,
34
+ "pass@8": 0.9727229765191304,
35
+ "pass@16": 0.978889970330367,
36
+ "pass@32": 0.982836458697959,
37
+ "pass@64": 0.9851793710344321,
38
+ "pass@128": 0.987
39
+ },
40
+ "3": {
41
+ "pass@1": 0.977046875,
42
+ "pass@2": 0.9854843750000004,
43
+ "pass@4": 0.9901338926696664,
44
+ "pass@8": 0.9932631652553967,
45
+ "pass@16": 0.9954292364313113,
46
+ "pass@32": 0.9967808357169232,
47
+ "pass@64": 0.9977499040291772,
48
+ "pass@128": 0.999
49
+ },
50
+ "4": {
51
+ "pass@1": 0.97296875,
52
+ "pass@2": 0.9835835383858268,
53
+ "pass@4": 0.9883603146794149,
54
+ "pass@8": 0.9911434994665775,
55
+ "pass@16": 0.9930824160789711,
56
+ "pass@32": 0.9947548746152566,
57
+ "pass@64": 0.9964802260461338,
58
+ "pass@128": 0.998
59
+ },
60
+ "5": {
61
+ "pass@1": 0.9220625,
62
+ "pass@2": 0.9477059547244095,
63
+ "pass@4": 0.963529338582677,
64
+ "pass@8": 0.9743781813113109,
65
+ "pass@16": 0.9822554465747628,
66
+ "pass@32": 0.9885125209347927,
67
+ "pass@64": 0.9929442929712965,
68
+ "pass@128": 0.995
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9371796875,
72
+ "pass@2": 0.9675015994094499,
73
+ "pass@4": 0.9815036453880766,
74
+ "pass@8": 0.9885290906061706,
75
+ "pass@16": 0.9921437299044742,
76
+ "pass@32": 0.99416612649408,
77
+ "pass@64": 0.995942199982249,
78
+ "pass@128": 0.998
79
+ },
80
+ "7": {
81
+ "pass@1": 0.908703125,
82
+ "pass@2": 0.9471446850393703,
83
+ "pass@4": 0.967179319928759,
84
+ "pass@8": 0.9779200100902848,
85
+ "pass@16": 0.9847850762647822,
86
+ "pass@32": 0.9895726876030048,
87
+ "pass@64": 0.9928923860328146,
88
+ "pass@128": 0.996
89
+ },
90
+ "8": {
91
+ "pass@1": 0.8838984375,
92
+ "pass@2": 0.9365091043307101,
93
+ "pass@4": 0.9626962175665545,
94
+ "pass@8": 0.9774632890377969,
95
+ "pass@16": 0.9864115667605743,
96
+ "pass@32": 0.9920013654389653,
97
+ "pass@64": 0.9956127630881643,
98
+ "pass@128": 0.998
99
+ },
100
+ "9": {
101
+ "pass@1": 0.8575,
102
+ "pass@2": 0.9211126968503948,
103
+ "pass@4": 0.9518551809148847,
104
+ "pass@8": 0.9696605210813218,
105
+ "pass@16": 0.9819000425192099,
106
+ "pass@32": 0.9902488112271479,
107
+ "pass@64": 0.9952551317449962,
108
+ "pass@128": 0.998
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.8259609375,
113
+ "2": 0.935625,
114
+ "3": 0.977046875,
115
+ "4": 0.97296875,
116
+ "5": 0.9220625,
117
+ "6": 0.9371796875,
118
+ "7": 0.908703125,
119
+ "8": 0.8838984375,
120
+ "9": 0.8575
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 281.24259375,
124
+ "2": 122.4647578125,
125
+ "3": 146.4731015625,
126
+ "4": 160.6047265625,
127
+ "5": 176.30828125,
128
+ "6": 201.5447421875,
129
+ "7": 226.3348359375,
130
+ "8": 246.195390625,
131
+ "9": 262.26378125
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 281.24259375,
136
+ "2": 122.4647578125,
137
+ "3": 146.4731015625,
138
+ "4": 160.6047265625,
139
+ "5": 176.30828125,
140
+ "6": 201.5447421875,
141
+ "7": 226.3348359375,
142
+ "8": 246.195390625,
143
+ "9": 262.26378125
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 344132,
150
+ "answer_accuracy": 0.9079808341776426,
151
+ "avg_response_len": 192.52982786643025,
152
+ "resp_tokens_sum": 72970345,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9079808341776426,
155
+ "pass@2": 0.945897512478498,
156
+ "pass@4": 0.966108559188462,
157
+ "pass@8": 0.9778946033419159,
158
+ "pass@16": 0.9851827312761958,
159
+ "pass@32": 0.9899476901168374,
160
+ "pass@64": 0.9931722876009214,
161
+ "pass@128": 0.9956095913542722
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.8337020648967551,
166
+ "pass@2": 0.8992248652823267,
167
+ "pass@4": 0.9340549527547986,
168
+ "pass@8": 0.9549133337554659,
169
+ "pass@16": 0.9676459667453204,
170
+ "pass@32": 0.9754629221119615,
171
+ "pass@64": 0.9816283664853948,
172
+ "pass@128": 0.9882005899705014
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9328514408099688,
176
+ "pass@2": 0.9459657934113376,
177
+ "pass@4": 0.9572665329801058,
178
+ "pass@8": 0.9661779608434554,
179
+ "pass@16": 0.9726776857540342,
180
+ "pass@32": 0.9771842431316051,
181
+ "pass@64": 0.9796170198954688,
182
+ "pass@128": 0.9813084112149533
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9756068425076453,
186
+ "pass@2": 0.9874613222085766,
187
+ "pass@4": 0.9940523552675177,
188
+ "pass@8": 0.9974856871737064,
189
+ "pass@16": 0.9991656932084267,
190
+ "pass@32": 0.9998836449467822,
191
+ "pass@64": 0.9999990650219919,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.975341796875,
196
+ "pass@2": 0.9855330339566928,
197
+ "pass@4": 0.9891930282152229,
198
+ "pass@8": 0.9909968147034551,
199
+ "pass@16": 0.9920335522380169,
200
+ "pass@32": 0.9932232658217899,
201
+ "pass@64": 0.9949311023622034,
202
+ "pass@128": 0.996875
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9057291666666667,
206
+ "pass@2": 0.9370816929133859,
207
+ "pass@4": 0.9557188610352269,
208
+ "pass@8": 0.9680085338015769,
209
+ "pass@16": 0.9772068251757668,
210
+ "pass@32": 0.9850956818037647,
211
+ "pass@64": 0.9907377001622719,
212
+ "pass@128": 0.9936507936507937
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9124719311377245,
216
+ "pass@2": 0.9472194881889765,
217
+ "pass@4": 0.967283268951411,
218
+ "pass@8": 0.9804620871830513,
219
+ "pass@16": 0.9883802999024801,
220
+ "pass@32": 0.9926513268166495,
221
+ "pass@64": 0.9960828629239575,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.8930981878698225,
226
+ "pass@2": 0.9425206314354944,
227
+ "pass@4": 0.9694562882857096,
228
+ "pass@8": 0.9828592027889445,
229
+ "pass@16": 0.9902744534062847,
230
+ "pass@32": 0.9950889757930396,
231
+ "pass@64": 0.9980455562464515,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.8852413862179487,
236
+ "pass@2": 0.9402193115283668,
237
+ "pass@4": 0.9669937225394901,
238
+ "pass@8": 0.9816570277298262,
239
+ "pass@16": 0.990420934947623,
240
+ "pass@32": 0.996331498679127,
241
+ "pass@64": 0.9995570396284617,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.8653389084507043,
246
+ "pass@2": 0.9311758345347674,
247
+ "pass@4": 0.9623128128350147,
248
+ "pass@8": 0.9789012564778983,
249
+ "pass@16": 0.9887982028366319,
250
+ "pass@32": 0.994471708460012,
251
+ "pass@64": 0.9977497769018057,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 358896,
259
+ "answer_accuracy": 0.9187008519003932,
260
+ "avg_response_len": 200.21991214777196,
261
+ "resp_tokens_sum": 78217110,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9187008519003932,
264
+ "pass@2": 0.9517420243986168,
265
+ "pass@4": 0.9687024149831882,
266
+ "pass@8": 0.9789000894335694,
267
+ "pass@16": 0.9858622215313799,
268
+ "pass@32": 0.9907073713245071,
269
+ "pass@64": 0.9940006009808476,
270
+ "pass@128": 0.9960681520314548
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.8397478911042945,
275
+ "pass@2": 0.9017401031351142,
276
+ "pass@4": 0.9367013483935669,
277
+ "pass@8": 0.9597981710002264,
278
+ "pass@16": 0.9760430566524732,
279
+ "pass@32": 0.9868893358013864,
280
+ "pass@64": 0.9939010380542732,
281
+ "pass@128": 0.9969325153374233
282
+ },
283
+ "2": {
284
+ "pass@1": 0.9402308130699089,
285
+ "pass@2": 0.9559446275997415,
286
+ "pass@4": 0.9671450020115263,
287
+ "pass@8": 0.9747791046336394,
288
+ "pass@16": 0.9798526018916596,
289
+ "pass@32": 0.982735766252235,
290
+ "pass@64": 0.9843437813422433,
291
+ "pass@128": 0.9848024316109423
292
+ },
293
+ "3": {
294
+ "pass@1": 0.98203125,
295
+ "pass@2": 0.9875126759723218,
296
+ "pass@4": 0.9907982465260023,
297
+ "pass@8": 0.9931905013110562,
298
+ "pass@16": 0.9947669019933483,
299
+ "pass@32": 0.996016072122554,
300
+ "pass@64": 0.9977332350126279,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9693047337278107,
305
+ "pass@2": 0.9827249947584216,
306
+ "pass@4": 0.9890199579342521,
307
+ "pass@8": 0.9924316167169746,
308
+ "pass@16": 0.9952105268225401,
309
+ "pass@32": 0.9977446704990522,
310
+ "pass@64": 0.9995571159589419,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.948052536231884,
315
+ "pass@2": 0.966407052379322,
316
+ "pass@4": 0.9765803837020368,
317
+ "pass@8": 0.9844381946054411,
318
+ "pass@16": 0.9908400253696799,
319
+ "pass@32": 0.9954288349662425,
320
+ "pass@64": 0.9982494068365969,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9469914040114613,
325
+ "pass@2": 0.9771242498251473,
326
+ "pass@4": 0.9882326420336424,
327
+ "pass@8": 0.991931583238474,
328
+ "pass@16": 0.9933659205077896,
329
+ "pass@32": 0.9940735913681014,
330
+ "pass@64": 0.9942652013230373,
331
+ "pass@128": 0.994269340974212
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9118722904624278,
335
+ "pass@2": 0.9468681728187156,
336
+ "pass@4": 0.9655806095047366,
337
+ "pass@8": 0.9770071107892204,
338
+ "pass@16": 0.9847477489910627,
339
+ "pass@32": 0.9900681636945455,
340
+ "pass@64": 0.9936873526856657,
341
+ "pass@128": 0.9971098265895953
342
+ },
343
+ "8": {
344
+ "pass@1": 0.8875,
345
+ "pass@2": 0.9402195159143837,
346
+ "pass@4": 0.9639932780409486,
347
+ "pass@8": 0.9756221536661233,
348
+ "pass@16": 0.9821909911786181,
349
+ "pass@32": 0.987287230265571,
350
+ "pass@64": 0.992438504371287,
351
+ "pass@128": 0.9971830985915493
352
+ },
353
+ "9": {
354
+ "pass@1": 0.8411302395209581,
355
+ "pass@2": 0.905336104366071,
356
+ "pass@4": 0.938773440464403,
357
+ "pass@8": 0.9598987045631255,
358
+ "pass@16": 0.9751816882329573,
359
+ "pass@32": 0.9858225193162448,
360
+ "pass@64": 0.9916186385525599,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 349253,
368
+ "answer_accuracy": 0.9134713968865082,
369
+ "avg_response_len": 215.02518203883494,
370
+ "resp_tokens_sum": 82211868,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9134713968865082,
373
+ "pass@2": 0.9466887269374664,
374
+ "pass@4": 0.9655089159636779,
375
+ "pass@8": 0.9770447589828205,
376
+ "pass@16": 0.9844082030787714,
377
+ "pass@32": 0.9892305069247003,
378
+ "pass@64": 0.9927193850283234,
379
+ "pass@128": 0.995647807164379
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.8047108208955224,
384
+ "pass@2": 0.8782744740862615,
385
+ "pass@4": 0.9260115345656411,
386
+ "pass@8": 0.9548988993170184,
387
+ "pass@16": 0.9710010924150604,
388
+ "pass@32": 0.980370140618196,
389
+ "pass@64": 0.9877186605551607,
390
+ "pass@128": 0.9940298507462687
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9338392857142858,
394
+ "pass@2": 0.9515691788526434,
395
+ "pass@4": 0.9661372604317318,
396
+ "pass@8": 0.9767929304683264,
397
+ "pass@16": 0.9836826490884742,
398
+ "pass@32": 0.9881149987306532,
399
+ "pass@64": 0.9910662959611092,
400
+ "pass@128": 0.9942857142857143
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9736242711370262,
404
+ "pass@2": 0.9816482203117467,
405
+ "pass@4": 0.9857590412350061,
406
+ "pass@8": 0.9893075222068398,
407
+ "pass@16": 0.9925043063975238,
408
+ "pass@32": 0.9945585423291038,
409
+ "pass@64": 0.9956216974134651,
410
+ "pass@128": 0.9970845481049563
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9743695175438597,
414
+ "pass@2": 0.9826079511442651,
415
+ "pass@4": 0.9869292393823577,
416
+ "pass@8": 0.9900076968161828,
417
+ "pass@16": 0.9919605885870972,
418
+ "pass@32": 0.9932331313555678,
419
+ "pass@64": 0.9948887956026504,
420
+ "pass@128": 0.9970760233918129
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9108226102941176,
424
+ "pass@2": 0.9385729070171378,
425
+ "pass@4": 0.9575225440569924,
426
+ "pass@8": 0.9700715177204019,
427
+ "pass@16": 0.9782220232113423,
428
+ "pass@32": 0.9846600973625091,
429
+ "pass@64": 0.9896055060633964,
430
+ "pass@128": 0.9911764705882353
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9524102917981072,
434
+ "pass@2": 0.9782773727613706,
435
+ "pass@4": 0.9890783643172356,
436
+ "pass@8": 0.9932827474347146,
437
+ "pass@16": 0.994763417917439,
438
+ "pass@32": 0.9958640376966922,
439
+ "pass@64": 0.9976402791921362,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9219244462025317,
444
+ "pass@2": 0.9523934298066383,
445
+ "pass@4": 0.9664943151884493,
446
+ "pass@8": 0.973636516501745,
447
+ "pass@16": 0.9789543982992107,
448
+ "pass@32": 0.983129839071662,
449
+ "pass@64": 0.9865100126337769,
450
+ "pass@128": 0.990506329113924
451
+ },
452
+ "8": {
453
+ "pass@1": 0.8788006756756757,
454
+ "pass@2": 0.9290773302830391,
455
+ "pass@4": 0.9572869742633514,
456
+ "pass@8": 0.9754967922961485,
457
+ "pass@16": 0.9871544540195394,
458
+ "pass@32": 0.992969883203604,
459
+ "pass@64": 0.9953011942110427,
460
+ "pass@128": 0.996996996996997
461
+ },
462
+ "9": {
463
+ "pass@1": 0.8661324356913184,
464
+ "pass@2": 0.9265691856343519,
465
+ "pass@4": 0.9539672130011069,
466
+ "pass@8": 0.9695961662623275,
467
+ "pass@16": 0.9812411467601234,
468
+ "pass@32": 0.9901821005537559,
469
+ "pass@64": 0.9963129764254025,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.381703125,
478
+ "avg_response_len": 351.9533,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.381703125,
484
+ "pass@2": 0.4851356299212578,
485
+ "pass@4": 0.57571848824521,
486
+ "pass@8": 0.6504443915725758,
487
+ "pass@16": 0.7106175845483492,
488
+ "pass@32": 0.7599688087801404,
489
+ "pass@64": 0.8018412728035076,
490
+ "pass@128": 0.8372
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.7578203125,
495
+ "pass@2": 0.847205954724411,
496
+ "pass@4": 0.9011952808398942,
497
+ "pass@8": 0.93377580565178,
498
+ "pass@16": 0.955070876530586,
499
+ "pass@32": 0.969590082180953,
500
+ "pass@64": 0.9787324423322356,
501
+ "pass@128": 0.983
502
+ },
503
+ "12": {
504
+ "pass@1": 0.641609375,
505
+ "pass@2": 0.7585755413385825,
506
+ "pass@4": 0.8381425514623176,
507
+ "pass@8": 0.8892629624537244,
508
+ "pass@16": 0.9220148867551953,
509
+ "pass@32": 0.9435080487950828,
510
+ "pass@64": 0.9590211028900316,
511
+ "pass@128": 0.972
512
+ },
513
+ "13": {
514
+ "pass@1": 0.465140625,
515
+ "pass@2": 0.5931075295275604,
516
+ "pass@4": 0.6977997386576684,
517
+ "pass@8": 0.7762012373108262,
518
+ "pass@16": 0.8331259756536721,
519
+ "pass@32": 0.8759206686558857,
520
+ "pass@64": 0.9090539128333244,
521
+ "pass@128": 0.932
522
+ },
523
+ "14": {
524
+ "pass@1": 0.43965625,
525
+ "pass@2": 0.5558090551181101,
526
+ "pass@4": 0.657769242407198,
527
+ "pass@8": 0.7426246540339614,
528
+ "pass@16": 0.8089162040581582,
529
+ "pass@32": 0.8580974144957311,
530
+ "pass@64": 0.8936584566878355,
531
+ "pass@128": 0.919
532
+ },
533
+ "15": {
534
+ "pass@1": 0.3664609375,
535
+ "pass@2": 0.49034571850393777,
536
+ "pass@4": 0.598005425571804,
537
+ "pass@8": 0.6826972018891915,
538
+ "pass@16": 0.7454130715416146,
539
+ "pass@32": 0.7926700168179653,
540
+ "pass@64": 0.8309058005394662,
541
+ "pass@128": 0.862
542
+ },
543
+ "16": {
544
+ "pass@1": 0.281734375,
545
+ "pass@2": 0.3871161417322831,
546
+ "pass@4": 0.4866833546119234,
547
+ "pass@8": 0.5734005358977272,
548
+ "pass@16": 0.6484476727811473,
549
+ "pass@32": 0.7138604538727543,
550
+ "pass@64": 0.7698033716622331,
551
+ "pass@128": 0.817
552
+ },
553
+ "17": {
554
+ "pass@1": 0.2525,
555
+ "pass@2": 0.35274815452755964,
556
+ "pass@4": 0.4507468760779903,
557
+ "pass@8": 0.5374077389510419,
558
+ "pass@16": 0.6103234449582415,
559
+ "pass@32": 0.6721658021677973,
560
+ "pass@64": 0.7278754450284157,
561
+ "pass@128": 0.781
562
+ },
563
+ "18": {
564
+ "pass@1": 0.2393671875,
565
+ "pass@2": 0.337944881889764,
566
+ "pass@4": 0.435632553618297,
567
+ "pass@8": 0.5206967651312548,
568
+ "pass@16": 0.5906179695379268,
569
+ "pass@32": 0.6484050092386298,
570
+ "pass@64": 0.6969525325161366,
571
+ "pass@128": 0.736
572
+ },
573
+ "19": {
574
+ "pass@1": 0.188734375,
575
+ "pass@2": 0.2653257874015747,
576
+ "pass@4": 0.3448389823772027,
577
+ "pass@8": 0.42249623713425216,
578
+ "pass@16": 0.4957875266004969,
579
+ "pass@32": 0.5655016837307463,
580
+ "pass@64": 0.6324997305839606,
581
+ "pass@128": 0.695
582
+ },
583
+ "20": {
584
+ "pass@1": 0.1840078125,
585
+ "pass@2": 0.2631775344488189,
586
+ "pass@4": 0.34637087682789613,
587
+ "pass@8": 0.4258807772719833,
588
+ "pass@16": 0.49645821706625787,
589
+ "pass@32": 0.5599689078458404,
590
+ "pass@64": 0.6199099329613685,
591
+ "pass@128": 0.675
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.7578203125,
596
+ "12": 0.641609375,
597
+ "13": 0.465140625,
598
+ "14": 0.43965625,
599
+ "15": 0.3664609375,
600
+ "16": 0.281734375,
601
+ "17": 0.2525,
602
+ "18": 0.2393671875,
603
+ "19": 0.188734375,
604
+ "20": 0.1840078125
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 298.0571640625,
608
+ "12": 304.7676015625,
609
+ "13": 319.7301328125,
610
+ "14": 341.958609375,
611
+ "15": 356.6907265625,
612
+ "16": 359.9821875,
613
+ "17": 375.317109375,
614
+ "18": 383.491953125,
615
+ "19": 383.327109375,
616
+ "20": 396.21040625
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 298.0571640625,
621
+ "12": 304.7676015625,
622
+ "13": 319.7301328125,
623
+ "14": 341.958609375,
624
+ "15": 356.6907265625,
625
+ "16": 359.9821875,
626
+ "17": 375.317109375,
627
+ "18": 383.491953125,
628
+ "19": 383.327109375,
629
+ "20": 396.21040625
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 166367,
636
+ "answer_accuracy": 0.386253250371471,
637
+ "avg_response_len": 331.85956770059437,
638
+ "resp_tokens_sum": 142938553,
639
+ "pass_at_k": {
640
+ "pass@1": 0.386253250371471,
641
+ "pass@2": 0.4909063088649968,
642
+ "pass@4": 0.5805378291773731,
643
+ "pass@8": 0.6526600886267144,
644
+ "pass@16": 0.7094506722937122,
645
+ "pass@32": 0.7551306301579921,
646
+ "pass@64": 0.7938336063060175,
647
+ "pass@128": 0.8276374442793462
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.7575690509915014,
652
+ "pass@2": 0.8498137449532688,
653
+ "pass@4": 0.9047521838629934,
654
+ "pass@8": 0.9377886732347469,
655
+ "pass@16": 0.960041149186458,
656
+ "pass@32": 0.9746239590826914,
657
+ "pass@64": 0.9825700799894423,
658
+ "pass@128": 0.9858356940509915
659
+ },
660
+ "12": {
661
+ "pass@1": 0.6333200918079096,
662
+ "pass@2": 0.7547513095555853,
663
+ "pass@4": 0.8386714274592795,
664
+ "pass@8": 0.8928341163275466,
665
+ "pass@16": 0.9259803364931931,
666
+ "pass@32": 0.9447383494093016,
667
+ "pass@64": 0.9572942446713538,
668
+ "pass@128": 0.9689265536723164
669
+ },
670
+ "13": {
671
+ "pass@1": 0.4841300940438871,
672
+ "pass@2": 0.6205060721249966,
673
+ "pass@4": 0.7295860708438091,
674
+ "pass@8": 0.8059492526033689,
675
+ "pass@16": 0.8568992077900991,
676
+ "pass@32": 0.8937514227248922,
677
+ "pass@64": 0.9225152134466567,
678
+ "pass@128": 0.9435736677115988
679
+ },
680
+ "14": {
681
+ "pass@1": 0.4497663551401869,
682
+ "pass@2": 0.571845027841146,
683
+ "pass@4": 0.670868385610677,
684
+ "pass@8": 0.7470698554728201,
685
+ "pass@16": 0.8054958898697235,
686
+ "pass@32": 0.8505787819273606,
687
+ "pass@64": 0.884048299310893,
688
+ "pass@128": 0.9065420560747663
689
+ },
690
+ "15": {
691
+ "pass@1": 0.36836163294797686,
692
+ "pass@2": 0.4893602646670612,
693
+ "pass@4": 0.5954076710425649,
694
+ "pass@8": 0.6802021581752167,
695
+ "pass@16": 0.7428461601629132,
696
+ "pass@32": 0.7885083115987753,
697
+ "pass@64": 0.8230281644538102,
698
+ "pass@128": 0.8497109826589595
699
+ },
700
+ "16": {
701
+ "pass@1": 0.28364415322580644,
702
+ "pass@2": 0.39256453200406405,
703
+ "pass@4": 0.4944607647834345,
704
+ "pass@8": 0.5805687140274347,
705
+ "pass@16": 0.6532026734319762,
706
+ "pass@32": 0.714863241409248,
707
+ "pass@64": 0.7669355763349044,
708
+ "pass@128": 0.8129032258064516
709
+ },
710
+ "17": {
711
+ "pass@1": 0.2501128973988439,
712
+ "pass@2": 0.34497284773109993,
713
+ "pass@4": 0.43463790797826557,
714
+ "pass@8": 0.5132459972107867,
715
+ "pass@16": 0.5819257752825826,
716
+ "pass@32": 0.643364780253834,
717
+ "pass@64": 0.7006827171900452,
718
+ "pass@128": 0.7543352601156069
719
+ },
720
+ "18": {
721
+ "pass@1": 0.23118303571428572,
722
+ "pass@2": 0.32596245781777267,
723
+ "pass@4": 0.41872846885210785,
724
+ "pass@8": 0.4994272862212975,
725
+ "pass@16": 0.5665476115510434,
726
+ "pass@32": 0.6217479887245179,
727
+ "pass@64": 0.6670019576727322,
728
+ "pass@128": 0.7
729
+ },
730
+ "19": {
731
+ "pass@1": 0.18700731981981983,
732
+ "pass@2": 0.2625891147052565,
733
+ "pass@4": 0.34095073307278057,
734
+ "pass@8": 0.4179914747194478,
735
+ "pass@16": 0.48919832053560813,
736
+ "pass@32": 0.5569396106072487,
737
+ "pass@64": 0.6262402386445067,
738
+ "pass@128": 0.7027027027027027
739
+ },
740
+ "20": {
741
+ "pass@1": 0.19280217717717718,
742
+ "pass@2": 0.27425383060225567,
743
+ "pass@4": 0.3585156604861329,
744
+ "pass@8": 0.43630993945564095,
745
+ "pass@16": 0.5004092983315865,
746
+ "pass@32": 0.5534055957447018,
747
+ "pass@64": 0.6021505971017288,
748
+ "pass@128": 0.6486486486486487
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 163345,
755
+ "answer_accuracy": 0.3794626263752602,
756
+ "avg_response_len": 347.8358074078204,
757
+ "resp_tokens_sum": 149730793,
758
+ "pass_at_k": {
759
+ "pass@1": 0.3794626263752602,
760
+ "pass@2": 0.48323833970185204,
761
+ "pass@4": 0.5751190968156217,
762
+ "pass@8": 0.6513041251105385,
763
+ "pass@16": 0.7129426720067712,
764
+ "pass@32": 0.7640267389110413,
765
+ "pass@64": 0.8065358289432528,
766
+ "pass@128": 0.8409158489443949
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.7506353539156626,
771
+ "pass@2": 0.8404700247841763,
772
+ "pass@4": 0.8961459129544045,
773
+ "pass@8": 0.9278766949955667,
774
+ "pass@16": 0.9467320736664068,
775
+ "pass@32": 0.9597594455810305,
776
+ "pass@64": 0.9681259167521782,
777
+ "pass@128": 0.9728915662650602
778
+ },
779
+ "12": {
780
+ "pass@1": 0.6611726097178683,
781
+ "pass@2": 0.7715347542764048,
782
+ "pass@4": 0.8428349935881834,
783
+ "pass@8": 0.8867303652307457,
784
+ "pass@16": 0.9159569131757227,
785
+ "pass@32": 0.9389731417905509,
786
+ "pass@64": 0.9577601020248315,
787
+ "pass@128": 0.9717868338557993
788
+ },
789
+ "13": {
790
+ "pass@1": 0.44421831232493,
791
+ "pass@2": 0.5660448234411872,
792
+ "pass@4": 0.66785195968151,
793
+ "pass@8": 0.7483170301361325,
794
+ "pass@16": 0.8105249949265751,
795
+ "pass@32": 0.8597446273776105,
796
+ "pass@64": 0.8981030316546639,
797
+ "pass@128": 0.9243697478991597
798
+ },
799
+ "14": {
800
+ "pass@1": 0.4415433114035088,
801
+ "pass@2": 0.5532334260026704,
802
+ "pass@4": 0.652206162826138,
803
+ "pass@8": 0.737487007203177,
804
+ "pass@16": 0.8066073874498777,
805
+ "pass@32": 0.8591202423066083,
806
+ "pass@64": 0.8958383932620966,
807
+ "pass@128": 0.9210526315789473
808
+ },
809
+ "15": {
810
+ "pass@1": 0.34607007575757576,
811
+ "pass@2": 0.4742386960152705,
812
+ "pass@4": 0.5880804396609509,
813
+ "pass@8": 0.6771553828886003,
814
+ "pass@16": 0.7422639165081902,
815
+ "pass@32": 0.7913001834008943,
816
+ "pass@64": 0.8312857431811411,
817
+ "pass@128": 0.8636363636363636
818
+ },
819
+ "16": {
820
+ "pass@1": 0.2948940527065527,
821
+ "pass@2": 0.4010665533795455,
822
+ "pass@4": 0.49861933924926044,
823
+ "pass@8": 0.5817780568854317,
824
+ "pass@16": 0.6520585361736072,
825
+ "pass@32": 0.7131258389048448,
826
+ "pass@64": 0.7662778139871184,
827
+ "pass@128": 0.811965811965812
828
+ },
829
+ "17": {
830
+ "pass@1": 0.2685364906832298,
831
+ "pass@2": 0.37610842972074143,
832
+ "pass@4": 0.48323010506838776,
833
+ "pass@8": 0.577938445005628,
834
+ "pass@16": 0.653921104286253,
835
+ "pass@32": 0.7138194380494706,
836
+ "pass@64": 0.7643783394471984,
837
+ "pass@128": 0.8136645962732919
838
+ },
839
+ "18": {
840
+ "pass@1": 0.24377297794117647,
841
+ "pass@2": 0.3515556247105141,
842
+ "pass@4": 0.46032718851320026,
843
+ "pass@8": 0.5538321588382297,
844
+ "pass@16": 0.6284314018506372,
845
+ "pass@32": 0.6875919900487388,
846
+ "pass@64": 0.7341714824652724,
847
+ "pass@128": 0.7705882352941177
848
+ },
849
+ "19": {
850
+ "pass@1": 0.1875,
851
+ "pass@2": 0.2628534958520808,
852
+ "pass@4": 0.34137497935525907,
853
+ "pass@8": 0.41886761741849504,
854
+ "pass@16": 0.4948485199164589,
855
+ "pass@32": 0.5684772948464838,
856
+ "pass@64": 0.6340076592011885,
857
+ "pass@128": 0.6845238095238095
858
+ },
859
+ "20": {
860
+ "pass@1": 0.16871725299401197,
861
+ "pass@2": 0.24689621740770412,
862
+ "pass@4": 0.3304425002638143,
863
+ "pass@8": 0.41089814433551575,
864
+ "pass@16": 0.483789637592383,
865
+ "pass@32": 0.5521726444199829,
866
+ "pass@64": 0.6178883001231452,
867
+ "pass@128": 0.6766467065868264
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 158868,
874
+ "answer_accuracy": 0.37932648227383864,
875
+ "avg_response_len": 376.8501633175428,
876
+ "resp_tokens_sum": 157830878,
877
+ "pass_at_k": {
878
+ "pass@1": 0.37932648227383864,
879
+ "pass@2": 0.4811509882226685,
880
+ "pass@4": 0.5713782288140037,
881
+ "pass@8": 0.6472820735788902,
882
+ "pass@16": 0.7094279117531237,
883
+ "pass@32": 0.7607737146582738,
884
+ "pass@64": 0.8052514211733067,
885
+ "pass@128": 0.8432151589242054
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.7656746031746032,
890
+ "pass@2": 0.8513830302462192,
891
+ "pass@4": 0.9025311645568108,
892
+ "pass@8": 0.9354963214647195,
893
+ "pass@16": 0.9582898489222895,
894
+ "pass@32": 0.974310122831304,
895
+ "pass@64": 0.9856107928071083,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.6314984709480123,
900
+ "pass@2": 0.7500733674059092,
901
+ "pass@4": 0.8329923644865482,
902
+ "pass@8": 0.887867580321604,
903
+ "pass@16": 0.923631780836543,
904
+ "pass@32": 0.946600124993897,
905
+ "pass@64": 0.9621206964233943,
906
+ "pass@128": 0.9755351681957186
907
+ },
908
+ "13": {
909
+ "pass@1": 0.4694974922839506,
910
+ "pass@2": 0.5959508967629046,
911
+ "pass@4": 0.6995019520129422,
912
+ "pass@8": 0.7776365616412141,
913
+ "pass@16": 0.8346225468513689,
914
+ "pass@32": 0.8761886815828364,
915
+ "pass@64": 0.9078665970466824,
916
+ "pass@128": 0.9290123456790124
917
+ },
918
+ "14": {
919
+ "pass@1": 0.42811109050445106,
920
+ "pass@2": 0.543148277413024,
921
+ "pass@4": 0.6509376348357051,
922
+ "pass@8": 0.7436043737795209,
923
+ "pass@16": 0.8145172014899712,
924
+ "pass@32": 0.8642210760480362,
925
+ "pass@64": 0.9006000656184018,
926
+ "pass@128": 0.9287833827893175
927
+ },
928
+ "15": {
929
+ "pass@1": 0.3851996527777778,
930
+ "pass@2": 0.5078033865558473,
931
+ "pass@4": 0.6108883527869196,
932
+ "pass@8": 0.6910061074300264,
933
+ "pass@16": 0.7513617520911909,
934
+ "pass@32": 0.7985095076620196,
935
+ "pass@64": 0.8389313592242905,
936
+ "pass@128": 0.8734567901234568
937
+ },
938
+ "16": {
939
+ "pass@1": 0.26636246312684364,
940
+ "pass@2": 0.36768960641534865,
941
+ "pass@4": 0.46721277124651445,
942
+ "pass@8": 0.5581714943434668,
943
+ "pass@16": 0.6403607605318539,
944
+ "pass@32": 0.7137040695583681,
945
+ "pass@64": 0.7760761955425775,
946
+ "pass@128": 0.8259587020648967
947
+ },
948
+ "17": {
949
+ "pass@1": 0.23943429969879518,
950
+ "pass@2": 0.33819468326060154,
951
+ "pass@4": 0.436030319534606,
952
+ "pass@8": 0.5232784476635474,
953
+ "pass@16": 0.5976341299105322,
954
+ "pass@32": 0.6617823167109694,
955
+ "pass@64": 0.7208114445140381,
956
+ "pass@128": 0.7771084337349398
957
+ },
958
+ "18": {
959
+ "pass@1": 0.24377520161290323,
960
+ "pass@2": 0.33654551371602726,
961
+ "pass@4": 0.4276333723405545,
962
+ "pass@8": 0.5083686482219439,
963
+ "pass@16": 0.5763213834382153,
964
+ "pass@32": 0.6355223760273493,
965
+ "pass@64": 0.6899469138467323,
966
+ "pass@128": 0.7387096774193549
967
+ },
968
+ "19": {
969
+ "pass@1": 0.19172488670694865,
970
+ "pass@2": 0.2705886332278707,
971
+ "pass@4": 0.35226705498670674,
972
+ "pass@8": 0.43071165135970435,
973
+ "pass@16": 0.5033697376743476,
974
+ "pass@32": 0.5710949315109166,
975
+ "pass@64": 0.6372663372318448,
976
+ "pass@128": 0.6978851963746223
977
+ },
978
+ "20": {
979
+ "pass@1": 0.19054992492492492,
980
+ "pass@2": 0.2684314481804639,
981
+ "pass@4": 0.35020230269639707,
982
+ "pass@8": 0.43047924091649253,
983
+ "pass@16": 0.5052137590570093,
984
+ "pass@32": 0.5743518955753166,
985
+ "pass@64": 0.639696972628713,
986
+ "pass@128": 0.6996996996996997
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.6335777138157894,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.6335777138157894,
1002
+ "pass@2": 0.7044544589204002,
1003
+ "pass@4": 0.7609624857221504,
1004
+ "pass@8": 0.8055802949607513,
1005
+ "pass@16": 0.8406621429211865,
1006
+ "pass@32": 0.8689154536686422,
1007
+ "pass@64": 0.8925336057312936,
1008
+ "pass@128": 0.9123157894736842
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.8259609375,
1013
+ "pass@2": 0.893026451771654,
1014
+ "pass@4": 0.9322231326396698,
1015
+ "pass@8": 0.9565009551603775,
1016
+ "pass@16": 0.971507385154414,
1017
+ "pass@32": 0.9808318511743037,
1018
+ "pass@64": 0.9876695059302211,
1019
+ "pass@128": 0.993
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.935625,
1023
+ "pass@2": 0.9512100147637794,
1024
+ "pass@4": 0.9636213038995125,
1025
+ "pass@8": 0.9727229765191304,
1026
+ "pass@16": 0.978889970330367,
1027
+ "pass@32": 0.982836458697959,
1028
+ "pass@64": 0.9851793710344321,
1029
+ "pass@128": 0.987
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.977046875,
1033
+ "pass@2": 0.9854843750000004,
1034
+ "pass@4": 0.9901338926696664,
1035
+ "pass@8": 0.9932631652553967,
1036
+ "pass@16": 0.9954292364313113,
1037
+ "pass@32": 0.9967808357169232,
1038
+ "pass@64": 0.9977499040291772,
1039
+ "pass@128": 0.999
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.97296875,
1043
+ "pass@2": 0.9835835383858268,
1044
+ "pass@4": 0.9883603146794149,
1045
+ "pass@8": 0.9911434994665775,
1046
+ "pass@16": 0.9930824160789711,
1047
+ "pass@32": 0.9947548746152566,
1048
+ "pass@64": 0.9964802260461338,
1049
+ "pass@128": 0.998
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.9220625,
1053
+ "pass@2": 0.9477059547244095,
1054
+ "pass@4": 0.963529338582677,
1055
+ "pass@8": 0.9743781813113109,
1056
+ "pass@16": 0.9822554465747628,
1057
+ "pass@32": 0.9885125209347927,
1058
+ "pass@64": 0.9929442929712965,
1059
+ "pass@128": 0.995
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9371796875,
1063
+ "pass@2": 0.9675015994094499,
1064
+ "pass@4": 0.9815036453880766,
1065
+ "pass@8": 0.9885290906061706,
1066
+ "pass@16": 0.9921437299044742,
1067
+ "pass@32": 0.99416612649408,
1068
+ "pass@64": 0.995942199982249,
1069
+ "pass@128": 0.998
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.908703125,
1073
+ "pass@2": 0.9471446850393703,
1074
+ "pass@4": 0.967179319928759,
1075
+ "pass@8": 0.9779200100902848,
1076
+ "pass@16": 0.9847850762647822,
1077
+ "pass@32": 0.9895726876030048,
1078
+ "pass@64": 0.9928923860328146,
1079
+ "pass@128": 0.996
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.8838984375,
1083
+ "pass@2": 0.9365091043307101,
1084
+ "pass@4": 0.9626962175665545,
1085
+ "pass@8": 0.9774632890377969,
1086
+ "pass@16": 0.9864115667605743,
1087
+ "pass@32": 0.9920013654389653,
1088
+ "pass@64": 0.9956127630881643,
1089
+ "pass@128": 0.998
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.8575,
1093
+ "pass@2": 0.9211126968503948,
1094
+ "pass@4": 0.9518551809148847,
1095
+ "pass@8": 0.9696605210813218,
1096
+ "pass@16": 0.9819000425192099,
1097
+ "pass@32": 0.9902488112271479,
1098
+ "pass@64": 0.9952551317449962,
1099
+ "pass@128": 0.998
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.7578203125,
1103
+ "pass@2": 0.847205954724411,
1104
+ "pass@4": 0.9011952808398942,
1105
+ "pass@8": 0.93377580565178,
1106
+ "pass@16": 0.955070876530586,
1107
+ "pass@32": 0.969590082180953,
1108
+ "pass@64": 0.9787324423322356,
1109
+ "pass@128": 0.983
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.641609375,
1113
+ "pass@2": 0.7585755413385825,
1114
+ "pass@4": 0.8381425514623176,
1115
+ "pass@8": 0.8892629624537244,
1116
+ "pass@16": 0.9220148867551953,
1117
+ "pass@32": 0.9435080487950828,
1118
+ "pass@64": 0.9590211028900316,
1119
+ "pass@128": 0.972
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.465140625,
1123
+ "pass@2": 0.5931075295275604,
1124
+ "pass@4": 0.6977997386576684,
1125
+ "pass@8": 0.7762012373108262,
1126
+ "pass@16": 0.8331259756536721,
1127
+ "pass@32": 0.8759206686558857,
1128
+ "pass@64": 0.9090539128333244,
1129
+ "pass@128": 0.932
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.43965625,
1133
+ "pass@2": 0.5558090551181101,
1134
+ "pass@4": 0.657769242407198,
1135
+ "pass@8": 0.7426246540339614,
1136
+ "pass@16": 0.8089162040581582,
1137
+ "pass@32": 0.8580974144957311,
1138
+ "pass@64": 0.8936584566878355,
1139
+ "pass@128": 0.919
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.3664609375,
1143
+ "pass@2": 0.49034571850393777,
1144
+ "pass@4": 0.598005425571804,
1145
+ "pass@8": 0.6826972018891915,
1146
+ "pass@16": 0.7454130715416146,
1147
+ "pass@32": 0.7926700168179653,
1148
+ "pass@64": 0.8309058005394662,
1149
+ "pass@128": 0.862
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.281734375,
1153
+ "pass@2": 0.3871161417322831,
1154
+ "pass@4": 0.4866833546119234,
1155
+ "pass@8": 0.5734005358977272,
1156
+ "pass@16": 0.6484476727811473,
1157
+ "pass@32": 0.7138604538727543,
1158
+ "pass@64": 0.7698033716622331,
1159
+ "pass@128": 0.817
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.2525,
1163
+ "pass@2": 0.35274815452755964,
1164
+ "pass@4": 0.4507468760779903,
1165
+ "pass@8": 0.5374077389510419,
1166
+ "pass@16": 0.6103234449582415,
1167
+ "pass@32": 0.6721658021677973,
1168
+ "pass@64": 0.7278754450284157,
1169
+ "pass@128": 0.781
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.2393671875,
1173
+ "pass@2": 0.337944881889764,
1174
+ "pass@4": 0.435632553618297,
1175
+ "pass@8": 0.5206967651312548,
1176
+ "pass@16": 0.5906179695379268,
1177
+ "pass@32": 0.6484050092386298,
1178
+ "pass@64": 0.6969525325161366,
1179
+ "pass@128": 0.736
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.188734375,
1183
+ "pass@2": 0.2653257874015747,
1184
+ "pass@4": 0.3448389823772027,
1185
+ "pass@8": 0.42249623713425216,
1186
+ "pass@16": 0.4957875266004969,
1187
+ "pass@32": 0.5655016837307463,
1188
+ "pass@64": 0.6324997305839606,
1189
+ "pass@128": 0.695
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.1840078125,
1193
+ "pass@2": 0.2631775344488189,
1194
+ "pass@4": 0.34637087682789613,
1195
+ "pass@8": 0.4258807772719833,
1196
+ "pass@16": 0.49645821706625787,
1197
+ "pass@32": 0.5599689078458404,
1198
+ "pass@64": 0.6199099329613685,
1199
+ "pass@128": 0.675
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.8259609375,
1204
+ "2": 0.935625,
1205
+ "3": 0.977046875,
1206
+ "4": 0.97296875,
1207
+ "5": 0.9220625,
1208
+ "6": 0.9371796875,
1209
+ "7": 0.908703125,
1210
+ "8": 0.8838984375,
1211
+ "9": 0.8575,
1212
+ "11": 0.7578203125,
1213
+ "12": 0.641609375,
1214
+ "13": 0.465140625,
1215
+ "14": 0.43965625,
1216
+ "15": 0.3664609375,
1217
+ "16": 0.281734375,
1218
+ "17": 0.2525,
1219
+ "18": 0.2393671875,
1220
+ "19": 0.188734375,
1221
+ "20": 0.1840078125
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 281.24259375,
1225
+ "2": 122.4647578125,
1226
+ "3": 146.4731015625,
1227
+ "4": 160.6047265625,
1228
+ "5": 176.30828125,
1229
+ "6": 201.5447421875,
1230
+ "7": 226.3348359375,
1231
+ "8": 246.195390625,
1232
+ "9": 262.26378125,
1233
+ "11": 298.0571640625,
1234
+ "12": 304.7676015625,
1235
+ "13": 319.7301328125,
1236
+ "14": 341.958609375,
1237
+ "15": 356.6907265625,
1238
+ "16": 359.9821875,
1239
+ "17": 375.317109375,
1240
+ "18": 383.491953125,
1241
+ "19": 383.327109375,
1242
+ "20": 396.21040625
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 281.24259375,
1247
+ "2": 122.4647578125,
1248
+ "3": 146.4731015625,
1249
+ "4": 160.6047265625,
1250
+ "5": 176.30828125,
1251
+ "6": 201.5447421875,
1252
+ "7": 226.3348359375,
1253
+ "8": 246.195390625,
1254
+ "9": 262.26378125,
1255
+ "11": 298.0571640625,
1256
+ "12": 304.7676015625,
1257
+ "13": 319.7301328125,
1258
+ "14": 341.958609375,
1259
+ "15": 356.6907265625,
1260
+ "16": 359.9821875,
1261
+ "17": 375.317109375,
1262
+ "18": 383.491953125,
1263
+ "19": 383.327109375,
1264
+ "20": 396.21040625
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 510499,
1271
+ "answer_accuracy": 0.6304573881599747,
1272
+ "avg_response_len": 266.6437346862156,
1273
+ "resp_tokens_sum": 215908898,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.6304573881599747,
1276
+ "pass@2": 0.7038732633227223,
1277
+ "pass@4": 0.7610112612928939,
1278
+ "pass@8": 0.8048920516478486,
1279
+ "pass@16": 0.838512105529119,
1280
+ "pass@32": 0.8650410497814764,
1281
+ "pass@64": 0.8871377219105482,
1282
+ "pass@128": 0.9062598798608915
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.8337020648967551,
1287
+ "pass@2": 0.8992248652823267,
1288
+ "pass@4": 0.9340549527547986,
1289
+ "pass@8": 0.9549133337554659,
1290
+ "pass@16": 0.9676459667453204,
1291
+ "pass@32": 0.9754629221119615,
1292
+ "pass@64": 0.9816283664853948,
1293
+ "pass@128": 0.9882005899705014
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9328514408099688,
1297
+ "pass@2": 0.9459657934113376,
1298
+ "pass@4": 0.9572665329801058,
1299
+ "pass@8": 0.9661779608434554,
1300
+ "pass@16": 0.9726776857540342,
1301
+ "pass@32": 0.9771842431316051,
1302
+ "pass@64": 0.9796170198954688,
1303
+ "pass@128": 0.9813084112149533
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9756068425076453,
1307
+ "pass@2": 0.9874613222085766,
1308
+ "pass@4": 0.9940523552675177,
1309
+ "pass@8": 0.9974856871737064,
1310
+ "pass@16": 0.9991656932084267,
1311
+ "pass@32": 0.9998836449467822,
1312
+ "pass@64": 0.9999990650219919,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.975341796875,
1317
+ "pass@2": 0.9855330339566928,
1318
+ "pass@4": 0.9891930282152229,
1319
+ "pass@8": 0.9909968147034551,
1320
+ "pass@16": 0.9920335522380169,
1321
+ "pass@32": 0.9932232658217899,
1322
+ "pass@64": 0.9949311023622034,
1323
+ "pass@128": 0.996875
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9057291666666667,
1327
+ "pass@2": 0.9370816929133859,
1328
+ "pass@4": 0.9557188610352269,
1329
+ "pass@8": 0.9680085338015769,
1330
+ "pass@16": 0.9772068251757668,
1331
+ "pass@32": 0.9850956818037647,
1332
+ "pass@64": 0.9907377001622719,
1333
+ "pass@128": 0.9936507936507937
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9124719311377245,
1337
+ "pass@2": 0.9472194881889765,
1338
+ "pass@4": 0.967283268951411,
1339
+ "pass@8": 0.9804620871830513,
1340
+ "pass@16": 0.9883802999024801,
1341
+ "pass@32": 0.9926513268166495,
1342
+ "pass@64": 0.9960828629239575,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.8930981878698225,
1347
+ "pass@2": 0.9425206314354944,
1348
+ "pass@4": 0.9694562882857096,
1349
+ "pass@8": 0.9828592027889445,
1350
+ "pass@16": 0.9902744534062847,
1351
+ "pass@32": 0.9950889757930396,
1352
+ "pass@64": 0.9980455562464515,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.8852413862179487,
1357
+ "pass@2": 0.9402193115283668,
1358
+ "pass@4": 0.9669937225394901,
1359
+ "pass@8": 0.9816570277298262,
1360
+ "pass@16": 0.990420934947623,
1361
+ "pass@32": 0.996331498679127,
1362
+ "pass@64": 0.9995570396284617,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.8653389084507043,
1367
+ "pass@2": 0.9311758345347674,
1368
+ "pass@4": 0.9623128128350147,
1369
+ "pass@8": 0.9789012564778983,
1370
+ "pass@16": 0.9887982028366319,
1371
+ "pass@32": 0.994471708460012,
1372
+ "pass@64": 0.9977497769018057,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.7575690509915014,
1377
+ "pass@2": 0.8498137449532688,
1378
+ "pass@4": 0.9047521838629934,
1379
+ "pass@8": 0.9377886732347469,
1380
+ "pass@16": 0.960041149186458,
1381
+ "pass@32": 0.9746239590826914,
1382
+ "pass@64": 0.9825700799894423,
1383
+ "pass@128": 0.9858356940509915
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.6333200918079096,
1387
+ "pass@2": 0.7547513095555853,
1388
+ "pass@4": 0.8386714274592795,
1389
+ "pass@8": 0.8928341163275466,
1390
+ "pass@16": 0.9259803364931931,
1391
+ "pass@32": 0.9447383494093016,
1392
+ "pass@64": 0.9572942446713538,
1393
+ "pass@128": 0.9689265536723164
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.4841300940438871,
1397
+ "pass@2": 0.6205060721249966,
1398
+ "pass@4": 0.7295860708438091,
1399
+ "pass@8": 0.8059492526033689,
1400
+ "pass@16": 0.8568992077900991,
1401
+ "pass@32": 0.8937514227248922,
1402
+ "pass@64": 0.9225152134466567,
1403
+ "pass@128": 0.9435736677115988
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.4497663551401869,
1407
+ "pass@2": 0.571845027841146,
1408
+ "pass@4": 0.670868385610677,
1409
+ "pass@8": 0.7470698554728201,
1410
+ "pass@16": 0.8054958898697235,
1411
+ "pass@32": 0.8505787819273606,
1412
+ "pass@64": 0.884048299310893,
1413
+ "pass@128": 0.9065420560747663
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.36836163294797686,
1417
+ "pass@2": 0.4893602646670612,
1418
+ "pass@4": 0.5954076710425649,
1419
+ "pass@8": 0.6802021581752167,
1420
+ "pass@16": 0.7428461601629132,
1421
+ "pass@32": 0.7885083115987753,
1422
+ "pass@64": 0.8230281644538102,
1423
+ "pass@128": 0.8497109826589595
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.28364415322580644,
1427
+ "pass@2": 0.39256453200406405,
1428
+ "pass@4": 0.4944607647834345,
1429
+ "pass@8": 0.5805687140274347,
1430
+ "pass@16": 0.6532026734319762,
1431
+ "pass@32": 0.714863241409248,
1432
+ "pass@64": 0.7669355763349044,
1433
+ "pass@128": 0.8129032258064516
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.2501128973988439,
1437
+ "pass@2": 0.34497284773109993,
1438
+ "pass@4": 0.43463790797826557,
1439
+ "pass@8": 0.5132459972107867,
1440
+ "pass@16": 0.5819257752825826,
1441
+ "pass@32": 0.643364780253834,
1442
+ "pass@64": 0.7006827171900452,
1443
+ "pass@128": 0.7543352601156069
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.23118303571428572,
1447
+ "pass@2": 0.32596245781777267,
1448
+ "pass@4": 0.41872846885210785,
1449
+ "pass@8": 0.4994272862212975,
1450
+ "pass@16": 0.5665476115510434,
1451
+ "pass@32": 0.6217479887245179,
1452
+ "pass@64": 0.6670019576727322,
1453
+ "pass@128": 0.7
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.18700731981981983,
1457
+ "pass@2": 0.2625891147052565,
1458
+ "pass@4": 0.34095073307278057,
1459
+ "pass@8": 0.4179914747194478,
1460
+ "pass@16": 0.48919832053560813,
1461
+ "pass@32": 0.5569396106072487,
1462
+ "pass@64": 0.6262402386445067,
1463
+ "pass@128": 0.7027027027027027
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.19280217717717718,
1467
+ "pass@2": 0.27425383060225567,
1468
+ "pass@4": 0.3585156604861329,
1469
+ "pass@8": 0.43630993945564095,
1470
+ "pass@16": 0.5004092983315865,
1471
+ "pass@32": 0.5534055957447018,
1472
+ "pass@64": 0.6021505971017288,
1473
+ "pass@128": 0.6486486486486487
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 522241,
1480
+ "answer_accuracy": 0.6360105709275137,
1481
+ "avg_response_len": 277.60607828332036,
1482
+ "resp_tokens_sum": 227947903,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.6360105709275137,
1485
+ "pass@2": 0.7061336235201687,
1486
+ "pass@4": 0.7623702717255768,
1487
+ "pass@8": 0.8071611606699909,
1488
+ "pass@16": 0.8427868598710215,
1489
+ "pass@32": 0.8718723024536624,
1490
+ "pass@64": 0.8957240571987134,
1491
+ "pass@128": 0.9147310989867498
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.8397478911042945,
1496
+ "pass@2": 0.9017401031351142,
1497
+ "pass@4": 0.9367013483935669,
1498
+ "pass@8": 0.9597981710002264,
1499
+ "pass@16": 0.9760430566524732,
1500
+ "pass@32": 0.9868893358013864,
1501
+ "pass@64": 0.9939010380542732,
1502
+ "pass@128": 0.9969325153374233
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.9402308130699089,
1506
+ "pass@2": 0.9559446275997415,
1507
+ "pass@4": 0.9671450020115263,
1508
+ "pass@8": 0.9747791046336394,
1509
+ "pass@16": 0.9798526018916596,
1510
+ "pass@32": 0.982735766252235,
1511
+ "pass@64": 0.9843437813422433,
1512
+ "pass@128": 0.9848024316109423
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.98203125,
1516
+ "pass@2": 0.9875126759723218,
1517
+ "pass@4": 0.9907982465260023,
1518
+ "pass@8": 0.9931905013110562,
1519
+ "pass@16": 0.9947669019933483,
1520
+ "pass@32": 0.996016072122554,
1521
+ "pass@64": 0.9977332350126279,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9693047337278107,
1526
+ "pass@2": 0.9827249947584216,
1527
+ "pass@4": 0.9890199579342521,
1528
+ "pass@8": 0.9924316167169746,
1529
+ "pass@16": 0.9952105268225401,
1530
+ "pass@32": 0.9977446704990522,
1531
+ "pass@64": 0.9995571159589419,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.948052536231884,
1536
+ "pass@2": 0.966407052379322,
1537
+ "pass@4": 0.9765803837020368,
1538
+ "pass@8": 0.9844381946054411,
1539
+ "pass@16": 0.9908400253696799,
1540
+ "pass@32": 0.9954288349662425,
1541
+ "pass@64": 0.9982494068365969,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9469914040114613,
1546
+ "pass@2": 0.9771242498251473,
1547
+ "pass@4": 0.9882326420336424,
1548
+ "pass@8": 0.991931583238474,
1549
+ "pass@16": 0.9933659205077896,
1550
+ "pass@32": 0.9940735913681014,
1551
+ "pass@64": 0.9942652013230373,
1552
+ "pass@128": 0.994269340974212
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9118722904624278,
1556
+ "pass@2": 0.9468681728187156,
1557
+ "pass@4": 0.9655806095047366,
1558
+ "pass@8": 0.9770071107892204,
1559
+ "pass@16": 0.9847477489910627,
1560
+ "pass@32": 0.9900681636945455,
1561
+ "pass@64": 0.9936873526856657,
1562
+ "pass@128": 0.9971098265895953
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.8875,
1566
+ "pass@2": 0.9402195159143837,
1567
+ "pass@4": 0.9639932780409486,
1568
+ "pass@8": 0.9756221536661233,
1569
+ "pass@16": 0.9821909911786181,
1570
+ "pass@32": 0.987287230265571,
1571
+ "pass@64": 0.992438504371287,
1572
+ "pass@128": 0.9971830985915493
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.8411302395209581,
1576
+ "pass@2": 0.905336104366071,
1577
+ "pass@4": 0.938773440464403,
1578
+ "pass@8": 0.9598987045631255,
1579
+ "pass@16": 0.9751816882329573,
1580
+ "pass@32": 0.9858225193162448,
1581
+ "pass@64": 0.9916186385525599,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.7506353539156626,
1586
+ "pass@2": 0.8404700247841763,
1587
+ "pass@4": 0.8961459129544045,
1588
+ "pass@8": 0.9278766949955667,
1589
+ "pass@16": 0.9467320736664068,
1590
+ "pass@32": 0.9597594455810305,
1591
+ "pass@64": 0.9681259167521782,
1592
+ "pass@128": 0.9728915662650602
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.6611726097178683,
1596
+ "pass@2": 0.7715347542764048,
1597
+ "pass@4": 0.8428349935881834,
1598
+ "pass@8": 0.8867303652307457,
1599
+ "pass@16": 0.9159569131757227,
1600
+ "pass@32": 0.9389731417905509,
1601
+ "pass@64": 0.9577601020248315,
1602
+ "pass@128": 0.9717868338557993
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.44421831232493,
1606
+ "pass@2": 0.5660448234411872,
1607
+ "pass@4": 0.66785195968151,
1608
+ "pass@8": 0.7483170301361325,
1609
+ "pass@16": 0.8105249949265751,
1610
+ "pass@32": 0.8597446273776105,
1611
+ "pass@64": 0.8981030316546639,
1612
+ "pass@128": 0.9243697478991597
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.4415433114035088,
1616
+ "pass@2": 0.5532334260026704,
1617
+ "pass@4": 0.652206162826138,
1618
+ "pass@8": 0.737487007203177,
1619
+ "pass@16": 0.8066073874498777,
1620
+ "pass@32": 0.8591202423066083,
1621
+ "pass@64": 0.8958383932620966,
1622
+ "pass@128": 0.9210526315789473
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.34607007575757576,
1626
+ "pass@2": 0.4742386960152705,
1627
+ "pass@4": 0.5880804396609509,
1628
+ "pass@8": 0.6771553828886003,
1629
+ "pass@16": 0.7422639165081902,
1630
+ "pass@32": 0.7913001834008943,
1631
+ "pass@64": 0.8312857431811411,
1632
+ "pass@128": 0.8636363636363636
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.2948940527065527,
1636
+ "pass@2": 0.4010665533795455,
1637
+ "pass@4": 0.49861933924926044,
1638
+ "pass@8": 0.5817780568854317,
1639
+ "pass@16": 0.6520585361736072,
1640
+ "pass@32": 0.7131258389048448,
1641
+ "pass@64": 0.7662778139871184,
1642
+ "pass@128": 0.811965811965812
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.2685364906832298,
1646
+ "pass@2": 0.37610842972074143,
1647
+ "pass@4": 0.48323010506838776,
1648
+ "pass@8": 0.577938445005628,
1649
+ "pass@16": 0.653921104286253,
1650
+ "pass@32": 0.7138194380494706,
1651
+ "pass@64": 0.7643783394471984,
1652
+ "pass@128": 0.8136645962732919
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.24377297794117647,
1656
+ "pass@2": 0.3515556247105141,
1657
+ "pass@4": 0.46032718851320026,
1658
+ "pass@8": 0.5538321588382297,
1659
+ "pass@16": 0.6284314018506372,
1660
+ "pass@32": 0.6875919900487388,
1661
+ "pass@64": 0.7341714824652724,
1662
+ "pass@128": 0.7705882352941177
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.1875,
1666
+ "pass@2": 0.2628534958520808,
1667
+ "pass@4": 0.34137497935525907,
1668
+ "pass@8": 0.41886761741849504,
1669
+ "pass@16": 0.4948485199164589,
1670
+ "pass@32": 0.5684772948464838,
1671
+ "pass@64": 0.6340076592011885,
1672
+ "pass@128": 0.6845238095238095
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.16871725299401197,
1676
+ "pass@2": 0.24689621740770412,
1677
+ "pass@4": 0.3304425002638143,
1678
+ "pass@8": 0.41089814433551575,
1679
+ "pass@16": 0.483789637592383,
1680
+ "pass@32": 0.5521726444199829,
1681
+ "pass@64": 0.6178883001231452,
1682
+ "pass@128": 0.6766467065868264
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 508121,
1689
+ "answer_accuracy": 0.6342379473558076,
1690
+ "avg_response_len": 299.6219768533312,
1691
+ "resp_tokens_sum": 240042746,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.6342379473558076,
1694
+ "pass@2": 0.703320859694323,
1695
+ "pass@4": 0.7594703142135963,
1696
+ "pass@8": 0.8046556382540034,
1697
+ "pass@16": 0.8406575219448124,
1698
+ "pass@32": 0.8698007858357516,
1699
+ "pass@64": 0.8947172796227344,
1700
+ "pass@128": 0.9159610161367631
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.8047108208955224,
1705
+ "pass@2": 0.8782744740862615,
1706
+ "pass@4": 0.9260115345656411,
1707
+ "pass@8": 0.9548988993170184,
1708
+ "pass@16": 0.9710010924150604,
1709
+ "pass@32": 0.980370140618196,
1710
+ "pass@64": 0.9877186605551607,
1711
+ "pass@128": 0.9940298507462687
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9338392857142858,
1715
+ "pass@2": 0.9515691788526434,
1716
+ "pass@4": 0.9661372604317318,
1717
+ "pass@8": 0.9767929304683264,
1718
+ "pass@16": 0.9836826490884742,
1719
+ "pass@32": 0.9881149987306532,
1720
+ "pass@64": 0.9910662959611092,
1721
+ "pass@128": 0.9942857142857143
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9736242711370262,
1725
+ "pass@2": 0.9816482203117467,
1726
+ "pass@4": 0.9857590412350061,
1727
+ "pass@8": 0.9893075222068398,
1728
+ "pass@16": 0.9925043063975238,
1729
+ "pass@32": 0.9945585423291038,
1730
+ "pass@64": 0.9956216974134651,
1731
+ "pass@128": 0.9970845481049563
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9743695175438597,
1735
+ "pass@2": 0.9826079511442651,
1736
+ "pass@4": 0.9869292393823577,
1737
+ "pass@8": 0.9900076968161828,
1738
+ "pass@16": 0.9919605885870972,
1739
+ "pass@32": 0.9932331313555678,
1740
+ "pass@64": 0.9948887956026504,
1741
+ "pass@128": 0.9970760233918129
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9108226102941176,
1745
+ "pass@2": 0.9385729070171378,
1746
+ "pass@4": 0.9575225440569924,
1747
+ "pass@8": 0.9700715177204019,
1748
+ "pass@16": 0.9782220232113423,
1749
+ "pass@32": 0.9846600973625091,
1750
+ "pass@64": 0.9896055060633964,
1751
+ "pass@128": 0.9911764705882353
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9524102917981072,
1755
+ "pass@2": 0.9782773727613706,
1756
+ "pass@4": 0.9890783643172356,
1757
+ "pass@8": 0.9932827474347146,
1758
+ "pass@16": 0.994763417917439,
1759
+ "pass@32": 0.9958640376966922,
1760
+ "pass@64": 0.9976402791921362,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9219244462025317,
1765
+ "pass@2": 0.9523934298066383,
1766
+ "pass@4": 0.9664943151884493,
1767
+ "pass@8": 0.973636516501745,
1768
+ "pass@16": 0.9789543982992107,
1769
+ "pass@32": 0.983129839071662,
1770
+ "pass@64": 0.9865100126337769,
1771
+ "pass@128": 0.990506329113924
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.8788006756756757,
1775
+ "pass@2": 0.9290773302830391,
1776
+ "pass@4": 0.9572869742633514,
1777
+ "pass@8": 0.9754967922961485,
1778
+ "pass@16": 0.9871544540195394,
1779
+ "pass@32": 0.992969883203604,
1780
+ "pass@64": 0.9953011942110427,
1781
+ "pass@128": 0.996996996996997
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.8661324356913184,
1785
+ "pass@2": 0.9265691856343519,
1786
+ "pass@4": 0.9539672130011069,
1787
+ "pass@8": 0.9695961662623275,
1788
+ "pass@16": 0.9812411467601234,
1789
+ "pass@32": 0.9901821005537559,
1790
+ "pass@64": 0.9963129764254025,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.7656746031746032,
1795
+ "pass@2": 0.8513830302462192,
1796
+ "pass@4": 0.9025311645568108,
1797
+ "pass@8": 0.9354963214647195,
1798
+ "pass@16": 0.9582898489222895,
1799
+ "pass@32": 0.974310122831304,
1800
+ "pass@64": 0.9856107928071083,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.6314984709480123,
1805
+ "pass@2": 0.7500733674059092,
1806
+ "pass@4": 0.8329923644865482,
1807
+ "pass@8": 0.887867580321604,
1808
+ "pass@16": 0.923631780836543,
1809
+ "pass@32": 0.946600124993897,
1810
+ "pass@64": 0.9621206964233943,
1811
+ "pass@128": 0.9755351681957186
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.4694974922839506,
1815
+ "pass@2": 0.5959508967629046,
1816
+ "pass@4": 0.6995019520129422,
1817
+ "pass@8": 0.7776365616412141,
1818
+ "pass@16": 0.8346225468513689,
1819
+ "pass@32": 0.8761886815828364,
1820
+ "pass@64": 0.9078665970466824,
1821
+ "pass@128": 0.9290123456790124
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.42811109050445106,
1825
+ "pass@2": 0.543148277413024,
1826
+ "pass@4": 0.6509376348357051,
1827
+ "pass@8": 0.7436043737795209,
1828
+ "pass@16": 0.8145172014899712,
1829
+ "pass@32": 0.8642210760480362,
1830
+ "pass@64": 0.9006000656184018,
1831
+ "pass@128": 0.9287833827893175
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.3851996527777778,
1835
+ "pass@2": 0.5078033865558473,
1836
+ "pass@4": 0.6108883527869196,
1837
+ "pass@8": 0.6910061074300264,
1838
+ "pass@16": 0.7513617520911909,
1839
+ "pass@32": 0.7985095076620196,
1840
+ "pass@64": 0.8389313592242905,
1841
+ "pass@128": 0.8734567901234568
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.26636246312684364,
1845
+ "pass@2": 0.36768960641534865,
1846
+ "pass@4": 0.46721277124651445,
1847
+ "pass@8": 0.5581714943434668,
1848
+ "pass@16": 0.6403607605318539,
1849
+ "pass@32": 0.7137040695583681,
1850
+ "pass@64": 0.7760761955425775,
1851
+ "pass@128": 0.8259587020648967
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.23943429969879518,
1855
+ "pass@2": 0.33819468326060154,
1856
+ "pass@4": 0.436030319534606,
1857
+ "pass@8": 0.5232784476635474,
1858
+ "pass@16": 0.5976341299105322,
1859
+ "pass@32": 0.6617823167109694,
1860
+ "pass@64": 0.7208114445140381,
1861
+ "pass@128": 0.7771084337349398
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.24377520161290323,
1865
+ "pass@2": 0.33654551371602726,
1866
+ "pass@4": 0.4276333723405545,
1867
+ "pass@8": 0.5083686482219439,
1868
+ "pass@16": 0.5763213834382153,
1869
+ "pass@32": 0.6355223760273493,
1870
+ "pass@64": 0.6899469138467323,
1871
+ "pass@128": 0.7387096774193549
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.19172488670694865,
1875
+ "pass@2": 0.2705886332278707,
1876
+ "pass@4": 0.35226705498670674,
1877
+ "pass@8": 0.43071165135970435,
1878
+ "pass@16": 0.5033697376743476,
1879
+ "pass@32": 0.5710949315109166,
1880
+ "pass@64": 0.6372663372318448,
1881
+ "pass@128": 0.6978851963746223
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.19054992492492492,
1885
+ "pass@2": 0.2684314481804639,
1886
+ "pass@4": 0.35020230269639707,
1887
+ "pass@8": 0.43047924091649253,
1888
+ "pass@16": 0.5052137590570093,
1889
+ "pass@32": 0.5743518955753166,
1890
+ "pass@64": 0.639696972628713,
1891
+ "pass@128": 0.6996996996996997
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch1-op17-20/SDAR-100M-block16-epoch1-op17-20_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e22b3eb97fe3857e6aa68e2b8d4f34312b580d5dfd860e8e8d52a60ab38ca4e5
3
+ size 2642913151
results/composition/SDAR-100M-block16-epoch1-op17-20/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch1-op17-20,0.9134383680555556,0.381703125,0.6335777138157894,0.0,0.0,0.0,202.60357899305555,351.9533,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch1-op17-20/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch1-op17-20",
4
+ "id_acc": 0.9134383680555556,
5
+ "ood_acc": 0.381703125,
6
+ "total_acc": 0.6335777138157894,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 202.60357899305555,
11
+ "ood_avg_resp_len": 351.9533,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 510499,
19
+ "answer_accuracy": 0.6304573881599747,
20
+ "avg_response_len": 266.6437346862156,
21
+ "resp_tokens_sum": 215908898,
22
+ "pass_at_k": {
23
+ "pass@1": 0.6304573881599747,
24
+ "pass@2": 0.7038732633227223,
25
+ "pass@4": 0.7610112612928939,
26
+ "pass@8": 0.8048920516478486,
27
+ "pass@16": 0.838512105529119,
28
+ "pass@32": 0.8650410497814764,
29
+ "pass@64": 0.8871377219105482,
30
+ "pass@128": 0.9062598798608915
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.8337020648967551,
35
+ "pass@2": 0.8992248652823267,
36
+ "pass@4": 0.9340549527547986,
37
+ "pass@8": 0.9549133337554659,
38
+ "pass@16": 0.9676459667453204,
39
+ "pass@32": 0.9754629221119615,
40
+ "pass@64": 0.9816283664853948,
41
+ "pass@128": 0.9882005899705014
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9328514408099688,
45
+ "pass@2": 0.9459657934113376,
46
+ "pass@4": 0.9572665329801058,
47
+ "pass@8": 0.9661779608434554,
48
+ "pass@16": 0.9726776857540342,
49
+ "pass@32": 0.9771842431316051,
50
+ "pass@64": 0.9796170198954688,
51
+ "pass@128": 0.9813084112149533
52
+ },
53
+ "3": {
54
+ "pass@1": 0.9756068425076453,
55
+ "pass@2": 0.9874613222085766,
56
+ "pass@4": 0.9940523552675177,
57
+ "pass@8": 0.9974856871737064,
58
+ "pass@16": 0.9991656932084267,
59
+ "pass@32": 0.9998836449467822,
60
+ "pass@64": 0.9999990650219919,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.975341796875,
65
+ "pass@2": 0.9855330339566928,
66
+ "pass@4": 0.9891930282152229,
67
+ "pass@8": 0.9909968147034551,
68
+ "pass@16": 0.9920335522380169,
69
+ "pass@32": 0.9932232658217899,
70
+ "pass@64": 0.9949311023622034,
71
+ "pass@128": 0.996875
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9057291666666667,
75
+ "pass@2": 0.9370816929133859,
76
+ "pass@4": 0.9557188610352269,
77
+ "pass@8": 0.9680085338015769,
78
+ "pass@16": 0.9772068251757668,
79
+ "pass@32": 0.9850956818037647,
80
+ "pass@64": 0.9907377001622719,
81
+ "pass@128": 0.9936507936507937
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9124719311377245,
85
+ "pass@2": 0.9472194881889765,
86
+ "pass@4": 0.967283268951411,
87
+ "pass@8": 0.9804620871830513,
88
+ "pass@16": 0.9883802999024801,
89
+ "pass@32": 0.9926513268166495,
90
+ "pass@64": 0.9960828629239575,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.8930981878698225,
95
+ "pass@2": 0.9425206314354944,
96
+ "pass@4": 0.9694562882857096,
97
+ "pass@8": 0.9828592027889445,
98
+ "pass@16": 0.9902744534062847,
99
+ "pass@32": 0.9950889757930396,
100
+ "pass@64": 0.9980455562464515,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.8852413862179487,
105
+ "pass@2": 0.9402193115283668,
106
+ "pass@4": 0.9669937225394901,
107
+ "pass@8": 0.9816570277298262,
108
+ "pass@16": 0.990420934947623,
109
+ "pass@32": 0.996331498679127,
110
+ "pass@64": 0.9995570396284617,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.8653389084507043,
115
+ "pass@2": 0.9311758345347674,
116
+ "pass@4": 0.9623128128350147,
117
+ "pass@8": 0.9789012564778983,
118
+ "pass@16": 0.9887982028366319,
119
+ "pass@32": 0.994471708460012,
120
+ "pass@64": 0.9977497769018057,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.7575690509915014,
125
+ "pass@2": 0.8498137449532688,
126
+ "pass@4": 0.9047521838629934,
127
+ "pass@8": 0.9377886732347469,
128
+ "pass@16": 0.960041149186458,
129
+ "pass@32": 0.9746239590826914,
130
+ "pass@64": 0.9825700799894423,
131
+ "pass@128": 0.9858356940509915
132
+ },
133
+ "12": {
134
+ "pass@1": 0.6333200918079096,
135
+ "pass@2": 0.7547513095555853,
136
+ "pass@4": 0.8386714274592795,
137
+ "pass@8": 0.8928341163275466,
138
+ "pass@16": 0.9259803364931931,
139
+ "pass@32": 0.9447383494093016,
140
+ "pass@64": 0.9572942446713538,
141
+ "pass@128": 0.9689265536723164
142
+ },
143
+ "13": {
144
+ "pass@1": 0.4841300940438871,
145
+ "pass@2": 0.6205060721249966,
146
+ "pass@4": 0.7295860708438091,
147
+ "pass@8": 0.8059492526033689,
148
+ "pass@16": 0.8568992077900991,
149
+ "pass@32": 0.8937514227248922,
150
+ "pass@64": 0.9225152134466567,
151
+ "pass@128": 0.9435736677115988
152
+ },
153
+ "14": {
154
+ "pass@1": 0.4497663551401869,
155
+ "pass@2": 0.571845027841146,
156
+ "pass@4": 0.670868385610677,
157
+ "pass@8": 0.7470698554728201,
158
+ "pass@16": 0.8054958898697235,
159
+ "pass@32": 0.8505787819273606,
160
+ "pass@64": 0.884048299310893,
161
+ "pass@128": 0.9065420560747663
162
+ },
163
+ "15": {
164
+ "pass@1": 0.36836163294797686,
165
+ "pass@2": 0.4893602646670612,
166
+ "pass@4": 0.5954076710425649,
167
+ "pass@8": 0.6802021581752167,
168
+ "pass@16": 0.7428461601629132,
169
+ "pass@32": 0.7885083115987753,
170
+ "pass@64": 0.8230281644538102,
171
+ "pass@128": 0.8497109826589595
172
+ },
173
+ "16": {
174
+ "pass@1": 0.28364415322580644,
175
+ "pass@2": 0.39256453200406405,
176
+ "pass@4": 0.4944607647834345,
177
+ "pass@8": 0.5805687140274347,
178
+ "pass@16": 0.6532026734319762,
179
+ "pass@32": 0.714863241409248,
180
+ "pass@64": 0.7669355763349044,
181
+ "pass@128": 0.8129032258064516
182
+ },
183
+ "17": {
184
+ "pass@1": 0.2501128973988439,
185
+ "pass@2": 0.34497284773109993,
186
+ "pass@4": 0.43463790797826557,
187
+ "pass@8": 0.5132459972107867,
188
+ "pass@16": 0.5819257752825826,
189
+ "pass@32": 0.643364780253834,
190
+ "pass@64": 0.7006827171900452,
191
+ "pass@128": 0.7543352601156069
192
+ },
193
+ "18": {
194
+ "pass@1": 0.23118303571428572,
195
+ "pass@2": 0.32596245781777267,
196
+ "pass@4": 0.41872846885210785,
197
+ "pass@8": 0.4994272862212975,
198
+ "pass@16": 0.5665476115510434,
199
+ "pass@32": 0.6217479887245179,
200
+ "pass@64": 0.6670019576727322,
201
+ "pass@128": 0.7
202
+ },
203
+ "19": {
204
+ "pass@1": 0.18700731981981983,
205
+ "pass@2": 0.2625891147052565,
206
+ "pass@4": 0.34095073307278057,
207
+ "pass@8": 0.4179914747194478,
208
+ "pass@16": 0.48919832053560813,
209
+ "pass@32": 0.5569396106072487,
210
+ "pass@64": 0.6262402386445067,
211
+ "pass@128": 0.7027027027027027
212
+ },
213
+ "20": {
214
+ "pass@1": 0.19280217717717718,
215
+ "pass@2": 0.27425383060225567,
216
+ "pass@4": 0.3585156604861329,
217
+ "pass@8": 0.43630993945564095,
218
+ "pass@16": 0.5004092983315865,
219
+ "pass@32": 0.5534055957447018,
220
+ "pass@64": 0.6021505971017288,
221
+ "pass@128": 0.6486486486486487
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 522241,
228
+ "answer_accuracy": 0.6360105709275137,
229
+ "avg_response_len": 277.60607828332036,
230
+ "resp_tokens_sum": 227947903,
231
+ "pass_at_k": {
232
+ "pass@1": 0.6360105709275137,
233
+ "pass@2": 0.7061336235201687,
234
+ "pass@4": 0.7623702717255768,
235
+ "pass@8": 0.8071611606699909,
236
+ "pass@16": 0.8427868598710215,
237
+ "pass@32": 0.8718723024536624,
238
+ "pass@64": 0.8957240571987134,
239
+ "pass@128": 0.9147310989867498
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.8397478911042945,
244
+ "pass@2": 0.9017401031351142,
245
+ "pass@4": 0.9367013483935669,
246
+ "pass@8": 0.9597981710002264,
247
+ "pass@16": 0.9760430566524732,
248
+ "pass@32": 0.9868893358013864,
249
+ "pass@64": 0.9939010380542732,
250
+ "pass@128": 0.9969325153374233
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9402308130699089,
254
+ "pass@2": 0.9559446275997415,
255
+ "pass@4": 0.9671450020115263,
256
+ "pass@8": 0.9747791046336394,
257
+ "pass@16": 0.9798526018916596,
258
+ "pass@32": 0.982735766252235,
259
+ "pass@64": 0.9843437813422433,
260
+ "pass@128": 0.9848024316109423
261
+ },
262
+ "3": {
263
+ "pass@1": 0.98203125,
264
+ "pass@2": 0.9875126759723218,
265
+ "pass@4": 0.9907982465260023,
266
+ "pass@8": 0.9931905013110562,
267
+ "pass@16": 0.9947669019933483,
268
+ "pass@32": 0.996016072122554,
269
+ "pass@64": 0.9977332350126279,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9693047337278107,
274
+ "pass@2": 0.9827249947584216,
275
+ "pass@4": 0.9890199579342521,
276
+ "pass@8": 0.9924316167169746,
277
+ "pass@16": 0.9952105268225401,
278
+ "pass@32": 0.9977446704990522,
279
+ "pass@64": 0.9995571159589419,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.948052536231884,
284
+ "pass@2": 0.966407052379322,
285
+ "pass@4": 0.9765803837020368,
286
+ "pass@8": 0.9844381946054411,
287
+ "pass@16": 0.9908400253696799,
288
+ "pass@32": 0.9954288349662425,
289
+ "pass@64": 0.9982494068365969,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9469914040114613,
294
+ "pass@2": 0.9771242498251473,
295
+ "pass@4": 0.9882326420336424,
296
+ "pass@8": 0.991931583238474,
297
+ "pass@16": 0.9933659205077896,
298
+ "pass@32": 0.9940735913681014,
299
+ "pass@64": 0.9942652013230373,
300
+ "pass@128": 0.994269340974212
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9118722904624278,
304
+ "pass@2": 0.9468681728187156,
305
+ "pass@4": 0.9655806095047366,
306
+ "pass@8": 0.9770071107892204,
307
+ "pass@16": 0.9847477489910627,
308
+ "pass@32": 0.9900681636945455,
309
+ "pass@64": 0.9936873526856657,
310
+ "pass@128": 0.9971098265895953
311
+ },
312
+ "8": {
313
+ "pass@1": 0.8875,
314
+ "pass@2": 0.9402195159143837,
315
+ "pass@4": 0.9639932780409486,
316
+ "pass@8": 0.9756221536661233,
317
+ "pass@16": 0.9821909911786181,
318
+ "pass@32": 0.987287230265571,
319
+ "pass@64": 0.992438504371287,
320
+ "pass@128": 0.9971830985915493
321
+ },
322
+ "9": {
323
+ "pass@1": 0.8411302395209581,
324
+ "pass@2": 0.905336104366071,
325
+ "pass@4": 0.938773440464403,
326
+ "pass@8": 0.9598987045631255,
327
+ "pass@16": 0.9751816882329573,
328
+ "pass@32": 0.9858225193162448,
329
+ "pass@64": 0.9916186385525599,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.7506353539156626,
334
+ "pass@2": 0.8404700247841763,
335
+ "pass@4": 0.8961459129544045,
336
+ "pass@8": 0.9278766949955667,
337
+ "pass@16": 0.9467320736664068,
338
+ "pass@32": 0.9597594455810305,
339
+ "pass@64": 0.9681259167521782,
340
+ "pass@128": 0.9728915662650602
341
+ },
342
+ "12": {
343
+ "pass@1": 0.6611726097178683,
344
+ "pass@2": 0.7715347542764048,
345
+ "pass@4": 0.8428349935881834,
346
+ "pass@8": 0.8867303652307457,
347
+ "pass@16": 0.9159569131757227,
348
+ "pass@32": 0.9389731417905509,
349
+ "pass@64": 0.9577601020248315,
350
+ "pass@128": 0.9717868338557993
351
+ },
352
+ "13": {
353
+ "pass@1": 0.44421831232493,
354
+ "pass@2": 0.5660448234411872,
355
+ "pass@4": 0.66785195968151,
356
+ "pass@8": 0.7483170301361325,
357
+ "pass@16": 0.8105249949265751,
358
+ "pass@32": 0.8597446273776105,
359
+ "pass@64": 0.8981030316546639,
360
+ "pass@128": 0.9243697478991597
361
+ },
362
+ "14": {
363
+ "pass@1": 0.4415433114035088,
364
+ "pass@2": 0.5532334260026704,
365
+ "pass@4": 0.652206162826138,
366
+ "pass@8": 0.737487007203177,
367
+ "pass@16": 0.8066073874498777,
368
+ "pass@32": 0.8591202423066083,
369
+ "pass@64": 0.8958383932620966,
370
+ "pass@128": 0.9210526315789473
371
+ },
372
+ "15": {
373
+ "pass@1": 0.34607007575757576,
374
+ "pass@2": 0.4742386960152705,
375
+ "pass@4": 0.5880804396609509,
376
+ "pass@8": 0.6771553828886003,
377
+ "pass@16": 0.7422639165081902,
378
+ "pass@32": 0.7913001834008943,
379
+ "pass@64": 0.8312857431811411,
380
+ "pass@128": 0.8636363636363636
381
+ },
382
+ "16": {
383
+ "pass@1": 0.2948940527065527,
384
+ "pass@2": 0.4010665533795455,
385
+ "pass@4": 0.49861933924926044,
386
+ "pass@8": 0.5817780568854317,
387
+ "pass@16": 0.6520585361736072,
388
+ "pass@32": 0.7131258389048448,
389
+ "pass@64": 0.7662778139871184,
390
+ "pass@128": 0.811965811965812
391
+ },
392
+ "17": {
393
+ "pass@1": 0.2685364906832298,
394
+ "pass@2": 0.37610842972074143,
395
+ "pass@4": 0.48323010506838776,
396
+ "pass@8": 0.577938445005628,
397
+ "pass@16": 0.653921104286253,
398
+ "pass@32": 0.7138194380494706,
399
+ "pass@64": 0.7643783394471984,
400
+ "pass@128": 0.8136645962732919
401
+ },
402
+ "18": {
403
+ "pass@1": 0.24377297794117647,
404
+ "pass@2": 0.3515556247105141,
405
+ "pass@4": 0.46032718851320026,
406
+ "pass@8": 0.5538321588382297,
407
+ "pass@16": 0.6284314018506372,
408
+ "pass@32": 0.6875919900487388,
409
+ "pass@64": 0.7341714824652724,
410
+ "pass@128": 0.7705882352941177
411
+ },
412
+ "19": {
413
+ "pass@1": 0.1875,
414
+ "pass@2": 0.2628534958520808,
415
+ "pass@4": 0.34137497935525907,
416
+ "pass@8": 0.41886761741849504,
417
+ "pass@16": 0.4948485199164589,
418
+ "pass@32": 0.5684772948464838,
419
+ "pass@64": 0.6340076592011885,
420
+ "pass@128": 0.6845238095238095
421
+ },
422
+ "20": {
423
+ "pass@1": 0.16871725299401197,
424
+ "pass@2": 0.24689621740770412,
425
+ "pass@4": 0.3304425002638143,
426
+ "pass@8": 0.41089814433551575,
427
+ "pass@16": 0.483789637592383,
428
+ "pass@32": 0.5521726444199829,
429
+ "pass@64": 0.6178883001231452,
430
+ "pass@128": 0.6766467065868264
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 508121,
437
+ "answer_accuracy": 0.6342379473558076,
438
+ "avg_response_len": 299.6219768533312,
439
+ "resp_tokens_sum": 240042746,
440
+ "pass_at_k": {
441
+ "pass@1": 0.6342379473558076,
442
+ "pass@2": 0.703320859694323,
443
+ "pass@4": 0.7594703142135963,
444
+ "pass@8": 0.8046556382540034,
445
+ "pass@16": 0.8406575219448124,
446
+ "pass@32": 0.8698007858357516,
447
+ "pass@64": 0.8947172796227344,
448
+ "pass@128": 0.9159610161367631
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.8047108208955224,
453
+ "pass@2": 0.8782744740862615,
454
+ "pass@4": 0.9260115345656411,
455
+ "pass@8": 0.9548988993170184,
456
+ "pass@16": 0.9710010924150604,
457
+ "pass@32": 0.980370140618196,
458
+ "pass@64": 0.9877186605551607,
459
+ "pass@128": 0.9940298507462687
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9338392857142858,
463
+ "pass@2": 0.9515691788526434,
464
+ "pass@4": 0.9661372604317318,
465
+ "pass@8": 0.9767929304683264,
466
+ "pass@16": 0.9836826490884742,
467
+ "pass@32": 0.9881149987306532,
468
+ "pass@64": 0.9910662959611092,
469
+ "pass@128": 0.9942857142857143
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9736242711370262,
473
+ "pass@2": 0.9816482203117467,
474
+ "pass@4": 0.9857590412350061,
475
+ "pass@8": 0.9893075222068398,
476
+ "pass@16": 0.9925043063975238,
477
+ "pass@32": 0.9945585423291038,
478
+ "pass@64": 0.9956216974134651,
479
+ "pass@128": 0.9970845481049563
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9743695175438597,
483
+ "pass@2": 0.9826079511442651,
484
+ "pass@4": 0.9869292393823577,
485
+ "pass@8": 0.9900076968161828,
486
+ "pass@16": 0.9919605885870972,
487
+ "pass@32": 0.9932331313555678,
488
+ "pass@64": 0.9948887956026504,
489
+ "pass@128": 0.9970760233918129
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9108226102941176,
493
+ "pass@2": 0.9385729070171378,
494
+ "pass@4": 0.9575225440569924,
495
+ "pass@8": 0.9700715177204019,
496
+ "pass@16": 0.9782220232113423,
497
+ "pass@32": 0.9846600973625091,
498
+ "pass@64": 0.9896055060633964,
499
+ "pass@128": 0.9911764705882353
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9524102917981072,
503
+ "pass@2": 0.9782773727613706,
504
+ "pass@4": 0.9890783643172356,
505
+ "pass@8": 0.9932827474347146,
506
+ "pass@16": 0.994763417917439,
507
+ "pass@32": 0.9958640376966922,
508
+ "pass@64": 0.9976402791921362,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9219244462025317,
513
+ "pass@2": 0.9523934298066383,
514
+ "pass@4": 0.9664943151884493,
515
+ "pass@8": 0.973636516501745,
516
+ "pass@16": 0.9789543982992107,
517
+ "pass@32": 0.983129839071662,
518
+ "pass@64": 0.9865100126337769,
519
+ "pass@128": 0.990506329113924
520
+ },
521
+ "8": {
522
+ "pass@1": 0.8788006756756757,
523
+ "pass@2": 0.9290773302830391,
524
+ "pass@4": 0.9572869742633514,
525
+ "pass@8": 0.9754967922961485,
526
+ "pass@16": 0.9871544540195394,
527
+ "pass@32": 0.992969883203604,
528
+ "pass@64": 0.9953011942110427,
529
+ "pass@128": 0.996996996996997
530
+ },
531
+ "9": {
532
+ "pass@1": 0.8661324356913184,
533
+ "pass@2": 0.9265691856343519,
534
+ "pass@4": 0.9539672130011069,
535
+ "pass@8": 0.9695961662623275,
536
+ "pass@16": 0.9812411467601234,
537
+ "pass@32": 0.9901821005537559,
538
+ "pass@64": 0.9963129764254025,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.7656746031746032,
543
+ "pass@2": 0.8513830302462192,
544
+ "pass@4": 0.9025311645568108,
545
+ "pass@8": 0.9354963214647195,
546
+ "pass@16": 0.9582898489222895,
547
+ "pass@32": 0.974310122831304,
548
+ "pass@64": 0.9856107928071083,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.6314984709480123,
553
+ "pass@2": 0.7500733674059092,
554
+ "pass@4": 0.8329923644865482,
555
+ "pass@8": 0.887867580321604,
556
+ "pass@16": 0.923631780836543,
557
+ "pass@32": 0.946600124993897,
558
+ "pass@64": 0.9621206964233943,
559
+ "pass@128": 0.9755351681957186
560
+ },
561
+ "13": {
562
+ "pass@1": 0.4694974922839506,
563
+ "pass@2": 0.5959508967629046,
564
+ "pass@4": 0.6995019520129422,
565
+ "pass@8": 0.7776365616412141,
566
+ "pass@16": 0.8346225468513689,
567
+ "pass@32": 0.8761886815828364,
568
+ "pass@64": 0.9078665970466824,
569
+ "pass@128": 0.9290123456790124
570
+ },
571
+ "14": {
572
+ "pass@1": 0.42811109050445106,
573
+ "pass@2": 0.543148277413024,
574
+ "pass@4": 0.6509376348357051,
575
+ "pass@8": 0.7436043737795209,
576
+ "pass@16": 0.8145172014899712,
577
+ "pass@32": 0.8642210760480362,
578
+ "pass@64": 0.9006000656184018,
579
+ "pass@128": 0.9287833827893175
580
+ },
581
+ "15": {
582
+ "pass@1": 0.3851996527777778,
583
+ "pass@2": 0.5078033865558473,
584
+ "pass@4": 0.6108883527869196,
585
+ "pass@8": 0.6910061074300264,
586
+ "pass@16": 0.7513617520911909,
587
+ "pass@32": 0.7985095076620196,
588
+ "pass@64": 0.8389313592242905,
589
+ "pass@128": 0.8734567901234568
590
+ },
591
+ "16": {
592
+ "pass@1": 0.26636246312684364,
593
+ "pass@2": 0.36768960641534865,
594
+ "pass@4": 0.46721277124651445,
595
+ "pass@8": 0.5581714943434668,
596
+ "pass@16": 0.6403607605318539,
597
+ "pass@32": 0.7137040695583681,
598
+ "pass@64": 0.7760761955425775,
599
+ "pass@128": 0.8259587020648967
600
+ },
601
+ "17": {
602
+ "pass@1": 0.23943429969879518,
603
+ "pass@2": 0.33819468326060154,
604
+ "pass@4": 0.436030319534606,
605
+ "pass@8": 0.5232784476635474,
606
+ "pass@16": 0.5976341299105322,
607
+ "pass@32": 0.6617823167109694,
608
+ "pass@64": 0.7208114445140381,
609
+ "pass@128": 0.7771084337349398
610
+ },
611
+ "18": {
612
+ "pass@1": 0.24377520161290323,
613
+ "pass@2": 0.33654551371602726,
614
+ "pass@4": 0.4276333723405545,
615
+ "pass@8": 0.5083686482219439,
616
+ "pass@16": 0.5763213834382153,
617
+ "pass@32": 0.6355223760273493,
618
+ "pass@64": 0.6899469138467323,
619
+ "pass@128": 0.7387096774193549
620
+ },
621
+ "19": {
622
+ "pass@1": 0.19172488670694865,
623
+ "pass@2": 0.2705886332278707,
624
+ "pass@4": 0.35226705498670674,
625
+ "pass@8": 0.43071165135970435,
626
+ "pass@16": 0.5033697376743476,
627
+ "pass@32": 0.5710949315109166,
628
+ "pass@64": 0.6372663372318448,
629
+ "pass@128": 0.6978851963746223
630
+ },
631
+ "20": {
632
+ "pass@1": 0.19054992492492492,
633
+ "pass@2": 0.2684314481804639,
634
+ "pass@4": 0.35020230269639707,
635
+ "pass@8": 0.43047924091649253,
636
+ "pass@16": 0.5052137590570093,
637
+ "pass@32": 0.5743518955753166,
638
+ "pass@64": 0.639696972628713,
639
+ "pass@128": 0.6996996996996997
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dee1f707cf113068b69edb919f48014cf0f1b22e99ed2c0f70166c454a0f4302
3
+ size 1791146605
results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch1-op7-10",
3
+ "id": {
4
+ "answer_accuracy": 0.9702256944444444,
5
+ "avg_response_len": 199.40936458333334,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9702256944444444,
11
+ "pass@2": 0.9793916502624618,
12
+ "pass@4": 0.9849351835395576,
13
+ "pass@8": 0.9888442521480617,
14
+ "pass@16": 0.9918560073630215,
15
+ "pass@32": 0.9942361825582353,
16
+ "pass@64": 0.9961668747436009,
17
+ "pass@128": 0.9978888888888889
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9150703125,
22
+ "pass@2": 0.9398096702755907,
23
+ "pass@4": 0.9555400461192349,
24
+ "pass@8": 0.9668906410889534,
25
+ "pass@16": 0.9758948266659789,
26
+ "pass@32": 0.9832636557874399,
27
+ "pass@64": 0.9892967113568902,
28
+ "pass@128": 0.995
29
+ },
30
+ "2": {
31
+ "pass@1": 0.990578125,
32
+ "pass@2": 0.9930145177165355,
33
+ "pass@4": 0.9943968228346454,
34
+ "pass@8": 0.9952873162624573,
35
+ "pass@16": 0.9960729357688821,
36
+ "pass@32": 0.996952502557328,
37
+ "pass@64": 0.9979116198042409,
38
+ "pass@128": 0.999
39
+ },
40
+ "3": {
41
+ "pass@1": 0.99659375,
42
+ "pass@2": 0.9988639271653545,
43
+ "pass@4": 0.9997500349643794,
44
+ "pass@8": 0.9999798240813113,
45
+ "pass@16": 0.9999998412707761,
46
+ "pass@32": 0.9999999999968697,
47
+ "pass@64": 1.0,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9929296875,
52
+ "pass@2": 0.9952704232283465,
53
+ "pass@4": 0.9970510909261342,
54
+ "pass@8": 0.9984842712829364,
55
+ "pass@16": 0.9994335154423146,
56
+ "pass@32": 0.999865478267178,
57
+ "pass@64": 0.9999934228152456,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.962078125,
62
+ "pass@2": 0.9698677411417324,
63
+ "pass@4": 0.9763977542182228,
64
+ "pass@8": 0.9819934096902104,
65
+ "pass@16": 0.9865521659361441,
66
+ "pass@32": 0.9903191098925549,
67
+ "pass@64": 0.9939691902252709,
68
+ "pass@128": 0.998
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9701875,
72
+ "pass@2": 0.9778256643700791,
73
+ "pass@4": 0.9825182044431944,
74
+ "pass@8": 0.9861330184093109,
75
+ "pass@16": 0.9892783904066073,
76
+ "pass@32": 0.9920173554056341,
77
+ "pass@64": 0.9943151204054765,
78
+ "pass@128": 0.996
79
+ },
80
+ "7": {
81
+ "pass@1": 0.97875,
82
+ "pass@2": 0.9858042568897639,
83
+ "pass@4": 0.9899331335770527,
84
+ "pass@8": 0.9927339385848097,
85
+ "pass@16": 0.9948275081708874,
86
+ "pass@32": 0.9963086569131329,
87
+ "pass@64": 0.9973199896660145,
88
+ "pass@128": 0.998
89
+ },
90
+ "8": {
91
+ "pass@1": 0.964484375,
92
+ "pass@2": 0.977889886811024,
93
+ "pass@4": 0.9856384359767532,
94
+ "pass@8": 0.9905634974996607,
95
+ "pass@16": 0.9937330000695483,
96
+ "pass@32": 0.9958854298336186,
97
+ "pass@64": 0.9975687158504051,
98
+ "pass@128": 0.999
99
+ },
100
+ "9": {
101
+ "pass@1": 0.961359375,
102
+ "pass@2": 0.97617876476378,
103
+ "pass@4": 0.9831911287964004,
104
+ "pass@8": 0.9875323524329049,
105
+ "pass@16": 0.9909118825360496,
106
+ "pass@32": 0.9935134543703622,
107
+ "pass@64": 0.9951271025688606,
108
+ "pass@128": 0.996
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9150703125,
113
+ "2": 0.990578125,
114
+ "3": 0.99659375,
115
+ "4": 0.9929296875,
116
+ "5": 0.962078125,
117
+ "6": 0.9701875,
118
+ "7": 0.97875,
119
+ "8": 0.964484375,
120
+ "9": 0.961359375
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 278.345546875,
124
+ "2": 116.3494296875,
125
+ "3": 144.50065625,
126
+ "4": 159.5111484375,
127
+ "5": 173.8181953125,
128
+ "6": 198.9171640625,
129
+ "7": 223.07409375,
130
+ "8": 242.34471875,
131
+ "9": 257.823328125
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 278.345546875,
136
+ "2": 116.3494296875,
137
+ "3": 144.50065625,
138
+ "4": 159.5111484375,
139
+ "5": 173.8181953125,
140
+ "6": 198.9171640625,
141
+ "7": 223.07409375,
142
+ "8": 242.34471875,
143
+ "9": 257.823328125
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 366904,
150
+ "answer_accuracy": 0.9680639986491051,
151
+ "avg_response_len": 189.9049070204323,
152
+ "resp_tokens_sum": 71975479,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9680639986491051,
155
+ "pass@2": 0.9787410217339862,
156
+ "pass@4": 0.9848598221701518,
157
+ "pass@8": 0.9887697662104195,
158
+ "pass@16": 0.9918066653339469,
159
+ "pass@32": 0.9943563851377097,
160
+ "pass@64": 0.9963836343066652,
161
+ "pass@128": 0.9979736575481256
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9222437315634219,
166
+ "pass@2": 0.9451468248437975,
167
+ "pass@4": 0.9604848572578869,
168
+ "pass@8": 0.9712668741187709,
169
+ "pass@16": 0.9797398811861905,
170
+ "pass@32": 0.986931871980946,
171
+ "pass@64": 0.9931403532607942,
172
+ "pass@128": 1.0
173
+ },
174
+ "2": {
175
+ "pass@1": 0.989242601246106,
176
+ "pass@2": 0.9923153224421711,
177
+ "pass@4": 0.9941436283665476,
178
+ "pass@8": 0.9953145921083346,
179
+ "pass@16": 0.9966062996683891,
180
+ "pass@32": 0.9983101878563488,
181
+ "pass@64": 0.9997246792749038,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9975630733944955,
186
+ "pass@2": 0.9996982530280046,
187
+ "pass@4": 0.9999930091880717,
188
+ "pass@8": 0.9999999967989345,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.992236328125,
196
+ "pass@2": 0.9945377860482283,
197
+ "pass@4": 0.9961152392083802,
198
+ "pass@8": 0.9975110009296438,
199
+ "pass@16": 0.9987271357419916,
200
+ "pass@32": 0.9996050186206877,
201
+ "pass@64": 0.99997946273747,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9551587301587302,
206
+ "pass@2": 0.963551431071116,
207
+ "pass@4": 0.9705333979085945,
208
+ "pass@8": 0.9768221559126241,
209
+ "pass@16": 0.9826882032901565,
210
+ "pass@32": 0.988084362473456,
211
+ "pass@64": 0.992762368619646,
212
+ "pass@128": 0.9968253968253968
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9563061377245509,
216
+ "pass@2": 0.9692498614974779,
217
+ "pass@4": 0.9767356600073196,
218
+ "pass@8": 0.9813628882947097,
219
+ "pass@16": 0.9850560727092349,
220
+ "pass@32": 0.9883559011032365,
221
+ "pass@64": 0.991386843357507,
222
+ "pass@128": 0.9940119760479041
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9775563979289941,
226
+ "pass@2": 0.9875821908633461,
227
+ "pass@4": 0.9925620808123836,
228
+ "pass@8": 0.9950344430852842,
229
+ "pass@16": 0.9964798475945668,
230
+ "pass@32": 0.997005436160734,
231
+ "pass@64": 0.9970413979979968,
232
+ "pass@128": 0.9970414201183432
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9700270432692307,
236
+ "pass@2": 0.9840835888602868,
237
+ "pass@4": 0.9896981008383567,
238
+ "pass@8": 0.9919003031750214,
239
+ "pass@16": 0.9935285960287868,
240
+ "pass@32": 0.9953138505545496,
241
+ "pass@64": 0.9966101352715523,
242
+ "pass@128": 0.9967948717948718
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9554577464788733,
246
+ "pass@2": 0.9742978540534544,
247
+ "pass@4": 0.984426338080979,
248
+ "pass@8": 0.9901960111237609,
249
+ "pass@16": 0.9936839096661296,
250
+ "pass@32": 0.9957886523598386,
251
+ "pass@64": 0.9969763514894339,
252
+ "pass@128": 0.9971830985915493
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 380695,
259
+ "answer_accuracy": 0.9745018635321101,
260
+ "avg_response_len": 197.31111770969855,
261
+ "resp_tokens_sum": 77080772,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9745018635321101,
264
+ "pass@2": 0.9822816908494244,
265
+ "pass@4": 0.986926477581931,
266
+ "pass@8": 0.9903302991720578,
267
+ "pass@16": 0.9929998545725408,
268
+ "pass@32": 0.9951115215182154,
269
+ "pass@64": 0.9968114142410952,
270
+ "pass@128": 0.9983617300131061
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9157400306748467,
275
+ "pass@2": 0.9375864239650259,
276
+ "pass@4": 0.9503189637721664,
277
+ "pass@8": 0.959476709600869,
278
+ "pass@16": 0.9678207783989702,
279
+ "pass@32": 0.976828113172386,
280
+ "pass@64": 0.9855899073216104,
281
+ "pass@128": 0.9938650306748467
282
+ },
283
+ "2": {
284
+ "pass@1": 0.9930661094224924,
285
+ "pass@2": 0.9945559497881914,
286
+ "pass@4": 0.9952527896398966,
287
+ "pass@8": 0.9960218094796784,
288
+ "pass@16": 0.996694876664963,
289
+ "pass@32": 0.9969450828883747,
290
+ "pass@64": 0.9969604795045498,
291
+ "pass@128": 0.9969604863221885
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9994791666666667,
295
+ "pass@2": 0.9999657003101885,
296
+ "pass@4": 0.9999997156605425,
297
+ "pass@8": 0.9999999999936351,
298
+ "pass@16": 1.0,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9902921597633136,
305
+ "pass@2": 0.9935943542375252,
306
+ "pass@4": 0.996376267685474,
307
+ "pass@8": 0.9984966632337737,
308
+ "pass@16": 0.9996459679388677,
309
+ "pass@32": 0.9999785901153704,
310
+ "pass@64": 0.9999999845150909,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9790760869565217,
315
+ "pass@2": 0.9860735478717333,
316
+ "pass@4": 0.990809667269852,
317
+ "pass@8": 0.9943024238372346,
318
+ "pass@16": 0.9967581848072867,
319
+ "pass@32": 0.9982186985410357,
320
+ "pass@64": 0.9992791299089842,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9788905802292264,
325
+ "pass@2": 0.9836452716422623,
326
+ "pass@4": 0.9860912927610409,
327
+ "pass@8": 0.9885082645987024,
328
+ "pass@16": 0.9914576644920206,
329
+ "pass@32": 0.994320594090395,
330
+ "pass@64": 0.9964244742548366,
331
+ "pass@128": 0.997134670487106
332
+ },
333
+ "7": {
334
+ "pass@1": 0.979091401734104,
335
+ "pass@2": 0.9841676954849575,
336
+ "pass@4": 0.9884633377655827,
337
+ "pass@8": 0.9922804183734506,
338
+ "pass@16": 0.9951181931156187,
339
+ "pass@32": 0.9970528714118317,
340
+ "pass@64": 0.9984707380171003,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9671654929577465,
345
+ "pass@2": 0.9811702894532549,
346
+ "pass@4": 0.9894243719535057,
347
+ "pass@8": 0.9944174851855382,
348
+ "pass@16": 0.9969106947639146,
349
+ "pass@32": 0.9978444559613356,
350
+ "pass@64": 0.9985914715905386,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9666448353293413,
355
+ "pass@2": 0.9787814872224057,
356
+ "pass@4": 0.984590454074977,
357
+ "pass@8": 0.9884570828941244,
358
+ "pass@16": 0.9916722775023806,
359
+ "pass@32": 0.9940966746741055,
360
+ "pass@64": 0.9954845654245074,
361
+ "pass@128": 0.9970059880239521
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 370101,
368
+ "answer_accuracy": 0.9679993513558754,
369
+ "avg_response_len": 210.97499843069969,
370
+ "resp_tokens_sum": 80663337,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9679993513558754,
373
+ "pass@2": 0.9770836848126664,
374
+ "pass@4": 0.9829752624138431,
375
+ "pass@8": 0.9873997048913209,
376
+ "pass@16": 0.9907361814723746,
377
+ "pass@32": 0.9932226390886398,
378
+ "pass@64": 0.9952934366409588,
379
+ "pass@128": 0.9973217274857716
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9071595149253732,
384
+ "pass@2": 0.9365723057938655,
385
+ "pass@4": 0.9556170069226421,
386
+ "pass@8": 0.9696769057695721,
387
+ "pass@16": 0.9798609945844656,
388
+ "pass@32": 0.9858142874379154,
389
+ "pass@64": 0.9890143934765259,
390
+ "pass@128": 0.991044776119403
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9894642857142857,
394
+ "pass@2": 0.99220683352081,
395
+ "pass@4": 0.9938244295355938,
396
+ "pass@8": 0.9945718767910509,
397
+ "pass@16": 0.9949991404358748,
398
+ "pass@32": 0.9957142856718992,
399
+ "pass@64": 0.9971428571428571,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9928935860058309,
404
+ "pass@2": 0.9970085053143868,
405
+ "pass@4": 0.9992781772358628,
406
+ "pass@8": 0.9999411811374931,
407
+ "pass@16": 0.9999995372325835,
408
+ "pass@32": 0.9999999999908739,
409
+ "pass@64": 1.0,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9961851242690059,
414
+ "pass@2": 0.9976123975457015,
415
+ "pass@4": 0.998593672227375,
416
+ "pass@8": 0.9993826865860662,
417
+ "pass@16": 0.99988448784076,
418
+ "pass@32": 0.9999973942969671,
419
+ "pass@64": 0.9999999999215047,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9512408088235295,
424
+ "pass@2": 0.9592754892311257,
425
+ "pass@4": 0.9672070549085778,
426
+ "pass@8": 0.9742943951584943,
427
+ "pass@16": 0.9797759062977376,
428
+ "pass@32": 0.9843737785787031,
429
+ "pass@64": 0.9896992479161258,
430
+ "pass@128": 0.9970588235294118
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9752316640378549,
434
+ "pass@2": 0.9804542928289322,
435
+ "pass@4": 0.9846770751645001,
436
+ "pass@8": 0.9885439412426841,
437
+ "pass@16": 0.9913278776467133,
438
+ "pass@32": 0.9933394230271457,
439
+ "pass@64": 0.9950781489247041,
440
+ "pass@128": 0.9968454258675079
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9796528876582279,
444
+ "pass@2": 0.985694473861258,
445
+ "pass@4": 0.9887304917581505,
446
+ "pass@8": 0.9907698483062334,
447
+ "pass@16": 0.9927418508415174,
448
+ "pass@32": 0.994748499944022,
449
+ "pass@64": 0.996357980344224,
450
+ "pass@128": 0.9968354430379747
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9564329954954955,
454
+ "pass@2": 0.9685896526447708,
455
+ "pass@4": 0.9777987281432166,
456
+ "pass@8": 0.9852023893939588,
457
+ "pass@16": 0.9905368812834143,
458
+ "pass@32": 0.9943325122952712,
459
+ "pass@64": 0.9973765202133318,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.96241961414791,
464
+ "pass@2": 0.9755305782717674,
465
+ "pass@4": 0.9802783508572683,
466
+ "pass@8": 0.9834987228210043,
467
+ "pass@16": 0.9869310415394815,
468
+ "pass@32": 0.9902900110658145,
469
+ "pass@64": 0.9926323245605343,
470
+ "pass@128": 0.9935691318327974
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.35165546875,
478
+ "avg_response_len": 320.09001796875,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.35165546875,
484
+ "pass@2": 0.4197094857283516,
485
+ "pass@4": 0.4786157665635541,
486
+ "pass@8": 0.5292358452246045,
487
+ "pass@16": 0.5726114897631275,
488
+ "pass@32": 0.609898000220599,
489
+ "pass@64": 0.6423396090115875,
490
+ "pass@128": 0.6712
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.8648359375,
495
+ "pass@2": 0.9100951033464573,
496
+ "pass@4": 0.9347501322647173,
497
+ "pass@8": 0.9492648414627204,
498
+ "pass@16": 0.9591873478095747,
499
+ "pass@32": 0.9663478688104976,
500
+ "pass@64": 0.9718211052876301,
501
+ "pass@128": 0.976
502
+ },
503
+ "12": {
504
+ "pass@1": 0.7038046875,
505
+ "pass@2": 0.7893309547244096,
506
+ "pass@4": 0.847530710161229,
507
+ "pass@8": 0.8872501406974933,
508
+ "pass@16": 0.913835808415057,
509
+ "pass@32": 0.9321118322142374,
510
+ "pass@64": 0.9463118241321323,
511
+ "pass@128": 0.957
512
+ },
513
+ "13": {
514
+ "pass@1": 0.4401484375,
515
+ "pass@2": 0.5271272145669298,
516
+ "pass@4": 0.6008414419760031,
517
+ "pass@8": 0.663756701110009,
518
+ "pass@16": 0.716035068325726,
519
+ "pass@32": 0.7594926715885558,
520
+ "pass@64": 0.796119320741206,
521
+ "pass@128": 0.829
522
+ },
523
+ "14": {
524
+ "pass@1": 0.3608828125,
525
+ "pass@2": 0.4397880167322835,
526
+ "pass@4": 0.508594947975253,
527
+ "pass@8": 0.5686336976073794,
528
+ "pass@16": 0.6218801756842585,
529
+ "pass@32": 0.6686538789478669,
530
+ "pass@64": 0.7076235209573943,
531
+ "pass@128": 0.741
532
+ },
533
+ "15": {
534
+ "pass@1": 0.2531953125,
535
+ "pass@2": 0.32912573818897634,
536
+ "pass@4": 0.40056516357330324,
537
+ "pass@8": 0.4651769969975074,
538
+ "pass@16": 0.5209395590458854,
539
+ "pass@32": 0.5685441656287817,
540
+ "pass@64": 0.6091709442131789,
541
+ "pass@128": 0.644
542
+ },
543
+ "16": {
544
+ "pass@1": 0.21353125,
545
+ "pass@2": 0.277843626968504,
546
+ "pass@4": 0.3358475869891258,
547
+ "pass@8": 0.38845783283534113,
548
+ "pass@16": 0.4373589747108126,
549
+ "pass@32": 0.481591610081772,
550
+ "pass@64": 0.5213278775966319,
551
+ "pass@128": 0.557
552
+ },
553
+ "17": {
554
+ "pass@1": 0.1928125,
555
+ "pass@2": 0.25692937992125997,
556
+ "pass@4": 0.3162516365766777,
557
+ "pass@8": 0.3696114083029441,
558
+ "pass@16": 0.4174144617061861,
559
+ "pass@32": 0.46007529401338054,
560
+ "pass@64": 0.5000393438805595,
561
+ "pass@128": 0.543
562
+ },
563
+ "18": {
564
+ "pass@1": 0.17340625,
565
+ "pass@2": 0.23288939468503944,
566
+ "pass@4": 0.2924173040869889,
567
+ "pass@8": 0.34748396702279544,
568
+ "pass@16": 0.39519400799025706,
569
+ "pass@32": 0.4368171730593396,
570
+ "pass@64": 0.47577441414294835,
571
+ "pass@128": 0.512
572
+ },
573
+ "19": {
574
+ "pass@1": 0.152296875,
575
+ "pass@2": 0.21116178641732283,
576
+ "pass@4": 0.26833084833145837,
577
+ "pass@8": 0.32048557651113246,
578
+ "pass@16": 0.3667058127901986,
579
+ "pass@32": 0.4060712287023584,
580
+ "pass@64": 0.43877021147199496,
581
+ "pass@128": 0.467
582
+ },
583
+ "20": {
584
+ "pass@1": 0.161640625,
585
+ "pass@2": 0.22280364173228356,
586
+ "pass@4": 0.2810278937007869,
587
+ "pass@8": 0.3322372896986801,
588
+ "pass@16": 0.37756368115318545,
589
+ "pass@32": 0.41927427915921184,
590
+ "pass@64": 0.45643752769216683,
591
+ "pass@128": 0.486
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.8648359375,
596
+ "12": 0.7038046875,
597
+ "13": 0.4401484375,
598
+ "14": 0.3608828125,
599
+ "15": 0.2531953125,
600
+ "16": 0.21353125,
601
+ "17": 0.1928125,
602
+ "18": 0.17340625,
603
+ "19": 0.152296875,
604
+ "20": 0.161640625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 294.4192421875,
608
+ "12": 301.11725,
609
+ "13": 310.0360859375,
610
+ "14": 320.811046875,
611
+ "15": 323.3763125,
612
+ "16": 327.6389453125,
613
+ "17": 333.5490078125,
614
+ "18": 329.8962734375,
615
+ "19": 329.6489375,
616
+ "20": 330.407078125
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 294.4192421875,
621
+ "12": 301.11725,
622
+ "13": 310.0360859375,
623
+ "14": 320.811046875,
624
+ "15": 323.3763125,
625
+ "16": 327.6389453125,
626
+ "17": 333.5490078125,
627
+ "18": 329.8962734375,
628
+ "19": 329.6489375,
629
+ "20": 330.407078125
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 152549,
636
+ "answer_accuracy": 0.3541720839524517,
637
+ "avg_response_len": 303.4558088781575,
638
+ "resp_tokens_sum": 130704486,
639
+ "pass_at_k": {
640
+ "pass@1": 0.3541720839524517,
641
+ "pass@2": 0.42192966035263496,
642
+ "pass@4": 0.4798859520160286,
643
+ "pass@8": 0.5286846055548091,
644
+ "pass@16": 0.5698062594651296,
645
+ "pass@32": 0.6048244550283293,
646
+ "pass@64": 0.6348565285474133,
647
+ "pass@128": 0.6612184249628529
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.8513411827195467,
652
+ "pass@2": 0.9002692054605079,
653
+ "pass@4": 0.9301656318278912,
654
+ "pass@8": 0.9489189290719003,
655
+ "pass@16": 0.9620749038932601,
656
+ "pass@32": 0.9706335966755882,
657
+ "pass@64": 0.9751169549682261,
658
+ "pass@128": 0.9773371104815864
659
+ },
660
+ "12": {
661
+ "pass@1": 0.7018449858757062,
662
+ "pass@2": 0.786281779661017,
663
+ "pass@4": 0.8425637121101385,
664
+ "pass@8": 0.8814481070980112,
665
+ "pass@16": 0.9089087677573177,
666
+ "pass@32": 0.9300411469087887,
667
+ "pass@64": 0.9476799728882689,
668
+ "pass@128": 0.96045197740113
669
+ },
670
+ "13": {
671
+ "pass@1": 0.46071708463949845,
672
+ "pass@2": 0.5479957822180537,
673
+ "pass@4": 0.6168102769481397,
674
+ "pass@8": 0.6711818283871478,
675
+ "pass@16": 0.714081813122014,
676
+ "pass@32": 0.7497040752868626,
677
+ "pass@64": 0.7797252476653084,
678
+ "pass@128": 0.8056426332288401
679
+ },
680
+ "14": {
681
+ "pass@1": 0.3388093847352025,
682
+ "pass@2": 0.41182321178894693,
683
+ "pass@4": 0.4764041463275034,
684
+ "pass@8": 0.534130178473544,
685
+ "pass@16": 0.5871949465142853,
686
+ "pass@32": 0.6363122142046183,
687
+ "pass@64": 0.6797314428750697,
688
+ "pass@128": 0.7165109034267912
689
+ },
690
+ "15": {
691
+ "pass@1": 0.26275740606936415,
692
+ "pass@2": 0.3398819751035454,
693
+ "pass@4": 0.41254214727855526,
694
+ "pass@8": 0.4781416760702823,
695
+ "pass@16": 0.5331817011339766,
696
+ "pass@32": 0.5794008648263183,
697
+ "pass@64": 0.6195336061079341,
698
+ "pass@128": 0.6560693641618497
699
+ },
700
+ "16": {
701
+ "pass@1": 0.19909274193548387,
702
+ "pass@2": 0.26266430657861306,
703
+ "pass@4": 0.3225466130241786,
704
+ "pass@8": 0.37652892333302973,
705
+ "pass@16": 0.4252326950906888,
706
+ "pass@32": 0.46738340757399127,
707
+ "pass@64": 0.505813596566932,
708
+ "pass@128": 0.5451612903225806
709
+ },
710
+ "17": {
711
+ "pass@1": 0.18987084537572255,
712
+ "pass@2": 0.25301889422420454,
713
+ "pass@4": 0.30949784781237605,
714
+ "pass@8": 0.35793822819914667,
715
+ "pass@16": 0.39987978151086667,
716
+ "pass@32": 0.4364144577614477,
717
+ "pass@64": 0.46841234177173346,
718
+ "pass@128": 0.5028901734104047
719
+ },
720
+ "18": {
721
+ "pass@1": 0.16642857142857143,
722
+ "pass@2": 0.22563835770528673,
723
+ "pass@4": 0.2839633877015374,
724
+ "pass@8": 0.3352756128571621,
725
+ "pass@16": 0.3794013307508793,
726
+ "pass@32": 0.41912972908665974,
727
+ "pass@64": 0.45480493264252136,
728
+ "pass@128": 0.4857142857142857
729
+ },
730
+ "19": {
731
+ "pass@1": 0.15287162162162163,
732
+ "pass@2": 0.21274547184980255,
733
+ "pass@4": 0.2720780853969829,
734
+ "pass@8": 0.3269558100570385,
735
+ "pass@16": 0.3740083610670051,
736
+ "pass@32": 0.41088179701472477,
737
+ "pass@64": 0.44192531549939784,
738
+ "pass@128": 0.47147147147147145
739
+ },
740
+ "20": {
741
+ "pass@1": 0.17898367117117117,
742
+ "pass@2": 0.24102498167458797,
743
+ "pass@4": 0.2964189808481148,
744
+ "pass@8": 0.34303163036602097,
745
+ "pass@16": 0.38364439504923087,
746
+ "pass@32": 0.4206433730102397,
747
+ "pass@64": 0.4507408603585201,
748
+ "pass@128": 0.46846846846846846
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 149088,
755
+ "answer_accuracy": 0.34634255129348795,
756
+ "avg_response_len": 315.6940580396967,
757
+ "resp_tokens_sum": 135894927,
758
+ "pass_at_k": {
759
+ "pass@1": 0.34634255129348795,
760
+ "pass@2": 0.4140275441289073,
761
+ "pass@4": 0.4736084765720087,
762
+ "pass@8": 0.5251538742682398,
763
+ "pass@16": 0.5689560116969068,
764
+ "pass@32": 0.6061521589618866,
765
+ "pass@64": 0.6385964343815986,
766
+ "pass@128": 0.6687481415402914
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.867211031626506,
771
+ "pass@2": 0.911044522578503,
772
+ "pass@4": 0.9335228710116054,
773
+ "pass@8": 0.9456478076783712,
774
+ "pass@16": 0.9524671555395998,
775
+ "pass@32": 0.9570610201613988,
776
+ "pass@64": 0.9621282660404785,
777
+ "pass@128": 0.9698795180722891
778
+ },
779
+ "12": {
780
+ "pass@1": 0.71608052507837,
781
+ "pass@2": 0.7989217967072296,
782
+ "pass@4": 0.8551101025185328,
783
+ "pass@8": 0.8918423946053244,
784
+ "pass@16": 0.9147090576639251,
785
+ "pass@32": 0.9288851299053897,
786
+ "pass@64": 0.9409932385294371,
787
+ "pass@128": 0.9529780564263323
788
+ },
789
+ "13": {
790
+ "pass@1": 0.40434611344537813,
791
+ "pass@2": 0.48740185050398094,
792
+ "pass@4": 0.5640730444408734,
793
+ "pass@8": 0.6355232978685984,
794
+ "pass@16": 0.6996316769009948,
795
+ "pass@32": 0.7527484934731532,
796
+ "pass@64": 0.7943710656136685,
797
+ "pass@128": 0.8291316526610645
798
+ },
799
+ "14": {
800
+ "pass@1": 0.3604486476608187,
801
+ "pass@2": 0.44008114322880676,
802
+ "pass@4": 0.5081644520312152,
803
+ "pass@8": 0.5657154899960264,
804
+ "pass@16": 0.6164087304791761,
805
+ "pass@32": 0.6613545821335254,
806
+ "pass@64": 0.6976279435761811,
807
+ "pass@128": 0.7309941520467836
808
+ },
809
+ "15": {
810
+ "pass@1": 0.22407670454545456,
811
+ "pass@2": 0.30122323132903833,
812
+ "pass@4": 0.37661032541386885,
813
+ "pass@8": 0.44410643751587464,
814
+ "pass@16": 0.4993273457645693,
815
+ "pass@32": 0.5427207028934755,
816
+ "pass@64": 0.5788425426868631,
817
+ "pass@128": 0.6121212121212121
818
+ },
819
+ "16": {
820
+ "pass@1": 0.23039084757834757,
821
+ "pass@2": 0.2978291074993831,
822
+ "pass@4": 0.35624147569160675,
823
+ "pass@8": 0.40731401792660654,
824
+ "pass@16": 0.45254311760056604,
825
+ "pass@32": 0.49147289445246195,
826
+ "pass@64": 0.5235553530937181,
827
+ "pass@128": 0.5470085470085471
828
+ },
829
+ "17": {
830
+ "pass@1": 0.21370341614906832,
831
+ "pass@2": 0.2806520363623024,
832
+ "pass@4": 0.3440671235971277,
833
+ "pass@8": 0.40228302586205095,
834
+ "pass@16": 0.4537379226594085,
835
+ "pass@32": 0.500026224710816,
836
+ "pass@64": 0.5454292278608023,
837
+ "pass@128": 0.593167701863354
838
+ },
839
+ "18": {
840
+ "pass@1": 0.17212775735294117,
841
+ "pass@2": 0.23196720125057887,
842
+ "pass@4": 0.2933669291338581,
843
+ "pass@8": 0.35235556676313684,
844
+ "pass@16": 0.4034011348916748,
845
+ "pass@32": 0.4468507861160033,
846
+ "pass@64": 0.48950582673115856,
847
+ "pass@128": 0.5352941176470588
848
+ },
849
+ "19": {
850
+ "pass@1": 0.15087890625,
851
+ "pass@2": 0.2072612017247844,
852
+ "pass@4": 0.2605428718731588,
853
+ "pass@8": 0.3074317250275432,
854
+ "pass@16": 0.34908958061974404,
855
+ "pass@32": 0.3879378881426714,
856
+ "pass@64": 0.42367879660082997,
857
+ "pass@128": 0.45535714285714285
858
+ },
859
+ "20": {
860
+ "pass@1": 0.1435254491017964,
861
+ "pass@2": 0.20275848401621951,
862
+ "pass@4": 0.26188721965517786,
863
+ "pass@8": 0.31475370332114117,
864
+ "pass@16": 0.3611758484951675,
865
+ "pass@32": 0.40307471926923255,
866
+ "pass@64": 0.43896682078335747,
867
+ "pass@128": 0.47005988023952094
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 148482,
874
+ "answer_accuracy": 0.35452800275061125,
875
+ "avg_response_len": 341.7152401054401,
876
+ "resp_tokens_sum": 143115810,
877
+ "pass_at_k": {
878
+ "pass@1": 0.35452800275061125,
879
+ "pass@2": 0.42326617337754974,
880
+ "pass@4": 0.4824560300702804,
881
+ "pass@8": 0.5339982504248119,
882
+ "pass@16": 0.5792535963307709,
883
+ "pass@32": 0.6189657703077097,
884
+ "pass@64": 0.6538826597580932,
885
+ "pass@128": 0.6839853300733496
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.8774553571428572,
890
+ "pass@2": 0.9201056899137611,
891
+ "pass@4": 0.9411811779479943,
892
+ "pass@8": 0.9534647217495896,
893
+ "pass@16": 0.9630343209400213,
894
+ "pass@32": 0.9713331761600971,
895
+ "pass@64": 0.9783436058997058,
896
+ "pass@128": 0.9809523809523809
897
+ },
898
+ "12": {
899
+ "pass@1": 0.6939506880733946,
900
+ "pass@2": 0.7832756928893064,
901
+ "pass@4": 0.8455138635193533,
902
+ "pass@8": 0.8890513361030553,
903
+ "pass@16": 0.9183177835907471,
904
+ "pass@32": 0.9375012531153086,
905
+ "pass@64": 0.9500191762654265,
906
+ "pass@128": 0.9571865443425076
907
+ },
908
+ "13": {
909
+ "pass@1": 0.4593460648148148,
910
+ "pass@2": 0.550352158671138,
911
+ "pass@4": 0.6256323665560319,
912
+ "pass@8": 0.6875551867759847,
913
+ "pass@16": 0.7360322879202137,
914
+ "pass@32": 0.7765612944510213,
915
+ "pass@64": 0.8141867170120144,
916
+ "pass@128": 0.8518518518518519
917
+ },
918
+ "14": {
919
+ "pass@1": 0.38234885014836795,
920
+ "pass@2": 0.46612764024860376,
921
+ "pass@4": 0.5396942860814502,
922
+ "pass@8": 0.6044605719250148,
923
+ "pass@16": 0.6604712523124485,
924
+ "pass@32": 0.7068676293724594,
925
+ "pass@64": 0.7443352257906309,
926
+ "pass@128": 0.7744807121661721
927
+ },
928
+ "15": {
929
+ "pass@1": 0.2726417824074074,
930
+ "pass@2": 0.3460583594585398,
931
+ "pass@4": 0.41217337416156297,
932
+ "pass@8": 0.4727927552375029,
933
+ "pass@16": 0.5298785998494511,
934
+ "pass@32": 0.5832519581605827,
935
+ "pass@64": 0.6289946833739761,
936
+ "pass@128": 0.6635802469135802
937
+ },
938
+ "16": {
939
+ "pass@1": 0.20927820796460178,
940
+ "pass@2": 0.2710314902561958,
941
+ "pass@4": 0.32689492325627434,
942
+ "pass@8": 0.37984261448337203,
943
+ "pass@16": 0.43272626830354155,
944
+ "pass@32": 0.48435329728914633,
945
+ "pass@64": 0.5332086540560117,
946
+ "pass@128": 0.5781710914454278
947
+ },
948
+ "17": {
949
+ "pass@1": 0.17561652861445784,
950
+ "pass@2": 0.2379966470211554,
951
+ "pass@4": 0.29631255251602584,
952
+ "pass@8": 0.3500892982483715,
953
+ "pass@16": 0.4004591753225813,
954
+ "pass@32": 0.44598628696095505,
955
+ "pass@64": 0.4889772959528953,
956
+ "pass@128": 0.536144578313253
957
+ },
958
+ "18": {
959
+ "pass@1": 0.18268649193548386,
960
+ "pass@2": 0.24208748729997448,
961
+ "pass@4": 0.3009205241481911,
962
+ "pass@8": 0.35592454781716937,
963
+ "pass@16": 0.40402308504606516,
964
+ "pass@32": 0.44578225999860455,
965
+ "pass@64": 0.4843893762241014,
966
+ "pass@128": 0.5161290322580645
967
+ },
968
+ "19": {
969
+ "pass@1": 0.15315804380664652,
970
+ "pass@2": 0.21352803780003343,
971
+ "pass@4": 0.27246658895734704,
972
+ "pass@8": 0.32722728747094926,
973
+ "pass@16": 0.3772414787512142,
974
+ "pass@32": 0.41963885160277065,
975
+ "pass@64": 0.4509154252350958,
976
+ "pass@128": 0.4743202416918429
977
+ },
978
+ "20": {
979
+ "pass@1": 0.16246715465465467,
980
+ "pass@2": 0.22468765517485983,
981
+ "pass@4": 0.284834960100708,
982
+ "pass@8": 0.33897903867127327,
983
+ "pass@16": 0.3879200126258433,
984
+ "pass@32": 0.4341533925251601,
985
+ "pass@64": 0.4796573665199342,
986
+ "pass@128": 0.5195195195195195
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.6446624177631579,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.6446624177631579,
1002
+ "pass@2": 0.6848220899813319,
1003
+ "pass@4": 0.7184512798679576,
1004
+ "pass@8": 0.746945090609406,
1005
+ "pass@16": 0.7712009980999249,
1006
+ "pass@32": 0.7919529286963355,
1007
+ "pass@64": 0.8099419980425414,
1008
+ "pass@128": 0.8259473684210527
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9150703125,
1013
+ "pass@2": 0.9398096702755907,
1014
+ "pass@4": 0.9555400461192349,
1015
+ "pass@8": 0.9668906410889534,
1016
+ "pass@16": 0.9758948266659789,
1017
+ "pass@32": 0.9832636557874399,
1018
+ "pass@64": 0.9892967113568902,
1019
+ "pass@128": 0.995
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.990578125,
1023
+ "pass@2": 0.9930145177165355,
1024
+ "pass@4": 0.9943968228346454,
1025
+ "pass@8": 0.9952873162624573,
1026
+ "pass@16": 0.9960729357688821,
1027
+ "pass@32": 0.996952502557328,
1028
+ "pass@64": 0.9979116198042409,
1029
+ "pass@128": 0.999
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.99659375,
1033
+ "pass@2": 0.9988639271653545,
1034
+ "pass@4": 0.9997500349643794,
1035
+ "pass@8": 0.9999798240813113,
1036
+ "pass@16": 0.9999998412707761,
1037
+ "pass@32": 0.9999999999968697,
1038
+ "pass@64": 1.0,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9929296875,
1043
+ "pass@2": 0.9952704232283465,
1044
+ "pass@4": 0.9970510909261342,
1045
+ "pass@8": 0.9984842712829364,
1046
+ "pass@16": 0.9994335154423146,
1047
+ "pass@32": 0.999865478267178,
1048
+ "pass@64": 0.9999934228152456,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.962078125,
1053
+ "pass@2": 0.9698677411417324,
1054
+ "pass@4": 0.9763977542182228,
1055
+ "pass@8": 0.9819934096902104,
1056
+ "pass@16": 0.9865521659361441,
1057
+ "pass@32": 0.9903191098925549,
1058
+ "pass@64": 0.9939691902252709,
1059
+ "pass@128": 0.998
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9701875,
1063
+ "pass@2": 0.9778256643700791,
1064
+ "pass@4": 0.9825182044431944,
1065
+ "pass@8": 0.9861330184093109,
1066
+ "pass@16": 0.9892783904066073,
1067
+ "pass@32": 0.9920173554056341,
1068
+ "pass@64": 0.9943151204054765,
1069
+ "pass@128": 0.996
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.97875,
1073
+ "pass@2": 0.9858042568897639,
1074
+ "pass@4": 0.9899331335770527,
1075
+ "pass@8": 0.9927339385848097,
1076
+ "pass@16": 0.9948275081708874,
1077
+ "pass@32": 0.9963086569131329,
1078
+ "pass@64": 0.9973199896660145,
1079
+ "pass@128": 0.998
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.964484375,
1083
+ "pass@2": 0.977889886811024,
1084
+ "pass@4": 0.9856384359767532,
1085
+ "pass@8": 0.9905634974996607,
1086
+ "pass@16": 0.9937330000695483,
1087
+ "pass@32": 0.9958854298336186,
1088
+ "pass@64": 0.9975687158504051,
1089
+ "pass@128": 0.999
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.961359375,
1093
+ "pass@2": 0.97617876476378,
1094
+ "pass@4": 0.9831911287964004,
1095
+ "pass@8": 0.9875323524329049,
1096
+ "pass@16": 0.9909118825360496,
1097
+ "pass@32": 0.9935134543703622,
1098
+ "pass@64": 0.9951271025688606,
1099
+ "pass@128": 0.996
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.8648359375,
1103
+ "pass@2": 0.9100951033464573,
1104
+ "pass@4": 0.9347501322647173,
1105
+ "pass@8": 0.9492648414627204,
1106
+ "pass@16": 0.9591873478095747,
1107
+ "pass@32": 0.9663478688104976,
1108
+ "pass@64": 0.9718211052876301,
1109
+ "pass@128": 0.976
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.7038046875,
1113
+ "pass@2": 0.7893309547244096,
1114
+ "pass@4": 0.847530710161229,
1115
+ "pass@8": 0.8872501406974933,
1116
+ "pass@16": 0.913835808415057,
1117
+ "pass@32": 0.9321118322142374,
1118
+ "pass@64": 0.9463118241321323,
1119
+ "pass@128": 0.957
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.4401484375,
1123
+ "pass@2": 0.5271272145669298,
1124
+ "pass@4": 0.6008414419760031,
1125
+ "pass@8": 0.663756701110009,
1126
+ "pass@16": 0.716035068325726,
1127
+ "pass@32": 0.7594926715885558,
1128
+ "pass@64": 0.796119320741206,
1129
+ "pass@128": 0.829
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.3608828125,
1133
+ "pass@2": 0.4397880167322835,
1134
+ "pass@4": 0.508594947975253,
1135
+ "pass@8": 0.5686336976073794,
1136
+ "pass@16": 0.6218801756842585,
1137
+ "pass@32": 0.6686538789478669,
1138
+ "pass@64": 0.7076235209573943,
1139
+ "pass@128": 0.741
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.2531953125,
1143
+ "pass@2": 0.32912573818897634,
1144
+ "pass@4": 0.40056516357330324,
1145
+ "pass@8": 0.4651769969975074,
1146
+ "pass@16": 0.5209395590458854,
1147
+ "pass@32": 0.5685441656287817,
1148
+ "pass@64": 0.6091709442131789,
1149
+ "pass@128": 0.644
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.21353125,
1153
+ "pass@2": 0.277843626968504,
1154
+ "pass@4": 0.3358475869891258,
1155
+ "pass@8": 0.38845783283534113,
1156
+ "pass@16": 0.4373589747108126,
1157
+ "pass@32": 0.481591610081772,
1158
+ "pass@64": 0.5213278775966319,
1159
+ "pass@128": 0.557
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.1928125,
1163
+ "pass@2": 0.25692937992125997,
1164
+ "pass@4": 0.3162516365766777,
1165
+ "pass@8": 0.3696114083029441,
1166
+ "pass@16": 0.4174144617061861,
1167
+ "pass@32": 0.46007529401338054,
1168
+ "pass@64": 0.5000393438805595,
1169
+ "pass@128": 0.543
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.17340625,
1173
+ "pass@2": 0.23288939468503944,
1174
+ "pass@4": 0.2924173040869889,
1175
+ "pass@8": 0.34748396702279544,
1176
+ "pass@16": 0.39519400799025706,
1177
+ "pass@32": 0.4368171730593396,
1178
+ "pass@64": 0.47577441414294835,
1179
+ "pass@128": 0.512
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.152296875,
1183
+ "pass@2": 0.21116178641732283,
1184
+ "pass@4": 0.26833084833145837,
1185
+ "pass@8": 0.32048557651113246,
1186
+ "pass@16": 0.3667058127901986,
1187
+ "pass@32": 0.4060712287023584,
1188
+ "pass@64": 0.43877021147199496,
1189
+ "pass@128": 0.467
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.161640625,
1193
+ "pass@2": 0.22280364173228356,
1194
+ "pass@4": 0.2810278937007869,
1195
+ "pass@8": 0.3322372896986801,
1196
+ "pass@16": 0.37756368115318545,
1197
+ "pass@32": 0.41927427915921184,
1198
+ "pass@64": 0.45643752769216683,
1199
+ "pass@128": 0.486
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9150703125,
1204
+ "2": 0.990578125,
1205
+ "3": 0.99659375,
1206
+ "4": 0.9929296875,
1207
+ "5": 0.962078125,
1208
+ "6": 0.9701875,
1209
+ "7": 0.97875,
1210
+ "8": 0.964484375,
1211
+ "9": 0.961359375,
1212
+ "11": 0.8648359375,
1213
+ "12": 0.7038046875,
1214
+ "13": 0.4401484375,
1215
+ "14": 0.3608828125,
1216
+ "15": 0.2531953125,
1217
+ "16": 0.21353125,
1218
+ "17": 0.1928125,
1219
+ "18": 0.17340625,
1220
+ "19": 0.152296875,
1221
+ "20": 0.161640625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 278.345546875,
1225
+ "2": 116.3494296875,
1226
+ "3": 144.50065625,
1227
+ "4": 159.5111484375,
1228
+ "5": 173.8181953125,
1229
+ "6": 198.9171640625,
1230
+ "7": 223.07409375,
1231
+ "8": 242.34471875,
1232
+ "9": 257.823328125,
1233
+ "11": 294.4192421875,
1234
+ "12": 301.11725,
1235
+ "13": 310.0360859375,
1236
+ "14": 320.811046875,
1237
+ "15": 323.3763125,
1238
+ "16": 327.6389453125,
1239
+ "17": 333.5490078125,
1240
+ "18": 329.8962734375,
1241
+ "19": 329.6489375,
1242
+ "20": 330.407078125
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 278.345546875,
1247
+ "2": 116.3494296875,
1248
+ "3": 144.50065625,
1249
+ "4": 159.5111484375,
1250
+ "5": 173.8181953125,
1251
+ "6": 198.9171640625,
1252
+ "7": 223.07409375,
1253
+ "8": 242.34471875,
1254
+ "9": 257.823328125,
1255
+ "11": 294.4192421875,
1256
+ "12": 301.11725,
1257
+ "13": 310.0360859375,
1258
+ "14": 320.811046875,
1259
+ "15": 323.3763125,
1260
+ "16": 327.6389453125,
1261
+ "17": 333.5490078125,
1262
+ "18": 329.8962734375,
1263
+ "19": 329.6489375,
1264
+ "20": 330.407078125
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 519453,
1271
+ "answer_accuracy": 0.6415154224628518,
1272
+ "avg_response_len": 250.30623246324691,
1273
+ "resp_tokens_sum": 202679965,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.6415154224628518,
1276
+ "pass@2": 0.6825554019034074,
1277
+ "pass@4": 0.7162482077109957,
1278
+ "pass@8": 0.7440358797725249,
1279
+ "pass@16": 0.7673312676500218,
1280
+ "pass@32": 0.7871519992986242,
1281
+ "pass@64": 0.8040759025836397,
1282
+ "pass@128": 0.8188428706923806
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9222437315634219,
1287
+ "pass@2": 0.9451468248437975,
1288
+ "pass@4": 0.9604848572578869,
1289
+ "pass@8": 0.9712668741187709,
1290
+ "pass@16": 0.9797398811861905,
1291
+ "pass@32": 0.986931871980946,
1292
+ "pass@64": 0.9931403532607942,
1293
+ "pass@128": 1.0
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.989242601246106,
1297
+ "pass@2": 0.9923153224421711,
1298
+ "pass@4": 0.9941436283665476,
1299
+ "pass@8": 0.9953145921083346,
1300
+ "pass@16": 0.9966062996683891,
1301
+ "pass@32": 0.9983101878563488,
1302
+ "pass@64": 0.9997246792749038,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9975630733944955,
1307
+ "pass@2": 0.9996982530280046,
1308
+ "pass@4": 0.9999930091880717,
1309
+ "pass@8": 0.9999999967989345,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.992236328125,
1317
+ "pass@2": 0.9945377860482283,
1318
+ "pass@4": 0.9961152392083802,
1319
+ "pass@8": 0.9975110009296438,
1320
+ "pass@16": 0.9987271357419916,
1321
+ "pass@32": 0.9996050186206877,
1322
+ "pass@64": 0.99997946273747,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9551587301587302,
1327
+ "pass@2": 0.963551431071116,
1328
+ "pass@4": 0.9705333979085945,
1329
+ "pass@8": 0.9768221559126241,
1330
+ "pass@16": 0.9826882032901565,
1331
+ "pass@32": 0.988084362473456,
1332
+ "pass@64": 0.992762368619646,
1333
+ "pass@128": 0.9968253968253968
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9563061377245509,
1337
+ "pass@2": 0.9692498614974779,
1338
+ "pass@4": 0.9767356600073196,
1339
+ "pass@8": 0.9813628882947097,
1340
+ "pass@16": 0.9850560727092349,
1341
+ "pass@32": 0.9883559011032365,
1342
+ "pass@64": 0.991386843357507,
1343
+ "pass@128": 0.9940119760479041
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9775563979289941,
1347
+ "pass@2": 0.9875821908633461,
1348
+ "pass@4": 0.9925620808123836,
1349
+ "pass@8": 0.9950344430852842,
1350
+ "pass@16": 0.9964798475945668,
1351
+ "pass@32": 0.997005436160734,
1352
+ "pass@64": 0.9970413979979968,
1353
+ "pass@128": 0.9970414201183432
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9700270432692307,
1357
+ "pass@2": 0.9840835888602868,
1358
+ "pass@4": 0.9896981008383567,
1359
+ "pass@8": 0.9919003031750214,
1360
+ "pass@16": 0.9935285960287868,
1361
+ "pass@32": 0.9953138505545496,
1362
+ "pass@64": 0.9966101352715523,
1363
+ "pass@128": 0.9967948717948718
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9554577464788733,
1367
+ "pass@2": 0.9742978540534544,
1368
+ "pass@4": 0.984426338080979,
1369
+ "pass@8": 0.9901960111237609,
1370
+ "pass@16": 0.9936839096661296,
1371
+ "pass@32": 0.9957886523598386,
1372
+ "pass@64": 0.9969763514894339,
1373
+ "pass@128": 0.9971830985915493
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.8513411827195467,
1377
+ "pass@2": 0.9002692054605079,
1378
+ "pass@4": 0.9301656318278912,
1379
+ "pass@8": 0.9489189290719003,
1380
+ "pass@16": 0.9620749038932601,
1381
+ "pass@32": 0.9706335966755882,
1382
+ "pass@64": 0.9751169549682261,
1383
+ "pass@128": 0.9773371104815864
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.7018449858757062,
1387
+ "pass@2": 0.786281779661017,
1388
+ "pass@4": 0.8425637121101385,
1389
+ "pass@8": 0.8814481070980112,
1390
+ "pass@16": 0.9089087677573177,
1391
+ "pass@32": 0.9300411469087887,
1392
+ "pass@64": 0.9476799728882689,
1393
+ "pass@128": 0.96045197740113
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.46071708463949845,
1397
+ "pass@2": 0.5479957822180537,
1398
+ "pass@4": 0.6168102769481397,
1399
+ "pass@8": 0.6711818283871478,
1400
+ "pass@16": 0.714081813122014,
1401
+ "pass@32": 0.7497040752868626,
1402
+ "pass@64": 0.7797252476653084,
1403
+ "pass@128": 0.8056426332288401
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.3388093847352025,
1407
+ "pass@2": 0.41182321178894693,
1408
+ "pass@4": 0.4764041463275034,
1409
+ "pass@8": 0.534130178473544,
1410
+ "pass@16": 0.5871949465142853,
1411
+ "pass@32": 0.6363122142046183,
1412
+ "pass@64": 0.6797314428750697,
1413
+ "pass@128": 0.7165109034267912
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.26275740606936415,
1417
+ "pass@2": 0.3398819751035454,
1418
+ "pass@4": 0.41254214727855526,
1419
+ "pass@8": 0.4781416760702823,
1420
+ "pass@16": 0.5331817011339766,
1421
+ "pass@32": 0.5794008648263183,
1422
+ "pass@64": 0.6195336061079341,
1423
+ "pass@128": 0.6560693641618497
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.19909274193548387,
1427
+ "pass@2": 0.26266430657861306,
1428
+ "pass@4": 0.3225466130241786,
1429
+ "pass@8": 0.37652892333302973,
1430
+ "pass@16": 0.4252326950906888,
1431
+ "pass@32": 0.46738340757399127,
1432
+ "pass@64": 0.505813596566932,
1433
+ "pass@128": 0.5451612903225806
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.18987084537572255,
1437
+ "pass@2": 0.25301889422420454,
1438
+ "pass@4": 0.30949784781237605,
1439
+ "pass@8": 0.35793822819914667,
1440
+ "pass@16": 0.39987978151086667,
1441
+ "pass@32": 0.4364144577614477,
1442
+ "pass@64": 0.46841234177173346,
1443
+ "pass@128": 0.5028901734104047
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.16642857142857143,
1447
+ "pass@2": 0.22563835770528673,
1448
+ "pass@4": 0.2839633877015374,
1449
+ "pass@8": 0.3352756128571621,
1450
+ "pass@16": 0.3794013307508793,
1451
+ "pass@32": 0.41912972908665974,
1452
+ "pass@64": 0.45480493264252136,
1453
+ "pass@128": 0.4857142857142857
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.15287162162162163,
1457
+ "pass@2": 0.21274547184980255,
1458
+ "pass@4": 0.2720780853969829,
1459
+ "pass@8": 0.3269558100570385,
1460
+ "pass@16": 0.3740083610670051,
1461
+ "pass@32": 0.41088179701472477,
1462
+ "pass@64": 0.44192531549939784,
1463
+ "pass@128": 0.47147147147147145
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.17898367117117117,
1467
+ "pass@2": 0.24102498167458797,
1468
+ "pass@4": 0.2964189808481148,
1469
+ "pass@8": 0.34303163036602097,
1470
+ "pass@16": 0.38364439504923087,
1471
+ "pass@32": 0.4206433730102397,
1472
+ "pass@64": 0.4507408603585201,
1473
+ "pass@128": 0.46846846846846846
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 529783,
1480
+ "answer_accuracy": 0.6451955865159782,
1481
+ "avg_response_len": 259.37219772992984,
1482
+ "resp_tokens_sum": 212975699,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.6451955865159782,
1485
+ "pass@2": 0.68438010153982,
1486
+ "pass@4": 0.7178246167251279,
1487
+ "pass@8": 0.7464661811749366,
1488
+ "pass@16": 0.770699083942655,
1489
+ "pass@32": 0.79120344103857,
1490
+ "pass@64": 0.8090207708634686,
1491
+ "pass@128": 0.8255650818394388
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9157400306748467,
1496
+ "pass@2": 0.9375864239650259,
1497
+ "pass@4": 0.9503189637721664,
1498
+ "pass@8": 0.959476709600869,
1499
+ "pass@16": 0.9678207783989702,
1500
+ "pass@32": 0.976828113172386,
1501
+ "pass@64": 0.9855899073216104,
1502
+ "pass@128": 0.9938650306748467
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.9930661094224924,
1506
+ "pass@2": 0.9945559497881914,
1507
+ "pass@4": 0.9952527896398966,
1508
+ "pass@8": 0.9960218094796784,
1509
+ "pass@16": 0.996694876664963,
1510
+ "pass@32": 0.9969450828883747,
1511
+ "pass@64": 0.9969604795045498,
1512
+ "pass@128": 0.9969604863221885
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9994791666666667,
1516
+ "pass@2": 0.9999657003101885,
1517
+ "pass@4": 0.9999997156605425,
1518
+ "pass@8": 0.9999999999936351,
1519
+ "pass@16": 1.0,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9902921597633136,
1526
+ "pass@2": 0.9935943542375252,
1527
+ "pass@4": 0.996376267685474,
1528
+ "pass@8": 0.9984966632337737,
1529
+ "pass@16": 0.9996459679388677,
1530
+ "pass@32": 0.9999785901153704,
1531
+ "pass@64": 0.9999999845150909,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9790760869565217,
1536
+ "pass@2": 0.9860735478717333,
1537
+ "pass@4": 0.990809667269852,
1538
+ "pass@8": 0.9943024238372346,
1539
+ "pass@16": 0.9967581848072867,
1540
+ "pass@32": 0.9982186985410357,
1541
+ "pass@64": 0.9992791299089842,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9788905802292264,
1546
+ "pass@2": 0.9836452716422623,
1547
+ "pass@4": 0.9860912927610409,
1548
+ "pass@8": 0.9885082645987024,
1549
+ "pass@16": 0.9914576644920206,
1550
+ "pass@32": 0.994320594090395,
1551
+ "pass@64": 0.9964244742548366,
1552
+ "pass@128": 0.997134670487106
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.979091401734104,
1556
+ "pass@2": 0.9841676954849575,
1557
+ "pass@4": 0.9884633377655827,
1558
+ "pass@8": 0.9922804183734506,
1559
+ "pass@16": 0.9951181931156187,
1560
+ "pass@32": 0.9970528714118317,
1561
+ "pass@64": 0.9984707380171003,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9671654929577465,
1566
+ "pass@2": 0.9811702894532549,
1567
+ "pass@4": 0.9894243719535057,
1568
+ "pass@8": 0.9944174851855382,
1569
+ "pass@16": 0.9969106947639146,
1570
+ "pass@32": 0.9978444559613356,
1571
+ "pass@64": 0.9985914715905386,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9666448353293413,
1576
+ "pass@2": 0.9787814872224057,
1577
+ "pass@4": 0.984590454074977,
1578
+ "pass@8": 0.9884570828941244,
1579
+ "pass@16": 0.9916722775023806,
1580
+ "pass@32": 0.9940966746741055,
1581
+ "pass@64": 0.9954845654245074,
1582
+ "pass@128": 0.9970059880239521
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.867211031626506,
1586
+ "pass@2": 0.911044522578503,
1587
+ "pass@4": 0.9335228710116054,
1588
+ "pass@8": 0.9456478076783712,
1589
+ "pass@16": 0.9524671555395998,
1590
+ "pass@32": 0.9570610201613988,
1591
+ "pass@64": 0.9621282660404785,
1592
+ "pass@128": 0.9698795180722891
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.71608052507837,
1596
+ "pass@2": 0.7989217967072296,
1597
+ "pass@4": 0.8551101025185328,
1598
+ "pass@8": 0.8918423946053244,
1599
+ "pass@16": 0.9147090576639251,
1600
+ "pass@32": 0.9288851299053897,
1601
+ "pass@64": 0.9409932385294371,
1602
+ "pass@128": 0.9529780564263323
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.40434611344537813,
1606
+ "pass@2": 0.48740185050398094,
1607
+ "pass@4": 0.5640730444408734,
1608
+ "pass@8": 0.6355232978685984,
1609
+ "pass@16": 0.6996316769009948,
1610
+ "pass@32": 0.7527484934731532,
1611
+ "pass@64": 0.7943710656136685,
1612
+ "pass@128": 0.8291316526610645
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.3604486476608187,
1616
+ "pass@2": 0.44008114322880676,
1617
+ "pass@4": 0.5081644520312152,
1618
+ "pass@8": 0.5657154899960264,
1619
+ "pass@16": 0.6164087304791761,
1620
+ "pass@32": 0.6613545821335254,
1621
+ "pass@64": 0.6976279435761811,
1622
+ "pass@128": 0.7309941520467836
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.22407670454545456,
1626
+ "pass@2": 0.30122323132903833,
1627
+ "pass@4": 0.37661032541386885,
1628
+ "pass@8": 0.44410643751587464,
1629
+ "pass@16": 0.4993273457645693,
1630
+ "pass@32": 0.5427207028934755,
1631
+ "pass@64": 0.5788425426868631,
1632
+ "pass@128": 0.6121212121212121
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.23039084757834757,
1636
+ "pass@2": 0.2978291074993831,
1637
+ "pass@4": 0.35624147569160675,
1638
+ "pass@8": 0.40731401792660654,
1639
+ "pass@16": 0.45254311760056604,
1640
+ "pass@32": 0.49147289445246195,
1641
+ "pass@64": 0.5235553530937181,
1642
+ "pass@128": 0.5470085470085471
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.21370341614906832,
1646
+ "pass@2": 0.2806520363623024,
1647
+ "pass@4": 0.3440671235971277,
1648
+ "pass@8": 0.40228302586205095,
1649
+ "pass@16": 0.4537379226594085,
1650
+ "pass@32": 0.500026224710816,
1651
+ "pass@64": 0.5454292278608023,
1652
+ "pass@128": 0.593167701863354
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.17212775735294117,
1656
+ "pass@2": 0.23196720125057887,
1657
+ "pass@4": 0.2933669291338581,
1658
+ "pass@8": 0.35235556676313684,
1659
+ "pass@16": 0.4034011348916748,
1660
+ "pass@32": 0.4468507861160033,
1661
+ "pass@64": 0.48950582673115856,
1662
+ "pass@128": 0.5352941176470588
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.15087890625,
1666
+ "pass@2": 0.2072612017247844,
1667
+ "pass@4": 0.2605428718731588,
1668
+ "pass@8": 0.3074317250275432,
1669
+ "pass@16": 0.34908958061974404,
1670
+ "pass@32": 0.3879378881426714,
1671
+ "pass@64": 0.42367879660082997,
1672
+ "pass@128": 0.45535714285714285
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.1435254491017964,
1676
+ "pass@2": 0.20275848401621951,
1677
+ "pass@4": 0.26188721965517786,
1678
+ "pass@8": 0.31475370332114117,
1679
+ "pass@16": 0.3611758484951675,
1680
+ "pass@32": 0.40307471926923255,
1681
+ "pass@64": 0.43896682078335747,
1682
+ "pass@128": 0.47005988023952094
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 518583,
1689
+ "answer_accuracy": 0.6472966428343185,
1690
+ "avg_response_len": 279.3217104868989,
1691
+ "resp_tokens_sum": 223779147,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.6472966428343185,
1694
+ "pass@2": 0.687566046625146,
1695
+ "pass@4": 0.7213202171624994,
1696
+ "pass@8": 0.7503762891676571,
1697
+ "pass@16": 0.7756265763304547,
1698
+ "pass@32": 0.7975734180227835,
1699
+ "pass@64": 0.8168150755671897,
1700
+ "pass@128": 0.8335197315865154
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9071595149253732,
1705
+ "pass@2": 0.9365723057938655,
1706
+ "pass@4": 0.9556170069226421,
1707
+ "pass@8": 0.9696769057695721,
1708
+ "pass@16": 0.9798609945844656,
1709
+ "pass@32": 0.9858142874379154,
1710
+ "pass@64": 0.9890143934765259,
1711
+ "pass@128": 0.991044776119403
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9894642857142857,
1715
+ "pass@2": 0.99220683352081,
1716
+ "pass@4": 0.9938244295355938,
1717
+ "pass@8": 0.9945718767910509,
1718
+ "pass@16": 0.9949991404358748,
1719
+ "pass@32": 0.9957142856718992,
1720
+ "pass@64": 0.9971428571428571,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9928935860058309,
1725
+ "pass@2": 0.9970085053143868,
1726
+ "pass@4": 0.9992781772358628,
1727
+ "pass@8": 0.9999411811374931,
1728
+ "pass@16": 0.9999995372325835,
1729
+ "pass@32": 0.9999999999908739,
1730
+ "pass@64": 1.0,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9961851242690059,
1735
+ "pass@2": 0.9976123975457015,
1736
+ "pass@4": 0.998593672227375,
1737
+ "pass@8": 0.9993826865860662,
1738
+ "pass@16": 0.99988448784076,
1739
+ "pass@32": 0.9999973942969671,
1740
+ "pass@64": 0.9999999999215047,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9512408088235295,
1745
+ "pass@2": 0.9592754892311257,
1746
+ "pass@4": 0.9672070549085778,
1747
+ "pass@8": 0.9742943951584943,
1748
+ "pass@16": 0.9797759062977376,
1749
+ "pass@32": 0.9843737785787031,
1750
+ "pass@64": 0.9896992479161258,
1751
+ "pass@128": 0.9970588235294118
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9752316640378549,
1755
+ "pass@2": 0.9804542928289322,
1756
+ "pass@4": 0.9846770751645001,
1757
+ "pass@8": 0.9885439412426841,
1758
+ "pass@16": 0.9913278776467133,
1759
+ "pass@32": 0.9933394230271457,
1760
+ "pass@64": 0.9950781489247041,
1761
+ "pass@128": 0.9968454258675079
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9796528876582279,
1765
+ "pass@2": 0.985694473861258,
1766
+ "pass@4": 0.9887304917581505,
1767
+ "pass@8": 0.9907698483062334,
1768
+ "pass@16": 0.9927418508415174,
1769
+ "pass@32": 0.994748499944022,
1770
+ "pass@64": 0.996357980344224,
1771
+ "pass@128": 0.9968354430379747
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9564329954954955,
1775
+ "pass@2": 0.9685896526447708,
1776
+ "pass@4": 0.9777987281432166,
1777
+ "pass@8": 0.9852023893939588,
1778
+ "pass@16": 0.9905368812834143,
1779
+ "pass@32": 0.9943325122952712,
1780
+ "pass@64": 0.9973765202133318,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.96241961414791,
1785
+ "pass@2": 0.9755305782717674,
1786
+ "pass@4": 0.9802783508572683,
1787
+ "pass@8": 0.9834987228210043,
1788
+ "pass@16": 0.9869310415394815,
1789
+ "pass@32": 0.9902900110658145,
1790
+ "pass@64": 0.9926323245605343,
1791
+ "pass@128": 0.9935691318327974
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.8774553571428572,
1795
+ "pass@2": 0.9201056899137611,
1796
+ "pass@4": 0.9411811779479943,
1797
+ "pass@8": 0.9534647217495896,
1798
+ "pass@16": 0.9630343209400213,
1799
+ "pass@32": 0.9713331761600971,
1800
+ "pass@64": 0.9783436058997058,
1801
+ "pass@128": 0.9809523809523809
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.6939506880733946,
1805
+ "pass@2": 0.7832756928893064,
1806
+ "pass@4": 0.8455138635193533,
1807
+ "pass@8": 0.8890513361030553,
1808
+ "pass@16": 0.9183177835907471,
1809
+ "pass@32": 0.9375012531153086,
1810
+ "pass@64": 0.9500191762654265,
1811
+ "pass@128": 0.9571865443425076
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.4593460648148148,
1815
+ "pass@2": 0.550352158671138,
1816
+ "pass@4": 0.6256323665560319,
1817
+ "pass@8": 0.6875551867759847,
1818
+ "pass@16": 0.7360322879202137,
1819
+ "pass@32": 0.7765612944510213,
1820
+ "pass@64": 0.8141867170120144,
1821
+ "pass@128": 0.8518518518518519
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.38234885014836795,
1825
+ "pass@2": 0.46612764024860376,
1826
+ "pass@4": 0.5396942860814502,
1827
+ "pass@8": 0.6044605719250148,
1828
+ "pass@16": 0.6604712523124485,
1829
+ "pass@32": 0.7068676293724594,
1830
+ "pass@64": 0.7443352257906309,
1831
+ "pass@128": 0.7744807121661721
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.2726417824074074,
1835
+ "pass@2": 0.3460583594585398,
1836
+ "pass@4": 0.41217337416156297,
1837
+ "pass@8": 0.4727927552375029,
1838
+ "pass@16": 0.5298785998494511,
1839
+ "pass@32": 0.5832519581605827,
1840
+ "pass@64": 0.6289946833739761,
1841
+ "pass@128": 0.6635802469135802
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.20927820796460178,
1845
+ "pass@2": 0.2710314902561958,
1846
+ "pass@4": 0.32689492325627434,
1847
+ "pass@8": 0.37984261448337203,
1848
+ "pass@16": 0.43272626830354155,
1849
+ "pass@32": 0.48435329728914633,
1850
+ "pass@64": 0.5332086540560117,
1851
+ "pass@128": 0.5781710914454278
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.17561652861445784,
1855
+ "pass@2": 0.2379966470211554,
1856
+ "pass@4": 0.29631255251602584,
1857
+ "pass@8": 0.3500892982483715,
1858
+ "pass@16": 0.4004591753225813,
1859
+ "pass@32": 0.44598628696095505,
1860
+ "pass@64": 0.4889772959528953,
1861
+ "pass@128": 0.536144578313253
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.18268649193548386,
1865
+ "pass@2": 0.24208748729997448,
1866
+ "pass@4": 0.3009205241481911,
1867
+ "pass@8": 0.35592454781716937,
1868
+ "pass@16": 0.40402308504606516,
1869
+ "pass@32": 0.44578225999860455,
1870
+ "pass@64": 0.4843893762241014,
1871
+ "pass@128": 0.5161290322580645
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.15315804380664652,
1875
+ "pass@2": 0.21352803780003343,
1876
+ "pass@4": 0.27246658895734704,
1877
+ "pass@8": 0.32722728747094926,
1878
+ "pass@16": 0.3772414787512142,
1879
+ "pass@32": 0.41963885160277065,
1880
+ "pass@64": 0.4509154252350958,
1881
+ "pass@128": 0.4743202416918429
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.16246715465465467,
1885
+ "pass@2": 0.22468765517485983,
1886
+ "pass@4": 0.284834960100708,
1887
+ "pass@8": 0.33897903867127327,
1888
+ "pass@16": 0.3879200126258433,
1889
+ "pass@32": 0.4341533925251601,
1890
+ "pass@64": 0.4796573665199342,
1891
+ "pass@128": 0.5195195195195195
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch1-op7-10/SDAR-100M-block16-epoch1-op7-10_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4522fe7823b8fc35f20b3eaa41c7e084ec452939dec099c31a313f7960cf15a2
3
+ size 2558540622
results/composition/SDAR-100M-block16-epoch1-op7-10/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch1-op7-10,0.9702256944444444,0.35165546875,0.6446624177631579,0.0,0.0,0.0,199.40936458333334,320.09001796875,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch1-op7-10/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch1-op7-10",
4
+ "id_acc": 0.9702256944444444,
5
+ "ood_acc": 0.35165546875,
6
+ "total_acc": 0.6446624177631579,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.40936458333334,
11
+ "ood_avg_resp_len": 320.09001796875,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 519453,
19
+ "answer_accuracy": 0.6415154224628518,
20
+ "avg_response_len": 250.30623246324691,
21
+ "resp_tokens_sum": 202679965,
22
+ "pass_at_k": {
23
+ "pass@1": 0.6415154224628518,
24
+ "pass@2": 0.6825554019034074,
25
+ "pass@4": 0.7162482077109957,
26
+ "pass@8": 0.7440358797725249,
27
+ "pass@16": 0.7673312676500218,
28
+ "pass@32": 0.7871519992986242,
29
+ "pass@64": 0.8040759025836397,
30
+ "pass@128": 0.8188428706923806
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9222437315634219,
35
+ "pass@2": 0.9451468248437975,
36
+ "pass@4": 0.9604848572578869,
37
+ "pass@8": 0.9712668741187709,
38
+ "pass@16": 0.9797398811861905,
39
+ "pass@32": 0.986931871980946,
40
+ "pass@64": 0.9931403532607942,
41
+ "pass@128": 1.0
42
+ },
43
+ "2": {
44
+ "pass@1": 0.989242601246106,
45
+ "pass@2": 0.9923153224421711,
46
+ "pass@4": 0.9941436283665476,
47
+ "pass@8": 0.9953145921083346,
48
+ "pass@16": 0.9966062996683891,
49
+ "pass@32": 0.9983101878563488,
50
+ "pass@64": 0.9997246792749038,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 0.9975630733944955,
55
+ "pass@2": 0.9996982530280046,
56
+ "pass@4": 0.9999930091880717,
57
+ "pass@8": 0.9999999967989345,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.992236328125,
65
+ "pass@2": 0.9945377860482283,
66
+ "pass@4": 0.9961152392083802,
67
+ "pass@8": 0.9975110009296438,
68
+ "pass@16": 0.9987271357419916,
69
+ "pass@32": 0.9996050186206877,
70
+ "pass@64": 0.99997946273747,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9551587301587302,
75
+ "pass@2": 0.963551431071116,
76
+ "pass@4": 0.9705333979085945,
77
+ "pass@8": 0.9768221559126241,
78
+ "pass@16": 0.9826882032901565,
79
+ "pass@32": 0.988084362473456,
80
+ "pass@64": 0.992762368619646,
81
+ "pass@128": 0.9968253968253968
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9563061377245509,
85
+ "pass@2": 0.9692498614974779,
86
+ "pass@4": 0.9767356600073196,
87
+ "pass@8": 0.9813628882947097,
88
+ "pass@16": 0.9850560727092349,
89
+ "pass@32": 0.9883559011032365,
90
+ "pass@64": 0.991386843357507,
91
+ "pass@128": 0.9940119760479041
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9775563979289941,
95
+ "pass@2": 0.9875821908633461,
96
+ "pass@4": 0.9925620808123836,
97
+ "pass@8": 0.9950344430852842,
98
+ "pass@16": 0.9964798475945668,
99
+ "pass@32": 0.997005436160734,
100
+ "pass@64": 0.9970413979979968,
101
+ "pass@128": 0.9970414201183432
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9700270432692307,
105
+ "pass@2": 0.9840835888602868,
106
+ "pass@4": 0.9896981008383567,
107
+ "pass@8": 0.9919003031750214,
108
+ "pass@16": 0.9935285960287868,
109
+ "pass@32": 0.9953138505545496,
110
+ "pass@64": 0.9966101352715523,
111
+ "pass@128": 0.9967948717948718
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9554577464788733,
115
+ "pass@2": 0.9742978540534544,
116
+ "pass@4": 0.984426338080979,
117
+ "pass@8": 0.9901960111237609,
118
+ "pass@16": 0.9936839096661296,
119
+ "pass@32": 0.9957886523598386,
120
+ "pass@64": 0.9969763514894339,
121
+ "pass@128": 0.9971830985915493
122
+ },
123
+ "11": {
124
+ "pass@1": 0.8513411827195467,
125
+ "pass@2": 0.9002692054605079,
126
+ "pass@4": 0.9301656318278912,
127
+ "pass@8": 0.9489189290719003,
128
+ "pass@16": 0.9620749038932601,
129
+ "pass@32": 0.9706335966755882,
130
+ "pass@64": 0.9751169549682261,
131
+ "pass@128": 0.9773371104815864
132
+ },
133
+ "12": {
134
+ "pass@1": 0.7018449858757062,
135
+ "pass@2": 0.786281779661017,
136
+ "pass@4": 0.8425637121101385,
137
+ "pass@8": 0.8814481070980112,
138
+ "pass@16": 0.9089087677573177,
139
+ "pass@32": 0.9300411469087887,
140
+ "pass@64": 0.9476799728882689,
141
+ "pass@128": 0.96045197740113
142
+ },
143
+ "13": {
144
+ "pass@1": 0.46071708463949845,
145
+ "pass@2": 0.5479957822180537,
146
+ "pass@4": 0.6168102769481397,
147
+ "pass@8": 0.6711818283871478,
148
+ "pass@16": 0.714081813122014,
149
+ "pass@32": 0.7497040752868626,
150
+ "pass@64": 0.7797252476653084,
151
+ "pass@128": 0.8056426332288401
152
+ },
153
+ "14": {
154
+ "pass@1": 0.3388093847352025,
155
+ "pass@2": 0.41182321178894693,
156
+ "pass@4": 0.4764041463275034,
157
+ "pass@8": 0.534130178473544,
158
+ "pass@16": 0.5871949465142853,
159
+ "pass@32": 0.6363122142046183,
160
+ "pass@64": 0.6797314428750697,
161
+ "pass@128": 0.7165109034267912
162
+ },
163
+ "15": {
164
+ "pass@1": 0.26275740606936415,
165
+ "pass@2": 0.3398819751035454,
166
+ "pass@4": 0.41254214727855526,
167
+ "pass@8": 0.4781416760702823,
168
+ "pass@16": 0.5331817011339766,
169
+ "pass@32": 0.5794008648263183,
170
+ "pass@64": 0.6195336061079341,
171
+ "pass@128": 0.6560693641618497
172
+ },
173
+ "16": {
174
+ "pass@1": 0.19909274193548387,
175
+ "pass@2": 0.26266430657861306,
176
+ "pass@4": 0.3225466130241786,
177
+ "pass@8": 0.37652892333302973,
178
+ "pass@16": 0.4252326950906888,
179
+ "pass@32": 0.46738340757399127,
180
+ "pass@64": 0.505813596566932,
181
+ "pass@128": 0.5451612903225806
182
+ },
183
+ "17": {
184
+ "pass@1": 0.18987084537572255,
185
+ "pass@2": 0.25301889422420454,
186
+ "pass@4": 0.30949784781237605,
187
+ "pass@8": 0.35793822819914667,
188
+ "pass@16": 0.39987978151086667,
189
+ "pass@32": 0.4364144577614477,
190
+ "pass@64": 0.46841234177173346,
191
+ "pass@128": 0.5028901734104047
192
+ },
193
+ "18": {
194
+ "pass@1": 0.16642857142857143,
195
+ "pass@2": 0.22563835770528673,
196
+ "pass@4": 0.2839633877015374,
197
+ "pass@8": 0.3352756128571621,
198
+ "pass@16": 0.3794013307508793,
199
+ "pass@32": 0.41912972908665974,
200
+ "pass@64": 0.45480493264252136,
201
+ "pass@128": 0.4857142857142857
202
+ },
203
+ "19": {
204
+ "pass@1": 0.15287162162162163,
205
+ "pass@2": 0.21274547184980255,
206
+ "pass@4": 0.2720780853969829,
207
+ "pass@8": 0.3269558100570385,
208
+ "pass@16": 0.3740083610670051,
209
+ "pass@32": 0.41088179701472477,
210
+ "pass@64": 0.44192531549939784,
211
+ "pass@128": 0.47147147147147145
212
+ },
213
+ "20": {
214
+ "pass@1": 0.17898367117117117,
215
+ "pass@2": 0.24102498167458797,
216
+ "pass@4": 0.2964189808481148,
217
+ "pass@8": 0.34303163036602097,
218
+ "pass@16": 0.38364439504923087,
219
+ "pass@32": 0.4206433730102397,
220
+ "pass@64": 0.4507408603585201,
221
+ "pass@128": 0.46846846846846846
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 529783,
228
+ "answer_accuracy": 0.6451955865159782,
229
+ "avg_response_len": 259.37219772992984,
230
+ "resp_tokens_sum": 212975699,
231
+ "pass_at_k": {
232
+ "pass@1": 0.6451955865159782,
233
+ "pass@2": 0.68438010153982,
234
+ "pass@4": 0.7178246167251279,
235
+ "pass@8": 0.7464661811749366,
236
+ "pass@16": 0.770699083942655,
237
+ "pass@32": 0.79120344103857,
238
+ "pass@64": 0.8090207708634686,
239
+ "pass@128": 0.8255650818394388
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9157400306748467,
244
+ "pass@2": 0.9375864239650259,
245
+ "pass@4": 0.9503189637721664,
246
+ "pass@8": 0.959476709600869,
247
+ "pass@16": 0.9678207783989702,
248
+ "pass@32": 0.976828113172386,
249
+ "pass@64": 0.9855899073216104,
250
+ "pass@128": 0.9938650306748467
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9930661094224924,
254
+ "pass@2": 0.9945559497881914,
255
+ "pass@4": 0.9952527896398966,
256
+ "pass@8": 0.9960218094796784,
257
+ "pass@16": 0.996694876664963,
258
+ "pass@32": 0.9969450828883747,
259
+ "pass@64": 0.9969604795045498,
260
+ "pass@128": 0.9969604863221885
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9994791666666667,
264
+ "pass@2": 0.9999657003101885,
265
+ "pass@4": 0.9999997156605425,
266
+ "pass@8": 0.9999999999936351,
267
+ "pass@16": 1.0,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9902921597633136,
274
+ "pass@2": 0.9935943542375252,
275
+ "pass@4": 0.996376267685474,
276
+ "pass@8": 0.9984966632337737,
277
+ "pass@16": 0.9996459679388677,
278
+ "pass@32": 0.9999785901153704,
279
+ "pass@64": 0.9999999845150909,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9790760869565217,
284
+ "pass@2": 0.9860735478717333,
285
+ "pass@4": 0.990809667269852,
286
+ "pass@8": 0.9943024238372346,
287
+ "pass@16": 0.9967581848072867,
288
+ "pass@32": 0.9982186985410357,
289
+ "pass@64": 0.9992791299089842,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9788905802292264,
294
+ "pass@2": 0.9836452716422623,
295
+ "pass@4": 0.9860912927610409,
296
+ "pass@8": 0.9885082645987024,
297
+ "pass@16": 0.9914576644920206,
298
+ "pass@32": 0.994320594090395,
299
+ "pass@64": 0.9964244742548366,
300
+ "pass@128": 0.997134670487106
301
+ },
302
+ "7": {
303
+ "pass@1": 0.979091401734104,
304
+ "pass@2": 0.9841676954849575,
305
+ "pass@4": 0.9884633377655827,
306
+ "pass@8": 0.9922804183734506,
307
+ "pass@16": 0.9951181931156187,
308
+ "pass@32": 0.9970528714118317,
309
+ "pass@64": 0.9984707380171003,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9671654929577465,
314
+ "pass@2": 0.9811702894532549,
315
+ "pass@4": 0.9894243719535057,
316
+ "pass@8": 0.9944174851855382,
317
+ "pass@16": 0.9969106947639146,
318
+ "pass@32": 0.9978444559613356,
319
+ "pass@64": 0.9985914715905386,
320
+ "pass@128": 1.0
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9666448353293413,
324
+ "pass@2": 0.9787814872224057,
325
+ "pass@4": 0.984590454074977,
326
+ "pass@8": 0.9884570828941244,
327
+ "pass@16": 0.9916722775023806,
328
+ "pass@32": 0.9940966746741055,
329
+ "pass@64": 0.9954845654245074,
330
+ "pass@128": 0.9970059880239521
331
+ },
332
+ "11": {
333
+ "pass@1": 0.867211031626506,
334
+ "pass@2": 0.911044522578503,
335
+ "pass@4": 0.9335228710116054,
336
+ "pass@8": 0.9456478076783712,
337
+ "pass@16": 0.9524671555395998,
338
+ "pass@32": 0.9570610201613988,
339
+ "pass@64": 0.9621282660404785,
340
+ "pass@128": 0.9698795180722891
341
+ },
342
+ "12": {
343
+ "pass@1": 0.71608052507837,
344
+ "pass@2": 0.7989217967072296,
345
+ "pass@4": 0.8551101025185328,
346
+ "pass@8": 0.8918423946053244,
347
+ "pass@16": 0.9147090576639251,
348
+ "pass@32": 0.9288851299053897,
349
+ "pass@64": 0.9409932385294371,
350
+ "pass@128": 0.9529780564263323
351
+ },
352
+ "13": {
353
+ "pass@1": 0.40434611344537813,
354
+ "pass@2": 0.48740185050398094,
355
+ "pass@4": 0.5640730444408734,
356
+ "pass@8": 0.6355232978685984,
357
+ "pass@16": 0.6996316769009948,
358
+ "pass@32": 0.7527484934731532,
359
+ "pass@64": 0.7943710656136685,
360
+ "pass@128": 0.8291316526610645
361
+ },
362
+ "14": {
363
+ "pass@1": 0.3604486476608187,
364
+ "pass@2": 0.44008114322880676,
365
+ "pass@4": 0.5081644520312152,
366
+ "pass@8": 0.5657154899960264,
367
+ "pass@16": 0.6164087304791761,
368
+ "pass@32": 0.6613545821335254,
369
+ "pass@64": 0.6976279435761811,
370
+ "pass@128": 0.7309941520467836
371
+ },
372
+ "15": {
373
+ "pass@1": 0.22407670454545456,
374
+ "pass@2": 0.30122323132903833,
375
+ "pass@4": 0.37661032541386885,
376
+ "pass@8": 0.44410643751587464,
377
+ "pass@16": 0.4993273457645693,
378
+ "pass@32": 0.5427207028934755,
379
+ "pass@64": 0.5788425426868631,
380
+ "pass@128": 0.6121212121212121
381
+ },
382
+ "16": {
383
+ "pass@1": 0.23039084757834757,
384
+ "pass@2": 0.2978291074993831,
385
+ "pass@4": 0.35624147569160675,
386
+ "pass@8": 0.40731401792660654,
387
+ "pass@16": 0.45254311760056604,
388
+ "pass@32": 0.49147289445246195,
389
+ "pass@64": 0.5235553530937181,
390
+ "pass@128": 0.5470085470085471
391
+ },
392
+ "17": {
393
+ "pass@1": 0.21370341614906832,
394
+ "pass@2": 0.2806520363623024,
395
+ "pass@4": 0.3440671235971277,
396
+ "pass@8": 0.40228302586205095,
397
+ "pass@16": 0.4537379226594085,
398
+ "pass@32": 0.500026224710816,
399
+ "pass@64": 0.5454292278608023,
400
+ "pass@128": 0.593167701863354
401
+ },
402
+ "18": {
403
+ "pass@1": 0.17212775735294117,
404
+ "pass@2": 0.23196720125057887,
405
+ "pass@4": 0.2933669291338581,
406
+ "pass@8": 0.35235556676313684,
407
+ "pass@16": 0.4034011348916748,
408
+ "pass@32": 0.4468507861160033,
409
+ "pass@64": 0.48950582673115856,
410
+ "pass@128": 0.5352941176470588
411
+ },
412
+ "19": {
413
+ "pass@1": 0.15087890625,
414
+ "pass@2": 0.2072612017247844,
415
+ "pass@4": 0.2605428718731588,
416
+ "pass@8": 0.3074317250275432,
417
+ "pass@16": 0.34908958061974404,
418
+ "pass@32": 0.3879378881426714,
419
+ "pass@64": 0.42367879660082997,
420
+ "pass@128": 0.45535714285714285
421
+ },
422
+ "20": {
423
+ "pass@1": 0.1435254491017964,
424
+ "pass@2": 0.20275848401621951,
425
+ "pass@4": 0.26188721965517786,
426
+ "pass@8": 0.31475370332114117,
427
+ "pass@16": 0.3611758484951675,
428
+ "pass@32": 0.40307471926923255,
429
+ "pass@64": 0.43896682078335747,
430
+ "pass@128": 0.47005988023952094
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 518583,
437
+ "answer_accuracy": 0.6472966428343185,
438
+ "avg_response_len": 279.3217104868989,
439
+ "resp_tokens_sum": 223779147,
440
+ "pass_at_k": {
441
+ "pass@1": 0.6472966428343185,
442
+ "pass@2": 0.687566046625146,
443
+ "pass@4": 0.7213202171624994,
444
+ "pass@8": 0.7503762891676571,
445
+ "pass@16": 0.7756265763304547,
446
+ "pass@32": 0.7975734180227835,
447
+ "pass@64": 0.8168150755671897,
448
+ "pass@128": 0.8335197315865154
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9071595149253732,
453
+ "pass@2": 0.9365723057938655,
454
+ "pass@4": 0.9556170069226421,
455
+ "pass@8": 0.9696769057695721,
456
+ "pass@16": 0.9798609945844656,
457
+ "pass@32": 0.9858142874379154,
458
+ "pass@64": 0.9890143934765259,
459
+ "pass@128": 0.991044776119403
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9894642857142857,
463
+ "pass@2": 0.99220683352081,
464
+ "pass@4": 0.9938244295355938,
465
+ "pass@8": 0.9945718767910509,
466
+ "pass@16": 0.9949991404358748,
467
+ "pass@32": 0.9957142856718992,
468
+ "pass@64": 0.9971428571428571,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9928935860058309,
473
+ "pass@2": 0.9970085053143868,
474
+ "pass@4": 0.9992781772358628,
475
+ "pass@8": 0.9999411811374931,
476
+ "pass@16": 0.9999995372325835,
477
+ "pass@32": 0.9999999999908739,
478
+ "pass@64": 1.0,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9961851242690059,
483
+ "pass@2": 0.9976123975457015,
484
+ "pass@4": 0.998593672227375,
485
+ "pass@8": 0.9993826865860662,
486
+ "pass@16": 0.99988448784076,
487
+ "pass@32": 0.9999973942969671,
488
+ "pass@64": 0.9999999999215047,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9512408088235295,
493
+ "pass@2": 0.9592754892311257,
494
+ "pass@4": 0.9672070549085778,
495
+ "pass@8": 0.9742943951584943,
496
+ "pass@16": 0.9797759062977376,
497
+ "pass@32": 0.9843737785787031,
498
+ "pass@64": 0.9896992479161258,
499
+ "pass@128": 0.9970588235294118
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9752316640378549,
503
+ "pass@2": 0.9804542928289322,
504
+ "pass@4": 0.9846770751645001,
505
+ "pass@8": 0.9885439412426841,
506
+ "pass@16": 0.9913278776467133,
507
+ "pass@32": 0.9933394230271457,
508
+ "pass@64": 0.9950781489247041,
509
+ "pass@128": 0.9968454258675079
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9796528876582279,
513
+ "pass@2": 0.985694473861258,
514
+ "pass@4": 0.9887304917581505,
515
+ "pass@8": 0.9907698483062334,
516
+ "pass@16": 0.9927418508415174,
517
+ "pass@32": 0.994748499944022,
518
+ "pass@64": 0.996357980344224,
519
+ "pass@128": 0.9968354430379747
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9564329954954955,
523
+ "pass@2": 0.9685896526447708,
524
+ "pass@4": 0.9777987281432166,
525
+ "pass@8": 0.9852023893939588,
526
+ "pass@16": 0.9905368812834143,
527
+ "pass@32": 0.9943325122952712,
528
+ "pass@64": 0.9973765202133318,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.96241961414791,
533
+ "pass@2": 0.9755305782717674,
534
+ "pass@4": 0.9802783508572683,
535
+ "pass@8": 0.9834987228210043,
536
+ "pass@16": 0.9869310415394815,
537
+ "pass@32": 0.9902900110658145,
538
+ "pass@64": 0.9926323245605343,
539
+ "pass@128": 0.9935691318327974
540
+ },
541
+ "11": {
542
+ "pass@1": 0.8774553571428572,
543
+ "pass@2": 0.9201056899137611,
544
+ "pass@4": 0.9411811779479943,
545
+ "pass@8": 0.9534647217495896,
546
+ "pass@16": 0.9630343209400213,
547
+ "pass@32": 0.9713331761600971,
548
+ "pass@64": 0.9783436058997058,
549
+ "pass@128": 0.9809523809523809
550
+ },
551
+ "12": {
552
+ "pass@1": 0.6939506880733946,
553
+ "pass@2": 0.7832756928893064,
554
+ "pass@4": 0.8455138635193533,
555
+ "pass@8": 0.8890513361030553,
556
+ "pass@16": 0.9183177835907471,
557
+ "pass@32": 0.9375012531153086,
558
+ "pass@64": 0.9500191762654265,
559
+ "pass@128": 0.9571865443425076
560
+ },
561
+ "13": {
562
+ "pass@1": 0.4593460648148148,
563
+ "pass@2": 0.550352158671138,
564
+ "pass@4": 0.6256323665560319,
565
+ "pass@8": 0.6875551867759847,
566
+ "pass@16": 0.7360322879202137,
567
+ "pass@32": 0.7765612944510213,
568
+ "pass@64": 0.8141867170120144,
569
+ "pass@128": 0.8518518518518519
570
+ },
571
+ "14": {
572
+ "pass@1": 0.38234885014836795,
573
+ "pass@2": 0.46612764024860376,
574
+ "pass@4": 0.5396942860814502,
575
+ "pass@8": 0.6044605719250148,
576
+ "pass@16": 0.6604712523124485,
577
+ "pass@32": 0.7068676293724594,
578
+ "pass@64": 0.7443352257906309,
579
+ "pass@128": 0.7744807121661721
580
+ },
581
+ "15": {
582
+ "pass@1": 0.2726417824074074,
583
+ "pass@2": 0.3460583594585398,
584
+ "pass@4": 0.41217337416156297,
585
+ "pass@8": 0.4727927552375029,
586
+ "pass@16": 0.5298785998494511,
587
+ "pass@32": 0.5832519581605827,
588
+ "pass@64": 0.6289946833739761,
589
+ "pass@128": 0.6635802469135802
590
+ },
591
+ "16": {
592
+ "pass@1": 0.20927820796460178,
593
+ "pass@2": 0.2710314902561958,
594
+ "pass@4": 0.32689492325627434,
595
+ "pass@8": 0.37984261448337203,
596
+ "pass@16": 0.43272626830354155,
597
+ "pass@32": 0.48435329728914633,
598
+ "pass@64": 0.5332086540560117,
599
+ "pass@128": 0.5781710914454278
600
+ },
601
+ "17": {
602
+ "pass@1": 0.17561652861445784,
603
+ "pass@2": 0.2379966470211554,
604
+ "pass@4": 0.29631255251602584,
605
+ "pass@8": 0.3500892982483715,
606
+ "pass@16": 0.4004591753225813,
607
+ "pass@32": 0.44598628696095505,
608
+ "pass@64": 0.4889772959528953,
609
+ "pass@128": 0.536144578313253
610
+ },
611
+ "18": {
612
+ "pass@1": 0.18268649193548386,
613
+ "pass@2": 0.24208748729997448,
614
+ "pass@4": 0.3009205241481911,
615
+ "pass@8": 0.35592454781716937,
616
+ "pass@16": 0.40402308504606516,
617
+ "pass@32": 0.44578225999860455,
618
+ "pass@64": 0.4843893762241014,
619
+ "pass@128": 0.5161290322580645
620
+ },
621
+ "19": {
622
+ "pass@1": 0.15315804380664652,
623
+ "pass@2": 0.21352803780003343,
624
+ "pass@4": 0.27246658895734704,
625
+ "pass@8": 0.32722728747094926,
626
+ "pass@16": 0.3772414787512142,
627
+ "pass@32": 0.41963885160277065,
628
+ "pass@64": 0.4509154252350958,
629
+ "pass@128": 0.4743202416918429
630
+ },
631
+ "20": {
632
+ "pass@1": 0.16246715465465467,
633
+ "pass@2": 0.22468765517485983,
634
+ "pass@4": 0.284834960100708,
635
+ "pass@8": 0.33897903867127327,
636
+ "pass@16": 0.3879200126258433,
637
+ "pass@32": 0.4341533925251601,
638
+ "pass@64": 0.4796573665199342,
639
+ "pass@128": 0.5195195195195195
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f437557eb8eecb54fa5efde88957ea338a563d8852db4b057dff138f33de7be2
3
+ size 1792857151
results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch1-op9-12",
3
+ "id": {
4
+ "answer_accuracy": 0.9653489583333333,
5
+ "avg_response_len": 199.63633854166667,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9653489583333333,
11
+ "pass@2": 0.9760651110017445,
12
+ "pass@4": 0.9827205345269335,
13
+ "pass@8": 0.9872042269695149,
14
+ "pass@16": 0.990437182192509,
15
+ "pass@32": 0.9929618421362073,
16
+ "pass@64": 0.994864366664554,
17
+ "pass@128": 0.996
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9176640625,
22
+ "pass@2": 0.940686146653544,
23
+ "pass@4": 0.9572751454818151,
24
+ "pass@8": 0.9697146074131044,
25
+ "pass@16": 0.9786371901621834,
26
+ "pass@32": 0.9846897936578861,
27
+ "pass@64": 0.9886130167733074,
28
+ "pass@128": 0.991
29
+ },
30
+ "2": {
31
+ "pass@1": 0.988515625,
32
+ "pass@2": 0.9917391732283464,
33
+ "pass@4": 0.9931179612860892,
34
+ "pass@8": 0.9939276029971762,
35
+ "pass@16": 0.9945250737957444,
36
+ "pass@32": 0.995013888448554,
37
+ "pass@64": 0.9954711788593589,
38
+ "pass@128": 0.996
39
+ },
40
+ "3": {
41
+ "pass@1": 0.993640625,
42
+ "pass@2": 0.997156619094488,
43
+ "pass@4": 0.9985739075740535,
44
+ "pass@8": 0.9990783257375846,
45
+ "pass@16": 0.9993271435678454,
46
+ "pass@32": 0.9995814496671865,
47
+ "pass@64": 0.9998779527559054,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9908671875,
52
+ "pass@2": 0.9935773868110238,
53
+ "pass@4": 0.9958510903637046,
54
+ "pass@8": 0.9974042594396715,
55
+ "pass@16": 0.9983575185071549,
56
+ "pass@32": 0.9991406729482755,
57
+ "pass@64": 0.9997315728158566,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.9576875,
62
+ "pass@2": 0.9662279773622043,
63
+ "pass@4": 0.9724250765841771,
64
+ "pass@8": 0.977084930700057,
65
+ "pass@16": 0.981167244820512,
66
+ "pass@32": 0.9850588622178171,
67
+ "pass@64": 0.9886003349090352,
68
+ "pass@128": 0.991
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9621640625,
72
+ "pass@2": 0.9733558070866145,
73
+ "pass@4": 0.9805783572365954,
74
+ "pass@8": 0.9853265539365899,
75
+ "pass@16": 0.9881276610501337,
76
+ "pass@32": 0.9902692290508687,
77
+ "pass@64": 0.9921569678703794,
78
+ "pass@128": 0.993
79
+ },
80
+ "7": {
81
+ "pass@1": 0.969,
82
+ "pass@2": 0.9790697588582676,
83
+ "pass@4": 0.9853942632170978,
84
+ "pass@8": 0.989212366296118,
85
+ "pass@16": 0.9919174897126309,
86
+ "pass@32": 0.9943850588471859,
87
+ "pass@64": 0.9962791607703998,
88
+ "pass@128": 0.997
89
+ },
90
+ "8": {
91
+ "pass@1": 0.9544375,
92
+ "pass@2": 0.9708692175196855,
93
+ "pass@4": 0.9808557603112115,
94
+ "pass@8": 0.9876466681787328,
95
+ "pass@16": 0.992855800260977,
96
+ "pass@32": 0.9967386933633018,
97
+ "pass@64": 0.9989564749136509,
98
+ "pass@128": 1.0
99
+ },
100
+ "9": {
101
+ "pass@1": 0.9541640625,
102
+ "pass@2": 0.9719039124015749,
103
+ "pass@4": 0.9804132486876638,
104
+ "pass@8": 0.9854427280266015,
105
+ "pass@16": 0.9890195178553929,
106
+ "pass@32": 0.99177893102479,
107
+ "pass@64": 0.9940926403130936,
108
+ "pass@128": 0.996
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9176640625,
113
+ "2": 0.988515625,
114
+ "3": 0.993640625,
115
+ "4": 0.9908671875,
116
+ "5": 0.9576875,
117
+ "6": 0.9621640625,
118
+ "7": 0.969,
119
+ "8": 0.9544375,
120
+ "9": 0.9541640625
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 278.3240859375,
124
+ "2": 116.5158515625,
125
+ "3": 144.4954140625,
126
+ "4": 159.5400078125,
127
+ "5": 174.3931875,
128
+ "6": 199.3752578125,
129
+ "7": 223.4614296875,
130
+ "8": 242.72865625,
131
+ "9": 257.89315625
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 278.3240859375,
136
+ "2": 116.5158515625,
137
+ "3": 144.4954140625,
138
+ "4": 159.5400078125,
139
+ "5": 174.3931875,
140
+ "6": 199.3752578125,
141
+ "7": 223.4614296875,
142
+ "8": 242.72865625,
143
+ "9": 257.89315625
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 364498,
150
+ "answer_accuracy": 0.9617158476865924,
151
+ "avg_response_len": 190.16717852921312,
152
+ "resp_tokens_sum": 72074882,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9617158476865924,
155
+ "pass@2": 0.9740216149842975,
156
+ "pass@4": 0.9816819834894001,
157
+ "pass@8": 0.9866704765932208,
158
+ "pass@16": 0.9901795736650141,
159
+ "pass@32": 0.9931229258326321,
160
+ "pass@64": 0.9954760430923487,
161
+ "pass@128": 0.9966227625802094
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9215984513274337,
166
+ "pass@2": 0.9421719305274897,
167
+ "pass@4": 0.9578695823973329,
168
+ "pass@8": 0.9698981562700201,
169
+ "pass@16": 0.9786144975781491,
170
+ "pass@32": 0.9852228034323779,
171
+ "pass@64": 0.9899066079157088,
172
+ "pass@128": 0.9911504424778761
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9882204049844237,
176
+ "pass@2": 0.9918561581671449,
177
+ "pass@4": 0.9931930275654794,
178
+ "pass@8": 0.9936832086849384,
179
+ "pass@16": 0.993767744760047,
180
+ "pass@32": 0.9937694702012028,
181
+ "pass@64": 0.9937694704049844,
182
+ "pass@128": 0.9937694704049844
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9939554663608563,
186
+ "pass@2": 0.9985781772977917,
187
+ "pass@4": 0.999890842199771,
188
+ "pass@8": 0.9999992999127619,
189
+ "pass@16": 0.9999999999827538,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.990234375,
196
+ "pass@2": 0.99342742679626,
197
+ "pass@4": 0.9958457609790964,
198
+ "pass@8": 0.9970974256312388,
199
+ "pass@16": 0.9976077641280157,
200
+ "pass@32": 0.9982468010091103,
201
+ "pass@64": 0.9992249015748031,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.948139880952381,
206
+ "pass@2": 0.9574174321959756,
207
+ "pass@4": 0.9648101639080825,
208
+ "pass@8": 0.971654004642766,
209
+ "pass@16": 0.9780823665716158,
210
+ "pass@32": 0.9842143450259111,
211
+ "pass@64": 0.9898102076387585,
212
+ "pass@128": 0.9936507936507937
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9406811377245509,
216
+ "pass@2": 0.9580407344759303,
217
+ "pass@4": 0.9713076608313183,
218
+ "pass@8": 0.9802488166001808,
219
+ "pass@16": 0.9856429524545877,
220
+ "pass@32": 0.9897026831061002,
221
+ "pass@64": 0.9927742684651953,
222
+ "pass@128": 0.9940119760479041
223
+ },
224
+ "7": {
225
+ "pass@1": 0.96875,
226
+ "pass@2": 0.980806722615664,
227
+ "pass@4": 0.988206052553342,
228
+ "pass@8": 0.9921328277288592,
229
+ "pass@16": 0.9942489886848938,
230
+ "pass@32": 0.9958450340249061,
231
+ "pass@64": 0.9968560505325343,
232
+ "pass@128": 0.9970414201183432
233
+ },
234
+ "8": {
235
+ "pass@1": 0.955879407051282,
236
+ "pass@2": 0.9741089680496671,
237
+ "pass@4": 0.9835598903502444,
238
+ "pass@8": 0.9884050733900349,
239
+ "pass@16": 0.9918600204545246,
240
+ "pass@32": 0.9954140492926473,
241
+ "pass@64": 0.9986018037303472,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9509242957746479,
246
+ "pass@2": 0.9714289675058222,
247
+ "pass@4": 0.9813418648373179,
248
+ "pass@8": 0.9873102931793774,
249
+ "pass@16": 0.991888158345391,
250
+ "pass@32": 0.9956463578845761,
251
+ "pass@64": 0.9982995101172762,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 378550,
259
+ "answer_accuracy": 0.9690110992791612,
260
+ "avg_response_len": 197.52059868528835,
261
+ "resp_tokens_sum": 77162607,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9690110992791612,
264
+ "pass@2": 0.978732119315075,
265
+ "pass@4": 0.9846058772060452,
266
+ "pass@8": 0.9885791851961677,
267
+ "pass@16": 0.991478696028919,
268
+ "pass@32": 0.9936181979373836,
269
+ "pass@64": 0.9950741600217173,
270
+ "pass@128": 0.9960681520314548
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9162432898773006,
275
+ "pass@2": 0.936796154171296,
276
+ "pass@4": 0.9512743250843644,
277
+ "pass@8": 0.9633956921405639,
278
+ "pass@16": 0.9736279940224598,
279
+ "pass@32": 0.9804423382093927,
280
+ "pass@64": 0.9842310694963466,
281
+ "pass@128": 0.9877300613496932
282
+ },
283
+ "2": {
284
+ "pass@1": 0.9920687689969605,
285
+ "pass@2": 0.9949422462484743,
286
+ "pass@4": 0.9961704575112003,
287
+ "pass@8": 0.9969137768076869,
288
+ "pass@16": 0.997325481498739,
289
+ "pass@32": 0.9977203377335374,
290
+ "pass@64": 0.9984802431610939,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9942708333333333,
295
+ "pass@2": 0.9958694971367216,
296
+ "pass@4": 0.9967973148242834,
297
+ "pass@8": 0.9974456186860212,
298
+ "pass@16": 0.9979759402219975,
299
+ "pass@32": 0.9987316925844025,
300
+ "pass@64": 0.9996301598663804,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9875416050295858,
305
+ "pass@2": 0.9915100987746355,
306
+ "pass@4": 0.9949397878519625,
307
+ "pass@8": 0.9972912179822564,
308
+ "pass@16": 0.9987250305872443,
309
+ "pass@32": 0.9996231767711692,
310
+ "pass@64": 0.9999805562449308,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9740942028985508,
315
+ "pass@2": 0.9819546673513638,
316
+ "pass@4": 0.987201654956174,
317
+ "pass@8": 0.9904344185485051,
318
+ "pass@16": 0.9930448124857775,
319
+ "pass@32": 0.9952773429073588,
320
+ "pass@64": 0.9967067319810339,
321
+ "pass@128": 0.9971014492753624
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9741001074498568,
325
+ "pass@2": 0.982676178282156,
326
+ "pass@4": 0.9865603822265768,
327
+ "pass@8": 0.9885311263764657,
328
+ "pass@16": 0.9894516215507114,
329
+ "pass@32": 0.9902045618641498,
330
+ "pass@64": 0.9910543058907616,
331
+ "pass@128": 0.9914040114613181
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9690435332369942,
335
+ "pass@2": 0.9785164250603066,
336
+ "pass@4": 0.9848848110929772,
337
+ "pass@8": 0.9886472236910547,
338
+ "pass@16": 0.991308776141372,
339
+ "pass@32": 0.993989628100775,
340
+ "pass@64": 0.9961800005089887,
341
+ "pass@128": 0.9971098265895953
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9597271126760564,
345
+ "pass@2": 0.9766656038593767,
346
+ "pass@4": 0.9864835997612974,
347
+ "pass@8": 0.992742027879815,
348
+ "pass@16": 0.9970229470709905,
349
+ "pass@32": 0.9994615576509764,
350
+ "pass@64": 0.9999920569141826,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9533589071856288,
355
+ "pass@2": 0.9688030435192612,
356
+ "pass@4": 0.9762362204724409,
357
+ "pass@8": 0.9810118633763555,
358
+ "pass@16": 0.9841736264077786,
359
+ "pass@32": 0.9865662118269425,
360
+ "pass@64": 0.988977219442525,
361
+ "pass@128": 0.9910179640718563
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 369034,
368
+ "answer_accuracy": 0.9652086123200536,
369
+ "avg_response_len": 211.1848557289923,
370
+ "resp_tokens_sum": 80743573,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9652086123200536,
373
+ "pass@2": 0.9753657746560565,
374
+ "pass@4": 0.981823676062086,
375
+ "pass@8": 0.986328452733312,
376
+ "pass@16": 0.9896283702143321,
377
+ "pass@32": 0.9921315218381473,
378
+ "pass@64": 0.9940436558413996,
379
+ "pass@128": 0.9953130231001004
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9150652985074627,
384
+ "pass@2": 0.9429681072981553,
385
+ "pass@4": 0.9625132151391523,
386
+ "pass@8": 0.9756780202977416,
387
+ "pass@16": 0.9835347744234902,
388
+ "pass@32": 0.9882837642926804,
389
+ "pass@64": 0.9915682030867851,
390
+ "pass@128": 0.9940298507462687
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9854464285714286,
394
+ "pass@2": 0.988620992688414,
395
+ "pass@4": 0.9901797683325299,
396
+ "pass@8": 0.991344744113091,
397
+ "pass@16": 0.9925872694705262,
398
+ "pass@32": 0.9936111354275262,
399
+ "pass@64": 0.9942033681695974,
400
+ "pass@128": 0.9942857142857143
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9927341472303207,
404
+ "pass@2": 0.9970397115539129,
405
+ "pass@4": 0.9990276626318211,
406
+ "pass@8": 0.9997711384831617,
407
+ "pass@16": 0.9999856655983261,
408
+ "pass@32": 0.9999999740942677,
409
+ "pass@64": 0.9999999999999997,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9947459795321637,
414
+ "pass@2": 0.9957608095040752,
415
+ "pass@4": 0.9967567209252353,
416
+ "pass@8": 0.997803074735884,
417
+ "pass@16": 0.9986958293792437,
418
+ "pass@32": 0.9995001838500145,
419
+ "pass@64": 0.9999595798278741,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9498851102941176,
424
+ "pass@2": 0.9584327234830942,
425
+ "pass@4": 0.9644861882154439,
426
+ "pass@8": 0.968570720112799,
427
+ "pass@16": 0.9719730560084113,
428
+ "pass@32": 0.9754725300929893,
429
+ "pass@64": 0.979253814615793,
430
+ "pass@128": 0.9823529411764705
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9716581230283912,
434
+ "pass@2": 0.979230963883852,
435
+ "pass@4": 0.9837603316146996,
436
+ "pass@8": 0.9871485365512419,
437
+ "pass@16": 0.9892880094924388,
438
+ "pass@32": 0.9909373526903568,
439
+ "pass@64": 0.992720534533528,
440
+ "pass@128": 0.9936908517350158
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9692197389240507,
444
+ "pass@2": 0.9778177327319845,
445
+ "pass@4": 0.9829445342275255,
446
+ "pass@8": 0.9867073769830967,
447
+ "pass@16": 0.9900901740260195,
448
+ "pass@32": 0.9932564114680369,
449
+ "pass@64": 0.995770682007257,
450
+ "pass@128": 0.9968354430379747
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9474474474474475,
454
+ "pass@2": 0.96165444479913,
455
+ "pass@4": 0.9723225123323547,
456
+ "pass@8": 0.9815041002513741,
457
+ "pass@16": 0.9893463293362276,
458
+ "pass@32": 0.9950770481017992,
459
+ "pass@64": 0.9981847806163591,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9587268890675241,
464
+ "pass@2": 0.9757762456642275,
465
+ "pass@4": 0.983839193030935,
466
+ "pass@8": 0.9880694906116392,
467
+ "pass@16": 0.9909492939632827,
468
+ "pass@32": 0.9929625698571264,
469
+ "pass@64": 0.9947843180953608,
470
+ "pass@128": 0.9967845659163987
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.4310265625,
478
+ "avg_response_len": 339.695190625,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.4310265625,
484
+ "pass@2": 0.5022684793307036,
485
+ "pass@4": 0.5631944049868662,
486
+ "pass@8": 0.6146080622451209,
487
+ "pass@16": 0.658063651886774,
488
+ "pass@32": 0.6956868152191318,
489
+ "pass@64": 0.7288434322106975,
490
+ "pass@128": 0.7572
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.897328125,
495
+ "pass@2": 0.93245250984252,
496
+ "pass@4": 0.9525756509186349,
497
+ "pass@8": 0.9645405289253838,
498
+ "pass@16": 0.9721646148438103,
499
+ "pass@32": 0.9774677539333159,
500
+ "pass@64": 0.9818126562253906,
501
+ "pass@128": 0.986
502
+ },
503
+ "12": {
504
+ "pass@1": 0.83025,
505
+ "pass@2": 0.8788818897637795,
506
+ "pass@4": 0.9077505954255725,
507
+ "pass@8": 0.9279219926000699,
508
+ "pass@16": 0.9429205001103443,
509
+ "pass@32": 0.9540556965113227,
510
+ "pass@64": 0.9624023478061977,
511
+ "pass@128": 0.968
512
+ },
513
+ "13": {
514
+ "pass@1": 0.6266796875,
515
+ "pass@2": 0.7173414124015755,
516
+ "pass@4": 0.7835768498312717,
517
+ "pass@8": 0.8310638378335047,
518
+ "pass@16": 0.8652153962130704,
519
+ "pass@32": 0.8921147478710862,
520
+ "pass@64": 0.9136172681639906,
521
+ "pass@128": 0.93
522
+ },
523
+ "14": {
524
+ "pass@1": 0.5015,
525
+ "pass@2": 0.589535925196851,
526
+ "pass@4": 0.6631350288713906,
527
+ "pass@8": 0.7234786940771628,
528
+ "pass@16": 0.7724374351005907,
529
+ "pass@32": 0.8117817021972529,
530
+ "pass@64": 0.8433248048181223,
531
+ "pass@128": 0.869
532
+ },
533
+ "15": {
534
+ "pass@1": 0.385046875,
535
+ "pass@2": 0.47474187992126016,
536
+ "pass@4": 0.5506543599550056,
537
+ "pass@8": 0.6135148960829805,
538
+ "pass@16": 0.6656588575667085,
539
+ "pass@32": 0.7095550865297331,
540
+ "pass@64": 0.7464043576049708,
541
+ "pass@128": 0.775
542
+ },
543
+ "16": {
544
+ "pass@1": 0.280765625,
545
+ "pass@2": 0.3638841043307086,
546
+ "pass@4": 0.43851758830146187,
547
+ "pass@8": 0.5031006368868065,
548
+ "pass@16": 0.5591009613517215,
549
+ "pass@32": 0.6095957808194802,
550
+ "pass@64": 0.6556664450191428,
551
+ "pass@128": 0.694
552
+ },
553
+ "17": {
554
+ "pass@1": 0.2475625,
555
+ "pass@2": 0.3250761564960633,
556
+ "pass@4": 0.39567444216347986,
557
+ "pass@8": 0.45756658041957493,
558
+ "pass@16": 0.5128672951589743,
559
+ "pass@32": 0.5644771191304162,
560
+ "pass@64": 0.6129670466840049,
561
+ "pass@128": 0.656
562
+ },
563
+ "18": {
564
+ "pass@1": 0.21215625,
565
+ "pass@2": 0.2847347440944884,
566
+ "pass@4": 0.35507883173978216,
567
+ "pass@8": 0.4180793864445228,
568
+ "pass@16": 0.4726955368933959,
569
+ "pass@32": 0.5197814062778024,
570
+ "pass@64": 0.5618646594106671,
571
+ "pass@128": 0.601
572
+ },
573
+ "19": {
574
+ "pass@1": 0.1663125,
575
+ "pass@2": 0.22953531003937006,
576
+ "pass@4": 0.2925731046119234,
577
+ "pass@8": 0.35215195152385936,
578
+ "pass@16": 0.4065216822086127,
579
+ "pass@32": 0.4569250687655832,
580
+ "pass@64": 0.5050905644534417,
581
+ "pass@128": 0.55
582
+ },
583
+ "20": {
584
+ "pass@1": 0.1626640625,
585
+ "pass@2": 0.22650086122047253,
586
+ "pass@4": 0.2924075980502435,
587
+ "pass@8": 0.3546621176573502,
588
+ "pass@16": 0.41105423942037367,
589
+ "pass@32": 0.4611137901553134,
590
+ "pass@64": 0.505284171920995,
591
+ "pass@128": 0.543
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.897328125,
596
+ "12": 0.83025,
597
+ "13": 0.6266796875,
598
+ "14": 0.5015,
599
+ "15": 0.385046875,
600
+ "16": 0.280765625,
601
+ "17": 0.2475625,
602
+ "18": 0.21215625,
603
+ "19": 0.1663125,
604
+ "20": 0.1626640625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 295.7106484375,
608
+ "12": 306.1600703125,
609
+ "13": 321.2292265625,
610
+ "14": 337.66740625,
611
+ "15": 344.118015625,
612
+ "16": 350.866875,
613
+ "17": 360.0460703125,
614
+ "18": 359.24096875,
615
+ "19": 359.20896875,
616
+ "20": 362.70365625
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 295.7106484375,
621
+ "12": 306.1600703125,
622
+ "13": 321.2292265625,
623
+ "14": 337.66740625,
624
+ "15": 344.118015625,
625
+ "16": 350.866875,
626
+ "17": 360.0460703125,
627
+ "18": 359.24096875,
628
+ "19": 359.20896875,
629
+ "20": 362.70365625
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 186658,
636
+ "answer_accuracy": 0.43336274145616643,
637
+ "avg_response_len": 322.15230544205053,
638
+ "resp_tokens_sum": 138757441,
639
+ "pass_at_k": {
640
+ "pass@1": 0.43336274145616643,
641
+ "pass@2": 0.5036887145932549,
642
+ "pass@4": 0.563023118395492,
643
+ "pass@8": 0.6126057036749238,
644
+ "pass@16": 0.6542006347449212,
645
+ "pass@32": 0.6899950752564268,
646
+ "pass@64": 0.7213229495910691,
647
+ "pass@128": 0.7465081723625557
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.8825469192634561,
652
+ "pass@2": 0.9204746018380142,
653
+ "pass@4": 0.9434641893205277,
654
+ "pass@8": 0.9584269403349234,
655
+ "pass@16": 0.9686552276126651,
656
+ "pass@32": 0.9741207162725538,
657
+ "pass@64": 0.9764454185753144,
658
+ "pass@128": 0.9773371104815864
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8271539548022598,
662
+ "pass@2": 0.8798437569509316,
663
+ "pass@4": 0.9131463289016838,
664
+ "pass@8": 0.9351818057985669,
665
+ "pass@16": 0.9499145912403741,
666
+ "pass@32": 0.9615198784184202,
667
+ "pass@64": 0.9709481160243175,
668
+ "pass@128": 0.9745762711864406
669
+ },
670
+ "13": {
671
+ "pass@1": 0.6312695924764891,
672
+ "pass@2": 0.7226765945498974,
673
+ "pass@4": 0.7903836963561368,
674
+ "pass@8": 0.838930594168827,
675
+ "pass@16": 0.8732006163685243,
676
+ "pass@32": 0.9012810590070837,
677
+ "pass@64": 0.9258364518367349,
678
+ "pass@128": 0.9435736677115988
679
+ },
680
+ "14": {
681
+ "pass@1": 0.4956678348909657,
682
+ "pass@2": 0.5795753152059264,
683
+ "pass@4": 0.6470877296879942,
684
+ "pass@8": 0.7036891591817919,
685
+ "pass@16": 0.7513465261882352,
686
+ "pass@32": 0.7915273102114493,
687
+ "pass@64": 0.8268042808049906,
688
+ "pass@128": 0.8598130841121495
689
+ },
690
+ "15": {
691
+ "pass@1": 0.3917765534682081,
692
+ "pass@2": 0.47504665240544336,
693
+ "pass@4": 0.5456971947328837,
694
+ "pass@8": 0.6072410800646598,
695
+ "pass@16": 0.6594419230378626,
696
+ "pass@32": 0.7028996498231647,
697
+ "pass@64": 0.7405890497226499,
698
+ "pass@128": 0.7658959537572254
699
+ },
700
+ "16": {
701
+ "pass@1": 0.26927923387096775,
702
+ "pass@2": 0.3519208788417577,
703
+ "pass@4": 0.42269466175599024,
704
+ "pass@8": 0.4817823935686336,
705
+ "pass@16": 0.5343682207500322,
706
+ "pass@32": 0.582402508764003,
707
+ "pass@64": 0.6250428507581135,
708
+ "pass@128": 0.6612903225806451
709
+ },
710
+ "17": {
711
+ "pass@1": 0.25090317919075145,
712
+ "pass@2": 0.3269800248054254,
713
+ "pass@4": 0.39415536226324305,
714
+ "pass@8": 0.45047162566799187,
715
+ "pass@16": 0.4981191025185966,
716
+ "pass@32": 0.5403608962010116,
717
+ "pass@64": 0.5799445864824998,
718
+ "pass@128": 0.6184971098265896
719
+ },
720
+ "18": {
721
+ "pass@1": 0.21515625,
722
+ "pass@2": 0.28855982845894257,
723
+ "pass@4": 0.3581241681396968,
724
+ "pass@8": 0.4172596510922582,
725
+ "pass@16": 0.46627537268369557,
726
+ "pass@32": 0.5088608026804442,
727
+ "pass@64": 0.5478183836857043,
728
+ "pass@128": 0.5828571428571429
729
+ },
730
+ "19": {
731
+ "pass@1": 0.16713588588588588,
732
+ "pass@2": 0.23041065179352582,
733
+ "pass@4": 0.2955489223756941,
734
+ "pass@8": 0.35859808935671944,
735
+ "pass@16": 0.4160070631029604,
736
+ "pass@32": 0.468307236198052,
737
+ "pass@64": 0.5149739853037154,
738
+ "pass@128": 0.5495495495495496
739
+ },
740
+ "20": {
741
+ "pass@1": 0.17004504504504506,
742
+ "pass@2": 0.23310330507673024,
743
+ "pass@4": 0.2956246063273622,
744
+ "pass@8": 0.35363622885925766,
745
+ "pass@16": 0.4070566006447026,
746
+ "pass@32": 0.4541781885859068,
747
+ "pass@64": 0.4931356276630924,
748
+ "pass@128": 0.5225225225225225
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 183610,
755
+ "answer_accuracy": 0.42653973386856975,
756
+ "avg_response_len": 334.73815696550696,
757
+ "resp_tokens_sum": 144092726,
758
+ "pass_at_k": {
759
+ "pass@1": 0.42653973386856975,
760
+ "pass@2": 0.4969607598671056,
761
+ "pass@4": 0.557970116246622,
762
+ "pass@8": 0.6104187474444682,
763
+ "pass@16": 0.6550365457245686,
764
+ "pass@32": 0.6931812007460348,
765
+ "pass@64": 0.7262481004101062,
766
+ "pass@128": 0.7555753791257805
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9048616340361446,
771
+ "pass@2": 0.9367618110236223,
772
+ "pass@4": 0.9549433162910352,
773
+ "pass@8": 0.964925632387157,
774
+ "pass@16": 0.9708902210043483,
775
+ "pass@32": 0.9769209821868238,
776
+ "pass@64": 0.9840171200594384,
777
+ "pass@128": 0.9909638554216867
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8286637931034483,
781
+ "pass@2": 0.8699518518747071,
782
+ "pass@4": 0.8934778219337001,
783
+ "pass@8": 0.9107875074125524,
784
+ "pass@16": 0.924846496240714,
785
+ "pass@32": 0.936201270194174,
786
+ "pass@64": 0.9454492653916154,
787
+ "pass@128": 0.9529780564263323
788
+ },
789
+ "13": {
790
+ "pass@1": 0.6203825280112045,
791
+ "pass@2": 0.7105237764397101,
792
+ "pass@4": 0.7767815107670366,
793
+ "pass@8": 0.8251601495483123,
794
+ "pass@16": 0.860732014990156,
795
+ "pass@32": 0.8879965216471163,
796
+ "pass@64": 0.9088056396870849,
797
+ "pass@128": 0.9243697478991597
798
+ },
799
+ "14": {
800
+ "pass@1": 0.4910453216374269,
801
+ "pass@2": 0.5799452905557857,
802
+ "pass@4": 0.6568933461168229,
803
+ "pass@8": 0.7221252947493525,
804
+ "pass@16": 0.7770240520072819,
805
+ "pass@32": 0.8219059508512782,
806
+ "pass@64": 0.8560598866085424,
807
+ "pass@128": 0.8830409356725146
808
+ },
809
+ "15": {
810
+ "pass@1": 0.3689157196969697,
811
+ "pass@2": 0.46035552374135,
812
+ "pass@4": 0.5369870879776393,
813
+ "pass@8": 0.5998998198723953,
814
+ "pass@16": 0.6521363093899889,
815
+ "pass@32": 0.6950081588345737,
816
+ "pass@64": 0.7272367323897406,
817
+ "pass@128": 0.7515151515151515
818
+ },
819
+ "16": {
820
+ "pass@1": 0.278957443019943,
821
+ "pass@2": 0.3574475065616797,
822
+ "pass@4": 0.4265418227636073,
823
+ "pass@8": 0.48653231264461394,
824
+ "pass@16": 0.5386797703066138,
825
+ "pass@32": 0.585372520779792,
826
+ "pass@64": 0.6303605102283197,
827
+ "pass@128": 0.6723646723646723
828
+ },
829
+ "17": {
830
+ "pass@1": 0.2574970885093168,
831
+ "pass@2": 0.33939995781777277,
832
+ "pass@4": 0.4177574955692649,
833
+ "pass@8": 0.49007486766441977,
834
+ "pass@16": 0.5536131251915021,
835
+ "pass@32": 0.6097010001622978,
836
+ "pass@64": 0.6588642656789311,
837
+ "pass@128": 0.6987577639751553
838
+ },
839
+ "18": {
840
+ "pass@1": 0.21059283088235295,
841
+ "pass@2": 0.2836838524779991,
842
+ "pass@4": 0.3568823333664176,
843
+ "pass@8": 0.4261368301347266,
844
+ "pass@16": 0.48815849016843194,
845
+ "pass@32": 0.5409295253300673,
846
+ "pass@64": 0.5854362229138341,
847
+ "pass@128": 0.6235294117647059
848
+ },
849
+ "19": {
850
+ "pass@1": 0.16171409970238096,
851
+ "pass@2": 0.2210187593738281,
852
+ "pass@4": 0.2790568570111771,
853
+ "pass@8": 0.3330461307196623,
854
+ "pass@16": 0.3823068390231134,
855
+ "pass@32": 0.4286068727382764,
856
+ "pass@64": 0.47582930473787466,
857
+ "pass@128": 0.5297619047619048
858
+ },
859
+ "20": {
860
+ "pass@1": 0.15501029191616766,
861
+ "pass@2": 0.21969520604460377,
862
+ "pass@4": 0.28707018359232045,
863
+ "pass@8": 0.35047540956249357,
864
+ "pass@16": 0.40577476078592467,
865
+ "pass@32": 0.4521128631680825,
866
+ "pass@64": 0.492629793596181,
867
+ "pass@128": 0.5299401197604791
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 181446,
874
+ "answer_accuracy": 0.4332355974938875,
875
+ "avg_response_len": 362.83159430394255,
876
+ "resp_tokens_sum": 151959677,
877
+ "pass_at_k": {
878
+ "pass@1": 0.4332355974938875,
879
+ "pass@2": 0.5062632131013821,
880
+ "pass@4": 0.5687401453333036,
881
+ "pass@8": 0.6209731607363563,
882
+ "pass@16": 0.6651477627381525,
883
+ "pass@32": 0.7041156313705635,
884
+ "pass@64": 0.7392451818623957,
885
+ "pass@128": 0.7698655256723717
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.905952380952381,
890
+ "pass@2": 0.9413335051868518,
891
+ "pass@4": 0.960290828824968,
892
+ "pass@8": 0.9709857429670468,
893
+ "pass@16": 0.9774405273749097,
894
+ "pass@32": 0.9817948412796146,
895
+ "pass@64": 0.9855039352653046,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8351490825688074,
900
+ "pass@2": 0.8865521683642764,
901
+ "pass@4": 0.9158329351606272,
902
+ "pass@8": 0.93677803817362,
903
+ "pass@16": 0.9529807416222166,
904
+ "pass@32": 0.9633928267867302,
905
+ "pass@64": 0.969689293803253,
906
+ "pass@128": 0.9755351681957186
907
+ },
908
+ "13": {
909
+ "pass@1": 0.6290991512345679,
910
+ "pass@2": 0.7196005881209291,
911
+ "pass@4": 0.7843625041661457,
912
+ "pass@8": 0.8298234719287071,
913
+ "pass@16": 0.8622934266667467,
914
+ "pass@32": 0.887627567345078,
915
+ "pass@64": 0.9068883538882164,
916
+ "pass@128": 0.9228395061728395
917
+ },
918
+ "14": {
919
+ "pass@1": 0.517665059347181,
920
+ "pass@2": 0.6087565568120749,
921
+ "pass@4": 0.684754727802941,
922
+ "pass@8": 0.7437021459214515,
923
+ "pass@16": 0.787872327619222,
924
+ "pass@32": 0.820800001270744,
925
+ "pass@64": 0.8461369420166132,
926
+ "pass@128": 0.8635014836795252
927
+ },
928
+ "15": {
929
+ "pass@1": 0.3942901234567901,
930
+ "pass@2": 0.4890691831923788,
931
+ "pass@4": 0.5698684924222436,
932
+ "pass@8": 0.6340819192059179,
933
+ "pass@16": 0.6860708953608408,
934
+ "pass@32": 0.731478750850336,
935
+ "pass@64": 0.7721371133096265,
936
+ "pass@128": 0.808641975308642
937
+ },
938
+ "16": {
939
+ "pass@1": 0.29314159292035397,
940
+ "pass@2": 0.3814883689870623,
941
+ "pass@4": 0.46538661760200317,
942
+ "pass@8": 0.5397500092692332,
943
+ "pass@16": 0.6028619868483489,
944
+ "pass@32": 0.6595435053360812,
945
+ "pass@64": 0.7098720418701685,
946
+ "pass@128": 0.7463126843657817
947
+ },
948
+ "17": {
949
+ "pass@1": 0.23444559487951808,
950
+ "pass@2": 0.30919964306043063,
951
+ "pass@4": 0.37583967845510263,
952
+ "pass@8": 0.4334315980436936,
953
+ "pass@16": 0.48871885354179795,
954
+ "pass@32": 0.5457485752789358,
955
+ "pass@64": 0.6028672476278413,
956
+ "pass@128": 0.6536144578313253
957
+ },
958
+ "18": {
959
+ "pass@1": 0.21048387096774193,
960
+ "pass@2": 0.2815686912623825,
961
+ "pass@4": 0.34966251466550535,
962
+ "pass@8": 0.41016769779492335,
963
+ "pass@16": 0.46298474128011696,
964
+ "pass@32": 0.5089164087981424,
965
+ "pass@64": 0.5518706752579567,
966
+ "pass@128": 0.5967741935483871
967
+ },
968
+ "19": {
969
+ "pass@1": 0.17015200151057402,
970
+ "pass@2": 0.23729987867830718,
971
+ "pass@4": 0.3032997266013954,
972
+ "pass@8": 0.36506129258690445,
973
+ "pass@16": 0.42155961414972953,
974
+ "pass@32": 0.4742200902464383,
975
+ "pass@64": 0.5248506977503865,
976
+ "pass@128": 0.5709969788519638
977
+ },
978
+ "20": {
979
+ "pass@1": 0.16295983483483484,
980
+ "pass@2": 0.22672450994301377,
981
+ "pass@4": 0.2945440325026939,
982
+ "pass@8": 0.35988728724728264,
983
+ "pass@16": 0.4203472111206872,
984
+ "pass@32": 0.4770773485227238,
985
+ "pass@64": 0.5301250956398212,
986
+ "pass@128": 0.5765765765765766
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.6841266447368421,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.6841266447368421,
1002
+ "pass@2": 0.7266984627538122,
1003
+ "pass@4": 0.7619173084531972,
1004
+ "pass@8": 0.7911009823777397,
1005
+ "pass@16": 0.8155037451894985,
1006
+ "pass@32": 0.8365013016535509,
1007
+ "pass@64": 0.8548533485309445,
1008
+ "pass@128": 0.8703157894736843
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9176640625,
1013
+ "pass@2": 0.940686146653544,
1014
+ "pass@4": 0.9572751454818151,
1015
+ "pass@8": 0.9697146074131044,
1016
+ "pass@16": 0.9786371901621834,
1017
+ "pass@32": 0.9846897936578861,
1018
+ "pass@64": 0.9886130167733074,
1019
+ "pass@128": 0.991
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.988515625,
1023
+ "pass@2": 0.9917391732283464,
1024
+ "pass@4": 0.9931179612860892,
1025
+ "pass@8": 0.9939276029971762,
1026
+ "pass@16": 0.9945250737957444,
1027
+ "pass@32": 0.995013888448554,
1028
+ "pass@64": 0.9954711788593589,
1029
+ "pass@128": 0.996
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.993640625,
1033
+ "pass@2": 0.997156619094488,
1034
+ "pass@4": 0.9985739075740535,
1035
+ "pass@8": 0.9990783257375846,
1036
+ "pass@16": 0.9993271435678454,
1037
+ "pass@32": 0.9995814496671865,
1038
+ "pass@64": 0.9998779527559054,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9908671875,
1043
+ "pass@2": 0.9935773868110238,
1044
+ "pass@4": 0.9958510903637046,
1045
+ "pass@8": 0.9974042594396715,
1046
+ "pass@16": 0.9983575185071549,
1047
+ "pass@32": 0.9991406729482755,
1048
+ "pass@64": 0.9997315728158566,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.9576875,
1053
+ "pass@2": 0.9662279773622043,
1054
+ "pass@4": 0.9724250765841771,
1055
+ "pass@8": 0.977084930700057,
1056
+ "pass@16": 0.981167244820512,
1057
+ "pass@32": 0.9850588622178171,
1058
+ "pass@64": 0.9886003349090352,
1059
+ "pass@128": 0.991
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9621640625,
1063
+ "pass@2": 0.9733558070866145,
1064
+ "pass@4": 0.9805783572365954,
1065
+ "pass@8": 0.9853265539365899,
1066
+ "pass@16": 0.9881276610501337,
1067
+ "pass@32": 0.9902692290508687,
1068
+ "pass@64": 0.9921569678703794,
1069
+ "pass@128": 0.993
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.969,
1073
+ "pass@2": 0.9790697588582676,
1074
+ "pass@4": 0.9853942632170978,
1075
+ "pass@8": 0.989212366296118,
1076
+ "pass@16": 0.9919174897126309,
1077
+ "pass@32": 0.9943850588471859,
1078
+ "pass@64": 0.9962791607703998,
1079
+ "pass@128": 0.997
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.9544375,
1083
+ "pass@2": 0.9708692175196855,
1084
+ "pass@4": 0.9808557603112115,
1085
+ "pass@8": 0.9876466681787328,
1086
+ "pass@16": 0.992855800260977,
1087
+ "pass@32": 0.9967386933633018,
1088
+ "pass@64": 0.9989564749136509,
1089
+ "pass@128": 1.0
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.9541640625,
1093
+ "pass@2": 0.9719039124015749,
1094
+ "pass@4": 0.9804132486876638,
1095
+ "pass@8": 0.9854427280266015,
1096
+ "pass@16": 0.9890195178553929,
1097
+ "pass@32": 0.99177893102479,
1098
+ "pass@64": 0.9940926403130936,
1099
+ "pass@128": 0.996
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.897328125,
1103
+ "pass@2": 0.93245250984252,
1104
+ "pass@4": 0.9525756509186349,
1105
+ "pass@8": 0.9645405289253838,
1106
+ "pass@16": 0.9721646148438103,
1107
+ "pass@32": 0.9774677539333159,
1108
+ "pass@64": 0.9818126562253906,
1109
+ "pass@128": 0.986
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.83025,
1113
+ "pass@2": 0.8788818897637795,
1114
+ "pass@4": 0.9077505954255725,
1115
+ "pass@8": 0.9279219926000699,
1116
+ "pass@16": 0.9429205001103443,
1117
+ "pass@32": 0.9540556965113227,
1118
+ "pass@64": 0.9624023478061977,
1119
+ "pass@128": 0.968
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.6266796875,
1123
+ "pass@2": 0.7173414124015755,
1124
+ "pass@4": 0.7835768498312717,
1125
+ "pass@8": 0.8310638378335047,
1126
+ "pass@16": 0.8652153962130704,
1127
+ "pass@32": 0.8921147478710862,
1128
+ "pass@64": 0.9136172681639906,
1129
+ "pass@128": 0.93
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.5015,
1133
+ "pass@2": 0.589535925196851,
1134
+ "pass@4": 0.6631350288713906,
1135
+ "pass@8": 0.7234786940771628,
1136
+ "pass@16": 0.7724374351005907,
1137
+ "pass@32": 0.8117817021972529,
1138
+ "pass@64": 0.8433248048181223,
1139
+ "pass@128": 0.869
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.385046875,
1143
+ "pass@2": 0.47474187992126016,
1144
+ "pass@4": 0.5506543599550056,
1145
+ "pass@8": 0.6135148960829805,
1146
+ "pass@16": 0.6656588575667085,
1147
+ "pass@32": 0.7095550865297331,
1148
+ "pass@64": 0.7464043576049708,
1149
+ "pass@128": 0.775
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.280765625,
1153
+ "pass@2": 0.3638841043307086,
1154
+ "pass@4": 0.43851758830146187,
1155
+ "pass@8": 0.5031006368868065,
1156
+ "pass@16": 0.5591009613517215,
1157
+ "pass@32": 0.6095957808194802,
1158
+ "pass@64": 0.6556664450191428,
1159
+ "pass@128": 0.694
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.2475625,
1163
+ "pass@2": 0.3250761564960633,
1164
+ "pass@4": 0.39567444216347986,
1165
+ "pass@8": 0.45756658041957493,
1166
+ "pass@16": 0.5128672951589743,
1167
+ "pass@32": 0.5644771191304162,
1168
+ "pass@64": 0.6129670466840049,
1169
+ "pass@128": 0.656
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.21215625,
1173
+ "pass@2": 0.2847347440944884,
1174
+ "pass@4": 0.35507883173978216,
1175
+ "pass@8": 0.4180793864445228,
1176
+ "pass@16": 0.4726955368933959,
1177
+ "pass@32": 0.5197814062778024,
1178
+ "pass@64": 0.5618646594106671,
1179
+ "pass@128": 0.601
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.1663125,
1183
+ "pass@2": 0.22953531003937006,
1184
+ "pass@4": 0.2925731046119234,
1185
+ "pass@8": 0.35215195152385936,
1186
+ "pass@16": 0.4065216822086127,
1187
+ "pass@32": 0.4569250687655832,
1188
+ "pass@64": 0.5050905644534417,
1189
+ "pass@128": 0.55
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.1626640625,
1193
+ "pass@2": 0.22650086122047253,
1194
+ "pass@4": 0.2924075980502435,
1195
+ "pass@8": 0.3546621176573502,
1196
+ "pass@16": 0.41105423942037367,
1197
+ "pass@32": 0.4611137901553134,
1198
+ "pass@64": 0.505284171920995,
1199
+ "pass@128": 0.543
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9176640625,
1204
+ "2": 0.988515625,
1205
+ "3": 0.993640625,
1206
+ "4": 0.9908671875,
1207
+ "5": 0.9576875,
1208
+ "6": 0.9621640625,
1209
+ "7": 0.969,
1210
+ "8": 0.9544375,
1211
+ "9": 0.9541640625,
1212
+ "11": 0.897328125,
1213
+ "12": 0.83025,
1214
+ "13": 0.6266796875,
1215
+ "14": 0.5015,
1216
+ "15": 0.385046875,
1217
+ "16": 0.280765625,
1218
+ "17": 0.2475625,
1219
+ "18": 0.21215625,
1220
+ "19": 0.1663125,
1221
+ "20": 0.1626640625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 278.3240859375,
1225
+ "2": 116.5158515625,
1226
+ "3": 144.4954140625,
1227
+ "4": 159.5400078125,
1228
+ "5": 174.3931875,
1229
+ "6": 199.3752578125,
1230
+ "7": 223.4614296875,
1231
+ "8": 242.72865625,
1232
+ "9": 257.89315625,
1233
+ "11": 295.7106484375,
1234
+ "12": 306.1600703125,
1235
+ "13": 321.2292265625,
1236
+ "14": 337.66740625,
1237
+ "15": 344.118015625,
1238
+ "16": 350.866875,
1239
+ "17": 360.0460703125,
1240
+ "18": 359.24096875,
1241
+ "19": 359.20896875,
1242
+ "20": 362.70365625
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 278.3240859375,
1247
+ "2": 116.5158515625,
1248
+ "3": 144.4954140625,
1249
+ "4": 159.5400078125,
1250
+ "5": 174.3931875,
1251
+ "6": 199.3752578125,
1252
+ "7": 223.4614296875,
1253
+ "8": 242.72865625,
1254
+ "9": 257.89315625,
1255
+ "11": 295.7106484375,
1256
+ "12": 306.1600703125,
1257
+ "13": 321.2292265625,
1258
+ "14": 337.66740625,
1259
+ "15": 344.118015625,
1260
+ "16": 350.866875,
1261
+ "17": 360.0460703125,
1262
+ "18": 359.24096875,
1263
+ "19": 359.20896875,
1264
+ "20": 362.70365625
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 551156,
1271
+ "answer_accuracy": 0.6806680761934872,
1272
+ "avg_response_len": 260.37425283552005,
1273
+ "resp_tokens_sum": 210832323,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.6806680761934872,
1276
+ "pass@2": 0.7238366308211794,
1277
+ "pass@4": 0.7589840573052321,
1278
+ "pass@8": 0.7876935621338345,
1279
+ "pass@16": 0.8114617220263678,
1280
+ "pass@32": 0.8318796098052982,
1281
+ "pass@64": 0.8496453191543497,
1282
+ "pass@128": 0.8635788808093582
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9215984513274337,
1287
+ "pass@2": 0.9421719305274897,
1288
+ "pass@4": 0.9578695823973329,
1289
+ "pass@8": 0.9698981562700201,
1290
+ "pass@16": 0.9786144975781491,
1291
+ "pass@32": 0.9852228034323779,
1292
+ "pass@64": 0.9899066079157088,
1293
+ "pass@128": 0.9911504424778761
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9882204049844237,
1297
+ "pass@2": 0.9918561581671449,
1298
+ "pass@4": 0.9931930275654794,
1299
+ "pass@8": 0.9936832086849384,
1300
+ "pass@16": 0.993767744760047,
1301
+ "pass@32": 0.9937694702012028,
1302
+ "pass@64": 0.9937694704049844,
1303
+ "pass@128": 0.9937694704049844
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9939554663608563,
1307
+ "pass@2": 0.9985781772977917,
1308
+ "pass@4": 0.999890842199771,
1309
+ "pass@8": 0.9999992999127619,
1310
+ "pass@16": 0.9999999999827538,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.990234375,
1317
+ "pass@2": 0.99342742679626,
1318
+ "pass@4": 0.9958457609790964,
1319
+ "pass@8": 0.9970974256312388,
1320
+ "pass@16": 0.9976077641280157,
1321
+ "pass@32": 0.9982468010091103,
1322
+ "pass@64": 0.9992249015748031,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.948139880952381,
1327
+ "pass@2": 0.9574174321959756,
1328
+ "pass@4": 0.9648101639080825,
1329
+ "pass@8": 0.971654004642766,
1330
+ "pass@16": 0.9780823665716158,
1331
+ "pass@32": 0.9842143450259111,
1332
+ "pass@64": 0.9898102076387585,
1333
+ "pass@128": 0.9936507936507937
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9406811377245509,
1337
+ "pass@2": 0.9580407344759303,
1338
+ "pass@4": 0.9713076608313183,
1339
+ "pass@8": 0.9802488166001808,
1340
+ "pass@16": 0.9856429524545877,
1341
+ "pass@32": 0.9897026831061002,
1342
+ "pass@64": 0.9927742684651953,
1343
+ "pass@128": 0.9940119760479041
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.96875,
1347
+ "pass@2": 0.980806722615664,
1348
+ "pass@4": 0.988206052553342,
1349
+ "pass@8": 0.9921328277288592,
1350
+ "pass@16": 0.9942489886848938,
1351
+ "pass@32": 0.9958450340249061,
1352
+ "pass@64": 0.9968560505325343,
1353
+ "pass@128": 0.9970414201183432
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.955879407051282,
1357
+ "pass@2": 0.9741089680496671,
1358
+ "pass@4": 0.9835598903502444,
1359
+ "pass@8": 0.9884050733900349,
1360
+ "pass@16": 0.9918600204545246,
1361
+ "pass@32": 0.9954140492926473,
1362
+ "pass@64": 0.9986018037303472,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9509242957746479,
1367
+ "pass@2": 0.9714289675058222,
1368
+ "pass@4": 0.9813418648373179,
1369
+ "pass@8": 0.9873102931793774,
1370
+ "pass@16": 0.991888158345391,
1371
+ "pass@32": 0.9956463578845761,
1372
+ "pass@64": 0.9982995101172762,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.8825469192634561,
1377
+ "pass@2": 0.9204746018380142,
1378
+ "pass@4": 0.9434641893205277,
1379
+ "pass@8": 0.9584269403349234,
1380
+ "pass@16": 0.9686552276126651,
1381
+ "pass@32": 0.9741207162725538,
1382
+ "pass@64": 0.9764454185753144,
1383
+ "pass@128": 0.9773371104815864
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8271539548022598,
1387
+ "pass@2": 0.8798437569509316,
1388
+ "pass@4": 0.9131463289016838,
1389
+ "pass@8": 0.9351818057985669,
1390
+ "pass@16": 0.9499145912403741,
1391
+ "pass@32": 0.9615198784184202,
1392
+ "pass@64": 0.9709481160243175,
1393
+ "pass@128": 0.9745762711864406
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.6312695924764891,
1397
+ "pass@2": 0.7226765945498974,
1398
+ "pass@4": 0.7903836963561368,
1399
+ "pass@8": 0.838930594168827,
1400
+ "pass@16": 0.8732006163685243,
1401
+ "pass@32": 0.9012810590070837,
1402
+ "pass@64": 0.9258364518367349,
1403
+ "pass@128": 0.9435736677115988
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.4956678348909657,
1407
+ "pass@2": 0.5795753152059264,
1408
+ "pass@4": 0.6470877296879942,
1409
+ "pass@8": 0.7036891591817919,
1410
+ "pass@16": 0.7513465261882352,
1411
+ "pass@32": 0.7915273102114493,
1412
+ "pass@64": 0.8268042808049906,
1413
+ "pass@128": 0.8598130841121495
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.3917765534682081,
1417
+ "pass@2": 0.47504665240544336,
1418
+ "pass@4": 0.5456971947328837,
1419
+ "pass@8": 0.6072410800646598,
1420
+ "pass@16": 0.6594419230378626,
1421
+ "pass@32": 0.7028996498231647,
1422
+ "pass@64": 0.7405890497226499,
1423
+ "pass@128": 0.7658959537572254
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.26927923387096775,
1427
+ "pass@2": 0.3519208788417577,
1428
+ "pass@4": 0.42269466175599024,
1429
+ "pass@8": 0.4817823935686336,
1430
+ "pass@16": 0.5343682207500322,
1431
+ "pass@32": 0.582402508764003,
1432
+ "pass@64": 0.6250428507581135,
1433
+ "pass@128": 0.6612903225806451
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.25090317919075145,
1437
+ "pass@2": 0.3269800248054254,
1438
+ "pass@4": 0.39415536226324305,
1439
+ "pass@8": 0.45047162566799187,
1440
+ "pass@16": 0.4981191025185966,
1441
+ "pass@32": 0.5403608962010116,
1442
+ "pass@64": 0.5799445864824998,
1443
+ "pass@128": 0.6184971098265896
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.21515625,
1447
+ "pass@2": 0.28855982845894257,
1448
+ "pass@4": 0.3581241681396968,
1449
+ "pass@8": 0.4172596510922582,
1450
+ "pass@16": 0.46627537268369557,
1451
+ "pass@32": 0.5088608026804442,
1452
+ "pass@64": 0.5478183836857043,
1453
+ "pass@128": 0.5828571428571429
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.16713588588588588,
1457
+ "pass@2": 0.23041065179352582,
1458
+ "pass@4": 0.2955489223756941,
1459
+ "pass@8": 0.35859808935671944,
1460
+ "pass@16": 0.4160070631029604,
1461
+ "pass@32": 0.468307236198052,
1462
+ "pass@64": 0.5149739853037154,
1463
+ "pass@128": 0.5495495495495496
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.17004504504504506,
1467
+ "pass@2": 0.23310330507673024,
1468
+ "pass@4": 0.2956246063273622,
1469
+ "pass@8": 0.35363622885925766,
1470
+ "pass@16": 0.4070566006447026,
1471
+ "pass@32": 0.4541781885859068,
1472
+ "pass@64": 0.4931356276630924,
1473
+ "pass@128": 0.5225225225225225
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 562160,
1480
+ "answer_accuracy": 0.6846258768511302,
1481
+ "avg_response_len": 269.4555399941543,
1482
+ "resp_tokens_sum": 221255333,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.6846258768511302,
1485
+ "pass@2": 0.7261682717977604,
1486
+ "pass@4": 0.7609463192782817,
1487
+ "pass@8": 0.7903323337294538,
1488
+ "pass@16": 0.8151022421749073,
1489
+ "pass@32": 0.8361170877963872,
1490
+ "pass@64": 0.8541447697685913,
1491
+ "pass@128": 0.8699922057677318
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9162432898773006,
1496
+ "pass@2": 0.936796154171296,
1497
+ "pass@4": 0.9512743250843644,
1498
+ "pass@8": 0.9633956921405639,
1499
+ "pass@16": 0.9736279940224598,
1500
+ "pass@32": 0.9804423382093927,
1501
+ "pass@64": 0.9842310694963466,
1502
+ "pass@128": 0.9877300613496932
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.9920687689969605,
1506
+ "pass@2": 0.9949422462484743,
1507
+ "pass@4": 0.9961704575112003,
1508
+ "pass@8": 0.9969137768076869,
1509
+ "pass@16": 0.997325481498739,
1510
+ "pass@32": 0.9977203377335374,
1511
+ "pass@64": 0.9984802431610939,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9942708333333333,
1516
+ "pass@2": 0.9958694971367216,
1517
+ "pass@4": 0.9967973148242834,
1518
+ "pass@8": 0.9974456186860212,
1519
+ "pass@16": 0.9979759402219975,
1520
+ "pass@32": 0.9987316925844025,
1521
+ "pass@64": 0.9996301598663804,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9875416050295858,
1526
+ "pass@2": 0.9915100987746355,
1527
+ "pass@4": 0.9949397878519625,
1528
+ "pass@8": 0.9972912179822564,
1529
+ "pass@16": 0.9987250305872443,
1530
+ "pass@32": 0.9996231767711692,
1531
+ "pass@64": 0.9999805562449308,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9740942028985508,
1536
+ "pass@2": 0.9819546673513638,
1537
+ "pass@4": 0.987201654956174,
1538
+ "pass@8": 0.9904344185485051,
1539
+ "pass@16": 0.9930448124857775,
1540
+ "pass@32": 0.9952773429073588,
1541
+ "pass@64": 0.9967067319810339,
1542
+ "pass@128": 0.9971014492753624
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9741001074498568,
1546
+ "pass@2": 0.982676178282156,
1547
+ "pass@4": 0.9865603822265768,
1548
+ "pass@8": 0.9885311263764657,
1549
+ "pass@16": 0.9894516215507114,
1550
+ "pass@32": 0.9902045618641498,
1551
+ "pass@64": 0.9910543058907616,
1552
+ "pass@128": 0.9914040114613181
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9690435332369942,
1556
+ "pass@2": 0.9785164250603066,
1557
+ "pass@4": 0.9848848110929772,
1558
+ "pass@8": 0.9886472236910547,
1559
+ "pass@16": 0.991308776141372,
1560
+ "pass@32": 0.993989628100775,
1561
+ "pass@64": 0.9961800005089887,
1562
+ "pass@128": 0.9971098265895953
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9597271126760564,
1566
+ "pass@2": 0.9766656038593767,
1567
+ "pass@4": 0.9864835997612974,
1568
+ "pass@8": 0.992742027879815,
1569
+ "pass@16": 0.9970229470709905,
1570
+ "pass@32": 0.9994615576509764,
1571
+ "pass@64": 0.9999920569141826,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9533589071856288,
1576
+ "pass@2": 0.9688030435192612,
1577
+ "pass@4": 0.9762362204724409,
1578
+ "pass@8": 0.9810118633763555,
1579
+ "pass@16": 0.9841736264077786,
1580
+ "pass@32": 0.9865662118269425,
1581
+ "pass@64": 0.988977219442525,
1582
+ "pass@128": 0.9910179640718563
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9048616340361446,
1586
+ "pass@2": 0.9367618110236223,
1587
+ "pass@4": 0.9549433162910352,
1588
+ "pass@8": 0.964925632387157,
1589
+ "pass@16": 0.9708902210043483,
1590
+ "pass@32": 0.9769209821868238,
1591
+ "pass@64": 0.9840171200594384,
1592
+ "pass@128": 0.9909638554216867
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8286637931034483,
1596
+ "pass@2": 0.8699518518747071,
1597
+ "pass@4": 0.8934778219337001,
1598
+ "pass@8": 0.9107875074125524,
1599
+ "pass@16": 0.924846496240714,
1600
+ "pass@32": 0.936201270194174,
1601
+ "pass@64": 0.9454492653916154,
1602
+ "pass@128": 0.9529780564263323
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.6203825280112045,
1606
+ "pass@2": 0.7105237764397101,
1607
+ "pass@4": 0.7767815107670366,
1608
+ "pass@8": 0.8251601495483123,
1609
+ "pass@16": 0.860732014990156,
1610
+ "pass@32": 0.8879965216471163,
1611
+ "pass@64": 0.9088056396870849,
1612
+ "pass@128": 0.9243697478991597
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.4910453216374269,
1616
+ "pass@2": 0.5799452905557857,
1617
+ "pass@4": 0.6568933461168229,
1618
+ "pass@8": 0.7221252947493525,
1619
+ "pass@16": 0.7770240520072819,
1620
+ "pass@32": 0.8219059508512782,
1621
+ "pass@64": 0.8560598866085424,
1622
+ "pass@128": 0.8830409356725146
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.3689157196969697,
1626
+ "pass@2": 0.46035552374135,
1627
+ "pass@4": 0.5369870879776393,
1628
+ "pass@8": 0.5998998198723953,
1629
+ "pass@16": 0.6521363093899889,
1630
+ "pass@32": 0.6950081588345737,
1631
+ "pass@64": 0.7272367323897406,
1632
+ "pass@128": 0.7515151515151515
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.278957443019943,
1636
+ "pass@2": 0.3574475065616797,
1637
+ "pass@4": 0.4265418227636073,
1638
+ "pass@8": 0.48653231264461394,
1639
+ "pass@16": 0.5386797703066138,
1640
+ "pass@32": 0.585372520779792,
1641
+ "pass@64": 0.6303605102283197,
1642
+ "pass@128": 0.6723646723646723
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.2574970885093168,
1646
+ "pass@2": 0.33939995781777277,
1647
+ "pass@4": 0.4177574955692649,
1648
+ "pass@8": 0.49007486766441977,
1649
+ "pass@16": 0.5536131251915021,
1650
+ "pass@32": 0.6097010001622978,
1651
+ "pass@64": 0.6588642656789311,
1652
+ "pass@128": 0.6987577639751553
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.21059283088235295,
1656
+ "pass@2": 0.2836838524779991,
1657
+ "pass@4": 0.3568823333664176,
1658
+ "pass@8": 0.4261368301347266,
1659
+ "pass@16": 0.48815849016843194,
1660
+ "pass@32": 0.5409295253300673,
1661
+ "pass@64": 0.5854362229138341,
1662
+ "pass@128": 0.6235294117647059
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.16171409970238096,
1666
+ "pass@2": 0.2210187593738281,
1667
+ "pass@4": 0.2790568570111771,
1668
+ "pass@8": 0.3330461307196623,
1669
+ "pass@16": 0.3823068390231134,
1670
+ "pass@32": 0.4286068727382764,
1671
+ "pass@64": 0.47582930473787466,
1672
+ "pass@128": 0.5297619047619048
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.15501029191616766,
1676
+ "pass@2": 0.21969520604460377,
1677
+ "pass@4": 0.28707018359232045,
1678
+ "pass@8": 0.35047540956249357,
1679
+ "pass@16": 0.40577476078592467,
1680
+ "pass@32": 0.4521128631680825,
1681
+ "pass@64": 0.492629793596181,
1682
+ "pass@128": 0.5299401197604791
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 550480,
1689
+ "answer_accuracy": 0.6871105607924589,
1690
+ "avg_response_len": 290.4607989495127,
1691
+ "resp_tokens_sum": 232703250,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.6871105607924589,
1694
+ "pass@2": 0.7301343349041897,
1695
+ "pass@4": 0.7658771490538376,
1696
+ "pass@8": 0.7953326841737876,
1697
+ "pass@16": 0.8200005466543361,
1698
+ "pass@32": 0.8415662568421538,
1699
+ "pass@64": 0.8608433671596183,
1700
+ "pass@128": 0.877456462693721
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9150652985074627,
1705
+ "pass@2": 0.9429681072981553,
1706
+ "pass@4": 0.9625132151391523,
1707
+ "pass@8": 0.9756780202977416,
1708
+ "pass@16": 0.9835347744234902,
1709
+ "pass@32": 0.9882837642926804,
1710
+ "pass@64": 0.9915682030867851,
1711
+ "pass@128": 0.9940298507462687
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9854464285714286,
1715
+ "pass@2": 0.988620992688414,
1716
+ "pass@4": 0.9901797683325299,
1717
+ "pass@8": 0.991344744113091,
1718
+ "pass@16": 0.9925872694705262,
1719
+ "pass@32": 0.9936111354275262,
1720
+ "pass@64": 0.9942033681695974,
1721
+ "pass@128": 0.9942857142857143
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9927341472303207,
1725
+ "pass@2": 0.9970397115539129,
1726
+ "pass@4": 0.9990276626318211,
1727
+ "pass@8": 0.9997711384831617,
1728
+ "pass@16": 0.9999856655983261,
1729
+ "pass@32": 0.9999999740942677,
1730
+ "pass@64": 0.9999999999999997,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9947459795321637,
1735
+ "pass@2": 0.9957608095040752,
1736
+ "pass@4": 0.9967567209252353,
1737
+ "pass@8": 0.997803074735884,
1738
+ "pass@16": 0.9986958293792437,
1739
+ "pass@32": 0.9995001838500145,
1740
+ "pass@64": 0.9999595798278741,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9498851102941176,
1745
+ "pass@2": 0.9584327234830942,
1746
+ "pass@4": 0.9644861882154439,
1747
+ "pass@8": 0.968570720112799,
1748
+ "pass@16": 0.9719730560084113,
1749
+ "pass@32": 0.9754725300929893,
1750
+ "pass@64": 0.979253814615793,
1751
+ "pass@128": 0.9823529411764705
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9716581230283912,
1755
+ "pass@2": 0.979230963883852,
1756
+ "pass@4": 0.9837603316146996,
1757
+ "pass@8": 0.9871485365512419,
1758
+ "pass@16": 0.9892880094924388,
1759
+ "pass@32": 0.9909373526903568,
1760
+ "pass@64": 0.992720534533528,
1761
+ "pass@128": 0.9936908517350158
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9692197389240507,
1765
+ "pass@2": 0.9778177327319845,
1766
+ "pass@4": 0.9829445342275255,
1767
+ "pass@8": 0.9867073769830967,
1768
+ "pass@16": 0.9900901740260195,
1769
+ "pass@32": 0.9932564114680369,
1770
+ "pass@64": 0.995770682007257,
1771
+ "pass@128": 0.9968354430379747
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9474474474474475,
1775
+ "pass@2": 0.96165444479913,
1776
+ "pass@4": 0.9723225123323547,
1777
+ "pass@8": 0.9815041002513741,
1778
+ "pass@16": 0.9893463293362276,
1779
+ "pass@32": 0.9950770481017992,
1780
+ "pass@64": 0.9981847806163591,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9587268890675241,
1785
+ "pass@2": 0.9757762456642275,
1786
+ "pass@4": 0.983839193030935,
1787
+ "pass@8": 0.9880694906116392,
1788
+ "pass@16": 0.9909492939632827,
1789
+ "pass@32": 0.9929625698571264,
1790
+ "pass@64": 0.9947843180953608,
1791
+ "pass@128": 0.9967845659163987
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.905952380952381,
1795
+ "pass@2": 0.9413335051868518,
1796
+ "pass@4": 0.960290828824968,
1797
+ "pass@8": 0.9709857429670468,
1798
+ "pass@16": 0.9774405273749097,
1799
+ "pass@32": 0.9817948412796146,
1800
+ "pass@64": 0.9855039352653046,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8351490825688074,
1805
+ "pass@2": 0.8865521683642764,
1806
+ "pass@4": 0.9158329351606272,
1807
+ "pass@8": 0.93677803817362,
1808
+ "pass@16": 0.9529807416222166,
1809
+ "pass@32": 0.9633928267867302,
1810
+ "pass@64": 0.969689293803253,
1811
+ "pass@128": 0.9755351681957186
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.6290991512345679,
1815
+ "pass@2": 0.7196005881209291,
1816
+ "pass@4": 0.7843625041661457,
1817
+ "pass@8": 0.8298234719287071,
1818
+ "pass@16": 0.8622934266667467,
1819
+ "pass@32": 0.887627567345078,
1820
+ "pass@64": 0.9068883538882164,
1821
+ "pass@128": 0.9228395061728395
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.517665059347181,
1825
+ "pass@2": 0.6087565568120749,
1826
+ "pass@4": 0.684754727802941,
1827
+ "pass@8": 0.7437021459214515,
1828
+ "pass@16": 0.787872327619222,
1829
+ "pass@32": 0.820800001270744,
1830
+ "pass@64": 0.8461369420166132,
1831
+ "pass@128": 0.8635014836795252
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.3942901234567901,
1835
+ "pass@2": 0.4890691831923788,
1836
+ "pass@4": 0.5698684924222436,
1837
+ "pass@8": 0.6340819192059179,
1838
+ "pass@16": 0.6860708953608408,
1839
+ "pass@32": 0.731478750850336,
1840
+ "pass@64": 0.7721371133096265,
1841
+ "pass@128": 0.808641975308642
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.29314159292035397,
1845
+ "pass@2": 0.3814883689870623,
1846
+ "pass@4": 0.46538661760200317,
1847
+ "pass@8": 0.5397500092692332,
1848
+ "pass@16": 0.6028619868483489,
1849
+ "pass@32": 0.6595435053360812,
1850
+ "pass@64": 0.7098720418701685,
1851
+ "pass@128": 0.7463126843657817
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.23444559487951808,
1855
+ "pass@2": 0.30919964306043063,
1856
+ "pass@4": 0.37583967845510263,
1857
+ "pass@8": 0.4334315980436936,
1858
+ "pass@16": 0.48871885354179795,
1859
+ "pass@32": 0.5457485752789358,
1860
+ "pass@64": 0.6028672476278413,
1861
+ "pass@128": 0.6536144578313253
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.21048387096774193,
1865
+ "pass@2": 0.2815686912623825,
1866
+ "pass@4": 0.34966251466550535,
1867
+ "pass@8": 0.41016769779492335,
1868
+ "pass@16": 0.46298474128011696,
1869
+ "pass@32": 0.5089164087981424,
1870
+ "pass@64": 0.5518706752579567,
1871
+ "pass@128": 0.5967741935483871
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.17015200151057402,
1875
+ "pass@2": 0.23729987867830718,
1876
+ "pass@4": 0.3032997266013954,
1877
+ "pass@8": 0.36506129258690445,
1878
+ "pass@16": 0.42155961414972953,
1879
+ "pass@32": 0.4742200902464383,
1880
+ "pass@64": 0.5248506977503865,
1881
+ "pass@128": 0.5709969788519638
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.16295983483483484,
1885
+ "pass@2": 0.22672450994301377,
1886
+ "pass@4": 0.2945440325026939,
1887
+ "pass@8": 0.35988728724728264,
1888
+ "pass@16": 0.4203472111206872,
1889
+ "pass@32": 0.4770773485227238,
1890
+ "pass@64": 0.5301250956398212,
1891
+ "pass@128": 0.5765765765765766
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch1-op9-12/SDAR-100M-block16-epoch1-op9-12_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ab0b360194a5945c6ff2a02d4c9ae86e2b4d71f892d8d5f2be441320d97008a
3
+ size 2616201721
results/composition/SDAR-100M-block16-epoch1-op9-12/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch1-op9-12,0.9653489583333333,0.4310265625,0.6841266447368421,0.0,0.0,0.0,199.63633854166667,339.695190625,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch1-op9-12/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch1-op9-12",
4
+ "id_acc": 0.9653489583333333,
5
+ "ood_acc": 0.4310265625,
6
+ "total_acc": 0.6841266447368421,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.63633854166667,
11
+ "ood_avg_resp_len": 339.695190625,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 551156,
19
+ "answer_accuracy": 0.6806680761934872,
20
+ "avg_response_len": 260.37425283552005,
21
+ "resp_tokens_sum": 210832323,
22
+ "pass_at_k": {
23
+ "pass@1": 0.6806680761934872,
24
+ "pass@2": 0.7238366308211794,
25
+ "pass@4": 0.7589840573052321,
26
+ "pass@8": 0.7876935621338345,
27
+ "pass@16": 0.8114617220263678,
28
+ "pass@32": 0.8318796098052982,
29
+ "pass@64": 0.8496453191543497,
30
+ "pass@128": 0.8635788808093582
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9215984513274337,
35
+ "pass@2": 0.9421719305274897,
36
+ "pass@4": 0.9578695823973329,
37
+ "pass@8": 0.9698981562700201,
38
+ "pass@16": 0.9786144975781491,
39
+ "pass@32": 0.9852228034323779,
40
+ "pass@64": 0.9899066079157088,
41
+ "pass@128": 0.9911504424778761
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9882204049844237,
45
+ "pass@2": 0.9918561581671449,
46
+ "pass@4": 0.9931930275654794,
47
+ "pass@8": 0.9936832086849384,
48
+ "pass@16": 0.993767744760047,
49
+ "pass@32": 0.9937694702012028,
50
+ "pass@64": 0.9937694704049844,
51
+ "pass@128": 0.9937694704049844
52
+ },
53
+ "3": {
54
+ "pass@1": 0.9939554663608563,
55
+ "pass@2": 0.9985781772977917,
56
+ "pass@4": 0.999890842199771,
57
+ "pass@8": 0.9999992999127619,
58
+ "pass@16": 0.9999999999827538,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.990234375,
65
+ "pass@2": 0.99342742679626,
66
+ "pass@4": 0.9958457609790964,
67
+ "pass@8": 0.9970974256312388,
68
+ "pass@16": 0.9976077641280157,
69
+ "pass@32": 0.9982468010091103,
70
+ "pass@64": 0.9992249015748031,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.948139880952381,
75
+ "pass@2": 0.9574174321959756,
76
+ "pass@4": 0.9648101639080825,
77
+ "pass@8": 0.971654004642766,
78
+ "pass@16": 0.9780823665716158,
79
+ "pass@32": 0.9842143450259111,
80
+ "pass@64": 0.9898102076387585,
81
+ "pass@128": 0.9936507936507937
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9406811377245509,
85
+ "pass@2": 0.9580407344759303,
86
+ "pass@4": 0.9713076608313183,
87
+ "pass@8": 0.9802488166001808,
88
+ "pass@16": 0.9856429524545877,
89
+ "pass@32": 0.9897026831061002,
90
+ "pass@64": 0.9927742684651953,
91
+ "pass@128": 0.9940119760479041
92
+ },
93
+ "7": {
94
+ "pass@1": 0.96875,
95
+ "pass@2": 0.980806722615664,
96
+ "pass@4": 0.988206052553342,
97
+ "pass@8": 0.9921328277288592,
98
+ "pass@16": 0.9942489886848938,
99
+ "pass@32": 0.9958450340249061,
100
+ "pass@64": 0.9968560505325343,
101
+ "pass@128": 0.9970414201183432
102
+ },
103
+ "8": {
104
+ "pass@1": 0.955879407051282,
105
+ "pass@2": 0.9741089680496671,
106
+ "pass@4": 0.9835598903502444,
107
+ "pass@8": 0.9884050733900349,
108
+ "pass@16": 0.9918600204545246,
109
+ "pass@32": 0.9954140492926473,
110
+ "pass@64": 0.9986018037303472,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9509242957746479,
115
+ "pass@2": 0.9714289675058222,
116
+ "pass@4": 0.9813418648373179,
117
+ "pass@8": 0.9873102931793774,
118
+ "pass@16": 0.991888158345391,
119
+ "pass@32": 0.9956463578845761,
120
+ "pass@64": 0.9982995101172762,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.8825469192634561,
125
+ "pass@2": 0.9204746018380142,
126
+ "pass@4": 0.9434641893205277,
127
+ "pass@8": 0.9584269403349234,
128
+ "pass@16": 0.9686552276126651,
129
+ "pass@32": 0.9741207162725538,
130
+ "pass@64": 0.9764454185753144,
131
+ "pass@128": 0.9773371104815864
132
+ },
133
+ "12": {
134
+ "pass@1": 0.8271539548022598,
135
+ "pass@2": 0.8798437569509316,
136
+ "pass@4": 0.9131463289016838,
137
+ "pass@8": 0.9351818057985669,
138
+ "pass@16": 0.9499145912403741,
139
+ "pass@32": 0.9615198784184202,
140
+ "pass@64": 0.9709481160243175,
141
+ "pass@128": 0.9745762711864406
142
+ },
143
+ "13": {
144
+ "pass@1": 0.6312695924764891,
145
+ "pass@2": 0.7226765945498974,
146
+ "pass@4": 0.7903836963561368,
147
+ "pass@8": 0.838930594168827,
148
+ "pass@16": 0.8732006163685243,
149
+ "pass@32": 0.9012810590070837,
150
+ "pass@64": 0.9258364518367349,
151
+ "pass@128": 0.9435736677115988
152
+ },
153
+ "14": {
154
+ "pass@1": 0.4956678348909657,
155
+ "pass@2": 0.5795753152059264,
156
+ "pass@4": 0.6470877296879942,
157
+ "pass@8": 0.7036891591817919,
158
+ "pass@16": 0.7513465261882352,
159
+ "pass@32": 0.7915273102114493,
160
+ "pass@64": 0.8268042808049906,
161
+ "pass@128": 0.8598130841121495
162
+ },
163
+ "15": {
164
+ "pass@1": 0.3917765534682081,
165
+ "pass@2": 0.47504665240544336,
166
+ "pass@4": 0.5456971947328837,
167
+ "pass@8": 0.6072410800646598,
168
+ "pass@16": 0.6594419230378626,
169
+ "pass@32": 0.7028996498231647,
170
+ "pass@64": 0.7405890497226499,
171
+ "pass@128": 0.7658959537572254
172
+ },
173
+ "16": {
174
+ "pass@1": 0.26927923387096775,
175
+ "pass@2": 0.3519208788417577,
176
+ "pass@4": 0.42269466175599024,
177
+ "pass@8": 0.4817823935686336,
178
+ "pass@16": 0.5343682207500322,
179
+ "pass@32": 0.582402508764003,
180
+ "pass@64": 0.6250428507581135,
181
+ "pass@128": 0.6612903225806451
182
+ },
183
+ "17": {
184
+ "pass@1": 0.25090317919075145,
185
+ "pass@2": 0.3269800248054254,
186
+ "pass@4": 0.39415536226324305,
187
+ "pass@8": 0.45047162566799187,
188
+ "pass@16": 0.4981191025185966,
189
+ "pass@32": 0.5403608962010116,
190
+ "pass@64": 0.5799445864824998,
191
+ "pass@128": 0.6184971098265896
192
+ },
193
+ "18": {
194
+ "pass@1": 0.21515625,
195
+ "pass@2": 0.28855982845894257,
196
+ "pass@4": 0.3581241681396968,
197
+ "pass@8": 0.4172596510922582,
198
+ "pass@16": 0.46627537268369557,
199
+ "pass@32": 0.5088608026804442,
200
+ "pass@64": 0.5478183836857043,
201
+ "pass@128": 0.5828571428571429
202
+ },
203
+ "19": {
204
+ "pass@1": 0.16713588588588588,
205
+ "pass@2": 0.23041065179352582,
206
+ "pass@4": 0.2955489223756941,
207
+ "pass@8": 0.35859808935671944,
208
+ "pass@16": 0.4160070631029604,
209
+ "pass@32": 0.468307236198052,
210
+ "pass@64": 0.5149739853037154,
211
+ "pass@128": 0.5495495495495496
212
+ },
213
+ "20": {
214
+ "pass@1": 0.17004504504504506,
215
+ "pass@2": 0.23310330507673024,
216
+ "pass@4": 0.2956246063273622,
217
+ "pass@8": 0.35363622885925766,
218
+ "pass@16": 0.4070566006447026,
219
+ "pass@32": 0.4541781885859068,
220
+ "pass@64": 0.4931356276630924,
221
+ "pass@128": 0.5225225225225225
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 562160,
228
+ "answer_accuracy": 0.6846258768511302,
229
+ "avg_response_len": 269.4555399941543,
230
+ "resp_tokens_sum": 221255333,
231
+ "pass_at_k": {
232
+ "pass@1": 0.6846258768511302,
233
+ "pass@2": 0.7261682717977604,
234
+ "pass@4": 0.7609463192782817,
235
+ "pass@8": 0.7903323337294538,
236
+ "pass@16": 0.8151022421749073,
237
+ "pass@32": 0.8361170877963872,
238
+ "pass@64": 0.8541447697685913,
239
+ "pass@128": 0.8699922057677318
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9162432898773006,
244
+ "pass@2": 0.936796154171296,
245
+ "pass@4": 0.9512743250843644,
246
+ "pass@8": 0.9633956921405639,
247
+ "pass@16": 0.9736279940224598,
248
+ "pass@32": 0.9804423382093927,
249
+ "pass@64": 0.9842310694963466,
250
+ "pass@128": 0.9877300613496932
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9920687689969605,
254
+ "pass@2": 0.9949422462484743,
255
+ "pass@4": 0.9961704575112003,
256
+ "pass@8": 0.9969137768076869,
257
+ "pass@16": 0.997325481498739,
258
+ "pass@32": 0.9977203377335374,
259
+ "pass@64": 0.9984802431610939,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9942708333333333,
264
+ "pass@2": 0.9958694971367216,
265
+ "pass@4": 0.9967973148242834,
266
+ "pass@8": 0.9974456186860212,
267
+ "pass@16": 0.9979759402219975,
268
+ "pass@32": 0.9987316925844025,
269
+ "pass@64": 0.9996301598663804,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9875416050295858,
274
+ "pass@2": 0.9915100987746355,
275
+ "pass@4": 0.9949397878519625,
276
+ "pass@8": 0.9972912179822564,
277
+ "pass@16": 0.9987250305872443,
278
+ "pass@32": 0.9996231767711692,
279
+ "pass@64": 0.9999805562449308,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9740942028985508,
284
+ "pass@2": 0.9819546673513638,
285
+ "pass@4": 0.987201654956174,
286
+ "pass@8": 0.9904344185485051,
287
+ "pass@16": 0.9930448124857775,
288
+ "pass@32": 0.9952773429073588,
289
+ "pass@64": 0.9967067319810339,
290
+ "pass@128": 0.9971014492753624
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9741001074498568,
294
+ "pass@2": 0.982676178282156,
295
+ "pass@4": 0.9865603822265768,
296
+ "pass@8": 0.9885311263764657,
297
+ "pass@16": 0.9894516215507114,
298
+ "pass@32": 0.9902045618641498,
299
+ "pass@64": 0.9910543058907616,
300
+ "pass@128": 0.9914040114613181
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9690435332369942,
304
+ "pass@2": 0.9785164250603066,
305
+ "pass@4": 0.9848848110929772,
306
+ "pass@8": 0.9886472236910547,
307
+ "pass@16": 0.991308776141372,
308
+ "pass@32": 0.993989628100775,
309
+ "pass@64": 0.9961800005089887,
310
+ "pass@128": 0.9971098265895953
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9597271126760564,
314
+ "pass@2": 0.9766656038593767,
315
+ "pass@4": 0.9864835997612974,
316
+ "pass@8": 0.992742027879815,
317
+ "pass@16": 0.9970229470709905,
318
+ "pass@32": 0.9994615576509764,
319
+ "pass@64": 0.9999920569141826,
320
+ "pass@128": 1.0
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9533589071856288,
324
+ "pass@2": 0.9688030435192612,
325
+ "pass@4": 0.9762362204724409,
326
+ "pass@8": 0.9810118633763555,
327
+ "pass@16": 0.9841736264077786,
328
+ "pass@32": 0.9865662118269425,
329
+ "pass@64": 0.988977219442525,
330
+ "pass@128": 0.9910179640718563
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9048616340361446,
334
+ "pass@2": 0.9367618110236223,
335
+ "pass@4": 0.9549433162910352,
336
+ "pass@8": 0.964925632387157,
337
+ "pass@16": 0.9708902210043483,
338
+ "pass@32": 0.9769209821868238,
339
+ "pass@64": 0.9840171200594384,
340
+ "pass@128": 0.9909638554216867
341
+ },
342
+ "12": {
343
+ "pass@1": 0.8286637931034483,
344
+ "pass@2": 0.8699518518747071,
345
+ "pass@4": 0.8934778219337001,
346
+ "pass@8": 0.9107875074125524,
347
+ "pass@16": 0.924846496240714,
348
+ "pass@32": 0.936201270194174,
349
+ "pass@64": 0.9454492653916154,
350
+ "pass@128": 0.9529780564263323
351
+ },
352
+ "13": {
353
+ "pass@1": 0.6203825280112045,
354
+ "pass@2": 0.7105237764397101,
355
+ "pass@4": 0.7767815107670366,
356
+ "pass@8": 0.8251601495483123,
357
+ "pass@16": 0.860732014990156,
358
+ "pass@32": 0.8879965216471163,
359
+ "pass@64": 0.9088056396870849,
360
+ "pass@128": 0.9243697478991597
361
+ },
362
+ "14": {
363
+ "pass@1": 0.4910453216374269,
364
+ "pass@2": 0.5799452905557857,
365
+ "pass@4": 0.6568933461168229,
366
+ "pass@8": 0.7221252947493525,
367
+ "pass@16": 0.7770240520072819,
368
+ "pass@32": 0.8219059508512782,
369
+ "pass@64": 0.8560598866085424,
370
+ "pass@128": 0.8830409356725146
371
+ },
372
+ "15": {
373
+ "pass@1": 0.3689157196969697,
374
+ "pass@2": 0.46035552374135,
375
+ "pass@4": 0.5369870879776393,
376
+ "pass@8": 0.5998998198723953,
377
+ "pass@16": 0.6521363093899889,
378
+ "pass@32": 0.6950081588345737,
379
+ "pass@64": 0.7272367323897406,
380
+ "pass@128": 0.7515151515151515
381
+ },
382
+ "16": {
383
+ "pass@1": 0.278957443019943,
384
+ "pass@2": 0.3574475065616797,
385
+ "pass@4": 0.4265418227636073,
386
+ "pass@8": 0.48653231264461394,
387
+ "pass@16": 0.5386797703066138,
388
+ "pass@32": 0.585372520779792,
389
+ "pass@64": 0.6303605102283197,
390
+ "pass@128": 0.6723646723646723
391
+ },
392
+ "17": {
393
+ "pass@1": 0.2574970885093168,
394
+ "pass@2": 0.33939995781777277,
395
+ "pass@4": 0.4177574955692649,
396
+ "pass@8": 0.49007486766441977,
397
+ "pass@16": 0.5536131251915021,
398
+ "pass@32": 0.6097010001622978,
399
+ "pass@64": 0.6588642656789311,
400
+ "pass@128": 0.6987577639751553
401
+ },
402
+ "18": {
403
+ "pass@1": 0.21059283088235295,
404
+ "pass@2": 0.2836838524779991,
405
+ "pass@4": 0.3568823333664176,
406
+ "pass@8": 0.4261368301347266,
407
+ "pass@16": 0.48815849016843194,
408
+ "pass@32": 0.5409295253300673,
409
+ "pass@64": 0.5854362229138341,
410
+ "pass@128": 0.6235294117647059
411
+ },
412
+ "19": {
413
+ "pass@1": 0.16171409970238096,
414
+ "pass@2": 0.2210187593738281,
415
+ "pass@4": 0.2790568570111771,
416
+ "pass@8": 0.3330461307196623,
417
+ "pass@16": 0.3823068390231134,
418
+ "pass@32": 0.4286068727382764,
419
+ "pass@64": 0.47582930473787466,
420
+ "pass@128": 0.5297619047619048
421
+ },
422
+ "20": {
423
+ "pass@1": 0.15501029191616766,
424
+ "pass@2": 0.21969520604460377,
425
+ "pass@4": 0.28707018359232045,
426
+ "pass@8": 0.35047540956249357,
427
+ "pass@16": 0.40577476078592467,
428
+ "pass@32": 0.4521128631680825,
429
+ "pass@64": 0.492629793596181,
430
+ "pass@128": 0.5299401197604791
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 550480,
437
+ "answer_accuracy": 0.6871105607924589,
438
+ "avg_response_len": 290.4607989495127,
439
+ "resp_tokens_sum": 232703250,
440
+ "pass_at_k": {
441
+ "pass@1": 0.6871105607924589,
442
+ "pass@2": 0.7301343349041897,
443
+ "pass@4": 0.7658771490538376,
444
+ "pass@8": 0.7953326841737876,
445
+ "pass@16": 0.8200005466543361,
446
+ "pass@32": 0.8415662568421538,
447
+ "pass@64": 0.8608433671596183,
448
+ "pass@128": 0.877456462693721
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9150652985074627,
453
+ "pass@2": 0.9429681072981553,
454
+ "pass@4": 0.9625132151391523,
455
+ "pass@8": 0.9756780202977416,
456
+ "pass@16": 0.9835347744234902,
457
+ "pass@32": 0.9882837642926804,
458
+ "pass@64": 0.9915682030867851,
459
+ "pass@128": 0.9940298507462687
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9854464285714286,
463
+ "pass@2": 0.988620992688414,
464
+ "pass@4": 0.9901797683325299,
465
+ "pass@8": 0.991344744113091,
466
+ "pass@16": 0.9925872694705262,
467
+ "pass@32": 0.9936111354275262,
468
+ "pass@64": 0.9942033681695974,
469
+ "pass@128": 0.9942857142857143
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9927341472303207,
473
+ "pass@2": 0.9970397115539129,
474
+ "pass@4": 0.9990276626318211,
475
+ "pass@8": 0.9997711384831617,
476
+ "pass@16": 0.9999856655983261,
477
+ "pass@32": 0.9999999740942677,
478
+ "pass@64": 0.9999999999999997,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9947459795321637,
483
+ "pass@2": 0.9957608095040752,
484
+ "pass@4": 0.9967567209252353,
485
+ "pass@8": 0.997803074735884,
486
+ "pass@16": 0.9986958293792437,
487
+ "pass@32": 0.9995001838500145,
488
+ "pass@64": 0.9999595798278741,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9498851102941176,
493
+ "pass@2": 0.9584327234830942,
494
+ "pass@4": 0.9644861882154439,
495
+ "pass@8": 0.968570720112799,
496
+ "pass@16": 0.9719730560084113,
497
+ "pass@32": 0.9754725300929893,
498
+ "pass@64": 0.979253814615793,
499
+ "pass@128": 0.9823529411764705
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9716581230283912,
503
+ "pass@2": 0.979230963883852,
504
+ "pass@4": 0.9837603316146996,
505
+ "pass@8": 0.9871485365512419,
506
+ "pass@16": 0.9892880094924388,
507
+ "pass@32": 0.9909373526903568,
508
+ "pass@64": 0.992720534533528,
509
+ "pass@128": 0.9936908517350158
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9692197389240507,
513
+ "pass@2": 0.9778177327319845,
514
+ "pass@4": 0.9829445342275255,
515
+ "pass@8": 0.9867073769830967,
516
+ "pass@16": 0.9900901740260195,
517
+ "pass@32": 0.9932564114680369,
518
+ "pass@64": 0.995770682007257,
519
+ "pass@128": 0.9968354430379747
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9474474474474475,
523
+ "pass@2": 0.96165444479913,
524
+ "pass@4": 0.9723225123323547,
525
+ "pass@8": 0.9815041002513741,
526
+ "pass@16": 0.9893463293362276,
527
+ "pass@32": 0.9950770481017992,
528
+ "pass@64": 0.9981847806163591,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9587268890675241,
533
+ "pass@2": 0.9757762456642275,
534
+ "pass@4": 0.983839193030935,
535
+ "pass@8": 0.9880694906116392,
536
+ "pass@16": 0.9909492939632827,
537
+ "pass@32": 0.9929625698571264,
538
+ "pass@64": 0.9947843180953608,
539
+ "pass@128": 0.9967845659163987
540
+ },
541
+ "11": {
542
+ "pass@1": 0.905952380952381,
543
+ "pass@2": 0.9413335051868518,
544
+ "pass@4": 0.960290828824968,
545
+ "pass@8": 0.9709857429670468,
546
+ "pass@16": 0.9774405273749097,
547
+ "pass@32": 0.9817948412796146,
548
+ "pass@64": 0.9855039352653046,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.8351490825688074,
553
+ "pass@2": 0.8865521683642764,
554
+ "pass@4": 0.9158329351606272,
555
+ "pass@8": 0.93677803817362,
556
+ "pass@16": 0.9529807416222166,
557
+ "pass@32": 0.9633928267867302,
558
+ "pass@64": 0.969689293803253,
559
+ "pass@128": 0.9755351681957186
560
+ },
561
+ "13": {
562
+ "pass@1": 0.6290991512345679,
563
+ "pass@2": 0.7196005881209291,
564
+ "pass@4": 0.7843625041661457,
565
+ "pass@8": 0.8298234719287071,
566
+ "pass@16": 0.8622934266667467,
567
+ "pass@32": 0.887627567345078,
568
+ "pass@64": 0.9068883538882164,
569
+ "pass@128": 0.9228395061728395
570
+ },
571
+ "14": {
572
+ "pass@1": 0.517665059347181,
573
+ "pass@2": 0.6087565568120749,
574
+ "pass@4": 0.684754727802941,
575
+ "pass@8": 0.7437021459214515,
576
+ "pass@16": 0.787872327619222,
577
+ "pass@32": 0.820800001270744,
578
+ "pass@64": 0.8461369420166132,
579
+ "pass@128": 0.8635014836795252
580
+ },
581
+ "15": {
582
+ "pass@1": 0.3942901234567901,
583
+ "pass@2": 0.4890691831923788,
584
+ "pass@4": 0.5698684924222436,
585
+ "pass@8": 0.6340819192059179,
586
+ "pass@16": 0.6860708953608408,
587
+ "pass@32": 0.731478750850336,
588
+ "pass@64": 0.7721371133096265,
589
+ "pass@128": 0.808641975308642
590
+ },
591
+ "16": {
592
+ "pass@1": 0.29314159292035397,
593
+ "pass@2": 0.3814883689870623,
594
+ "pass@4": 0.46538661760200317,
595
+ "pass@8": 0.5397500092692332,
596
+ "pass@16": 0.6028619868483489,
597
+ "pass@32": 0.6595435053360812,
598
+ "pass@64": 0.7098720418701685,
599
+ "pass@128": 0.7463126843657817
600
+ },
601
+ "17": {
602
+ "pass@1": 0.23444559487951808,
603
+ "pass@2": 0.30919964306043063,
604
+ "pass@4": 0.37583967845510263,
605
+ "pass@8": 0.4334315980436936,
606
+ "pass@16": 0.48871885354179795,
607
+ "pass@32": 0.5457485752789358,
608
+ "pass@64": 0.6028672476278413,
609
+ "pass@128": 0.6536144578313253
610
+ },
611
+ "18": {
612
+ "pass@1": 0.21048387096774193,
613
+ "pass@2": 0.2815686912623825,
614
+ "pass@4": 0.34966251466550535,
615
+ "pass@8": 0.41016769779492335,
616
+ "pass@16": 0.46298474128011696,
617
+ "pass@32": 0.5089164087981424,
618
+ "pass@64": 0.5518706752579567,
619
+ "pass@128": 0.5967741935483871
620
+ },
621
+ "19": {
622
+ "pass@1": 0.17015200151057402,
623
+ "pass@2": 0.23729987867830718,
624
+ "pass@4": 0.3032997266013954,
625
+ "pass@8": 0.36506129258690445,
626
+ "pass@16": 0.42155961414972953,
627
+ "pass@32": 0.4742200902464383,
628
+ "pass@64": 0.5248506977503865,
629
+ "pass@128": 0.5709969788519638
630
+ },
631
+ "20": {
632
+ "pass@1": 0.16295983483483484,
633
+ "pass@2": 0.22672450994301377,
634
+ "pass@4": 0.2945440325026939,
635
+ "pass@8": 0.35988728724728264,
636
+ "pass@16": 0.4203472111206872,
637
+ "pass@32": 0.4770773485227238,
638
+ "pass@64": 0.5301250956398212,
639
+ "pass@128": 0.5765765765765766
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffda0fba780922d1325dca30094a37946bab378b2bf3e00243675c7f3778b039
3
+ size 1789772232
results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch4-op11-14",
3
+ "id": {
4
+ "answer_accuracy": 0.9905703125,
5
+ "avg_response_len": 199.1858359375,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9905703125,
11
+ "pass@2": 0.9931240567585284,
12
+ "pass@4": 0.9946121005395152,
13
+ "pass@8": 0.9956282392841028,
14
+ "pass@16": 0.9963701188054171,
15
+ "pass@32": 0.996918025082482,
16
+ "pass@64": 0.9973719313555771,
17
+ "pass@128": 0.9977777777777778
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.942828125,
22
+ "pass@2": 0.9543673720472442,
23
+ "pass@4": 0.9623778480502434,
24
+ "pass@8": 0.9682502041026957,
25
+ "pass@16": 0.972860979589154,
26
+ "pass@32": 0.9767782353474479,
27
+ "pass@64": 0.9804701699275815,
28
+ "pass@128": 0.984
29
+ },
30
+ "2": {
31
+ "pass@1": 0.999984375,
32
+ "pass@2": 0.9999998769685039,
33
+ "pass@4": 1.0,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 0.999953125,
42
+ "pass@2": 0.9999992618110236,
43
+ "pass@4": 0.9999999999062618,
44
+ "pass@8": 1.0,
45
+ "pass@16": 1.0,
46
+ "pass@32": 1.0,
47
+ "pass@64": 1.0,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.998390625,
52
+ "pass@2": 0.9988737696850394,
53
+ "pass@4": 0.9993528392388451,
54
+ "pass@8": 0.9997678683581743,
55
+ "pass@16": 0.9999654222866098,
56
+ "pass@32": 0.9999993776062154,
57
+ "pass@64": 0.9999999999893633,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.9933984375,
62
+ "pass@2": 0.9961841781496062,
63
+ "pass@4": 0.9975373475815523,
64
+ "pass@8": 0.9983680305456933,
65
+ "pass@16": 0.9990184921424964,
66
+ "pass@32": 0.9995291759985511,
67
+ "pass@64": 0.9998772846233093,
68
+ "pass@128": 1.0
69
+ },
70
+ "6": {
71
+ "pass@1": 0.99665625,
72
+ "pass@2": 0.9977063238188978,
73
+ "pass@4": 0.9982420018747656,
74
+ "pass@8": 0.9986003427777526,
75
+ "pass@16": 0.9988616968167805,
76
+ "pass@32": 0.9989862397601972,
77
+ "pass@64": 0.998999972481556,
78
+ "pass@128": 0.999
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9970390625,
82
+ "pass@2": 0.9983011811023623,
83
+ "pass@4": 0.9990445734908138,
84
+ "pass@8": 0.9995488522357531,
85
+ "pass@16": 0.9998741388348003,
86
+ "pass@32": 0.9999900821873651,
87
+ "pass@64": 0.9999999879305058,
88
+ "pass@128": 1.0
89
+ },
90
+ "8": {
91
+ "pass@1": 0.9941171875,
92
+ "pass@2": 0.9971940206692916,
93
+ "pass@4": 0.9985053151481066,
94
+ "pass@8": 0.9989375417785472,
95
+ "pass@16": 0.9989989424149476,
96
+ "pass@32": 0.9989999998940483,
97
+ "pass@64": 0.999,
98
+ "pass@128": 0.999
99
+ },
100
+ "9": {
101
+ "pass@1": 0.992765625,
102
+ "pass@2": 0.9954905265748032,
103
+ "pass@4": 0.9964489795650546,
104
+ "pass@8": 0.9971813137583089,
105
+ "pass@16": 0.9977513971639642,
106
+ "pass@32": 0.9979791149485119,
107
+ "pass@64": 0.9979999672478784,
108
+ "pass@128": 0.998
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.942828125,
113
+ "2": 0.999984375,
114
+ "3": 0.999953125,
115
+ "4": 0.998390625,
116
+ "5": 0.9933984375,
117
+ "6": 0.99665625,
118
+ "7": 0.9970390625,
119
+ "8": 0.9941171875,
120
+ "9": 0.992765625
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 279.0269453125,
124
+ "2": 115.345625,
125
+ "3": 144.2695546875,
126
+ "4": 159.5131875,
127
+ "5": 172.7417421875,
128
+ "6": 198.263265625,
129
+ "7": 223.090140625,
130
+ "8": 242.3848515625,
131
+ "9": 258.0372109375
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 279.0269453125,
136
+ "2": 115.345625,
137
+ "3": 144.2695546875,
138
+ "4": 159.5131875,
139
+ "5": 172.7417421875,
140
+ "6": 198.263265625,
141
+ "7": 223.090140625,
142
+ "8": 242.3848515625,
143
+ "9": 258.0372109375
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 375744,
150
+ "answer_accuracy": 0.9913880445795339,
151
+ "avg_response_len": 189.61989720533603,
152
+ "resp_tokens_sum": 71867458,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9913880445795339,
155
+ "pass@2": 0.9933423381917686,
156
+ "pass@4": 0.9944101428494183,
157
+ "pass@8": 0.995316754918926,
158
+ "pass@16": 0.9962039266427511,
159
+ "pass@32": 0.9970168336391735,
160
+ "pass@64": 0.9977520459936974,
161
+ "pass@128": 0.9983113812901047
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9453585914454278,
166
+ "pass@2": 0.9535728491626602,
167
+ "pass@4": 0.95873854180838,
168
+ "pass@8": 0.9629986555885872,
169
+ "pass@16": 0.9679666317812168,
170
+ "pass@32": 0.9740341139349422,
171
+ "pass@64": 0.9803653794455086,
172
+ "pass@128": 0.9852507374631269
173
+ },
174
+ "2": {
175
+ "pass@1": 1.0,
176
+ "pass@2": 1.0,
177
+ "pass@4": 1.0,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 1.0,
186
+ "pass@2": 1.0,
187
+ "pass@4": 1.0,
188
+ "pass@8": 1.0,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.9971923828125,
196
+ "pass@2": 0.9978869340551182,
197
+ "pass@4": 0.9985996801181102,
198
+ "pass@8": 0.9993905663162155,
199
+ "pass@16": 0.9998951809167032,
200
+ "pass@32": 0.9999980558628433,
201
+ "pass@64": 0.9999999999667597,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9949900793650793,
206
+ "pass@2": 0.9973585333083365,
207
+ "pass@4": 0.9983299611358104,
208
+ "pass@8": 0.9991563945664312,
209
+ "pass@16": 0.9997971079638622,
210
+ "pass@32": 0.9999918808807343,
211
+ "pass@64": 0.9999999987391617,
212
+ "pass@128": 1.0
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9971229416167665,
216
+ "pass@2": 0.9976215138620398,
217
+ "pass@4": 0.9981272295117304,
218
+ "pass@8": 0.998847506991621,
219
+ "pass@16": 0.9995863571069055,
220
+ "pass@32": 0.999958801686582,
221
+ "pass@64": 0.9999999176094492,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9964866863905325,
226
+ "pass@2": 0.9979623363462704,
227
+ "pass@4": 0.9990334570789599,
228
+ "pass@8": 0.9997250598303445,
229
+ "pass@16": 0.9999790093720811,
230
+ "pass@32": 0.9999999405368915,
231
+ "pass@64": 0.9999999999999963,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9981971153846154,
236
+ "pass@2": 0.9998284657026045,
237
+ "pass@4": 0.999995967751627,
238
+ "pass@8": 0.9999999975724242,
239
+ "pass@16": 1.0,
240
+ "pass@32": 1.0,
241
+ "pass@64": 1.0,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9949603873239437,
246
+ "pass@2": 0.9973813352556282,
247
+ "pass@4": 0.9982075685081618,
248
+ "pass@8": 0.9989154500291274,
249
+ "pass@16": 0.9996108261180917,
250
+ "pass@32": 0.9999612387699112,
251
+ "pass@64": 0.9999999224832565,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 386609,
259
+ "answer_accuracy": 0.989640502129751,
260
+ "avg_response_len": 197.2131773222477,
261
+ "resp_tokens_sum": 77042511,
262
+ "pass_at_k": {
263
+ "pass@1": 0.989640502129751,
264
+ "pass@2": 0.9919891258217151,
265
+ "pass@4": 0.993503951013332,
266
+ "pass@8": 0.9945439716423881,
267
+ "pass@16": 0.99522975394596,
268
+ "pass@32": 0.9956609074616072,
269
+ "pass@64": 0.9959856221204596,
270
+ "pass@128": 0.9963958060288336
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9309815950920245,
275
+ "pass@2": 0.9449852966040289,
276
+ "pass@4": 0.9559561028797782,
277
+ "pass@8": 0.9638793675635439,
278
+ "pass@16": 0.9687407182381751,
279
+ "pass@32": 0.9718304058076516,
280
+ "pass@64": 0.9746895660017418,
281
+ "pass@128": 0.9785276073619632
282
+ },
283
+ "2": {
284
+ "pass@1": 1.0,
285
+ "pass@2": 1.0,
286
+ "pass@4": 1.0,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9999763257575758,
295
+ "pass@2": 1.0,
296
+ "pass@4": 1.0,
297
+ "pass@8": 1.0,
298
+ "pass@16": 1.0,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9999537721893491,
305
+ "pass@2": 1.0,
306
+ "pass@4": 1.0,
307
+ "pass@8": 1.0,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.99375,
315
+ "pass@2": 0.9960658450302408,
316
+ "pass@4": 0.9975695505453124,
317
+ "pass@8": 0.9984974444758158,
318
+ "pass@16": 0.9992768151012691,
319
+ "pass@32": 0.9998558695467676,
320
+ "pass@64": 0.999998064534979,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9964183381088825,
325
+ "pass@2": 0.9970275026509938,
326
+ "pass@4": 0.9971312728100964,
327
+ "pass@8": 0.9971346683194743,
328
+ "pass@16": 0.997134670487106,
329
+ "pass@32": 0.997134670487106,
330
+ "pass@64": 0.997134670487106,
331
+ "pass@128": 0.997134670487106
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9973582008670521,
335
+ "pass@2": 0.9977495192526513,
336
+ "pass@4": 0.9982556172530457,
337
+ "pass@8": 0.9989655495774096,
338
+ "pass@16": 0.9996567447729243,
339
+ "pass@32": 0.999971393889872,
340
+ "pass@64": 0.9999999651170723,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9936839788732394,
345
+ "pass@2": 0.9963724769879118,
346
+ "pass@4": 0.9971180363018003,
347
+ "pass@8": 0.9971826215324542,
348
+ "pass@16": 0.9971830985804359,
349
+ "pass@32": 0.9971830985915493,
350
+ "pass@64": 0.9971830985915493,
351
+ "pass@128": 0.9971830985915493
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9924214071856288,
355
+ "pass@2": 0.9938524771323495,
356
+ "pass@4": 0.9940081237973997,
357
+ "pass@8": 0.9940119737766329,
358
+ "pass@16": 0.9940119760479041,
359
+ "pass@32": 0.9940119760479041,
360
+ "pass@64": 0.9940119760479041,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 378784,
368
+ "answer_accuracy": 0.9907097422162705,
369
+ "avg_response_len": 210.68409461834617,
370
+ "resp_tokens_sum": 80552114,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9907097422162705,
373
+ "pass@2": 0.9940673034593474,
374
+ "pass@4": 0.995944564240317,
375
+ "pass@8": 0.9970448747202597,
376
+ "pass@16": 0.9977000446657175,
377
+ "pass@32": 0.9981045503059652,
378
+ "pass@64": 0.9984116020425892,
379
+ "pass@128": 0.9986608637428859
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9517957089552239,
384
+ "pass@2": 0.9643014014572805,
385
+ "pass@4": 0.9723098293683435,
386
+ "pass@8": 0.9778178687535791,
387
+ "pass@16": 0.9818233351333638,
388
+ "pass@32": 0.9843700251647998,
389
+ "pass@64": 0.9862015157581675,
390
+ "pass@128": 0.9880597014925373
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9999553571428571,
394
+ "pass@2": 0.9999996484814399,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9998861151603499,
404
+ "pass@2": 0.99999784784555,
405
+ "pass@4": 0.9999999997267104,
406
+ "pass@8": 1.0,
407
+ "pass@16": 1.0,
408
+ "pass@32": 1.0,
409
+ "pass@64": 1.0,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9979669225146199,
414
+ "pass@2": 0.9986840666298292,
415
+ "pass@4": 0.9994179578978066,
416
+ "pass@8": 0.9998914828566827,
417
+ "pass@16": 0.999996971910131,
418
+ "pass@32": 0.9999999992108352,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9915670955882353,
424
+ "pass@2": 0.9952162459471977,
425
+ "pass@4": 0.996770337898939,
426
+ "pass@8": 0.9975063173915031,
427
+ "pass@16": 0.9980350056586529,
428
+ "pass@32": 0.9987689956690733,
429
+ "pass@64": 0.9996410375173692,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9964264589905363,
434
+ "pass@2": 0.9985430276459921,
435
+ "pass@4": 0.9995857823568582,
436
+ "pass@8": 0.9999535526784058,
437
+ "pass@16": 0.9999995379907698,
438
+ "pass@32": 0.9999999999901541,
439
+ "pass@64": 1.0,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9972804588607594,
444
+ "pass@2": 0.9992676511262834,
445
+ "pass@4": 0.9999203209764919,
446
+ "pass@8": 0.9999990565168133,
447
+ "pass@16": 0.9999999999228011,
448
+ "pass@32": 1.0,
449
+ "pass@64": 1.0,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9907563813813813,
454
+ "pass@2": 0.99560153164976,
455
+ "pass@4": 0.998587598565945,
456
+ "pass@8": 0.9998129486244137,
457
+ "pass@16": 0.999996824080759,
458
+ "pass@32": 0.9999999996818268,
459
+ "pass@64": 1.0,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9906300241157556,
464
+ "pass@2": 0.9950913993467858,
465
+ "pass@4": 0.9970629562582811,
466
+ "pass@8": 0.9986055619182425,
467
+ "pass@16": 0.99964486782007,
468
+ "pass@32": 0.9999770906276318,
469
+ "pass@64": 0.9999999831714547,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.6678171875,
478
+ "avg_response_len": 367.13116015625,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.6678171875,
484
+ "pass@2": 0.7378225639763673,
485
+ "pass@4": 0.7893652439351264,
486
+ "pass@8": 0.8270737370351559,
487
+ "pass@16": 0.8550486463709078,
488
+ "pass@32": 0.8767224088959323,
489
+ "pass@64": 0.8945746837166629,
490
+ "pass@128": 0.9096
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.963046875,
495
+ "pass@2": 0.9739500492125989,
496
+ "pass@4": 0.9801725765841772,
497
+ "pass@8": 0.9841731737862502,
498
+ "pass@16": 0.986679806459084,
499
+ "pass@32": 0.9887283379415103,
500
+ "pass@64": 0.9903890917527084,
501
+ "pass@128": 0.991
502
+ },
503
+ "12": {
504
+ "pass@1": 0.9079453125,
505
+ "pass@2": 0.9249128937007879,
506
+ "pass@4": 0.9345272270341208,
507
+ "pass@8": 0.9416891861472778,
508
+ "pass@16": 0.9476419112166174,
509
+ "pass@32": 0.9523055926849554,
510
+ "pass@64": 0.9562749449531718,
511
+ "pass@128": 0.961
512
+ },
513
+ "13": {
514
+ "pass@1": 0.8557421875,
515
+ "pass@2": 0.8929637057086628,
516
+ "pass@4": 0.9139262857142856,
517
+ "pass@8": 0.9291272854564019,
518
+ "pass@16": 0.9415987007431647,
519
+ "pass@32": 0.9518761615883907,
520
+ "pass@64": 0.9606307335381308,
521
+ "pass@128": 0.968
522
+ },
523
+ "14": {
524
+ "pass@1": 0.837,
525
+ "pass@2": 0.8863106545275599,
526
+ "pass@4": 0.9137613407386579,
527
+ "pass@8": 0.9312510179501858,
528
+ "pass@16": 0.9439751905099076,
529
+ "pass@32": 0.9542164522470538,
530
+ "pass@64": 0.9626050622049743,
531
+ "pass@128": 0.969
532
+ },
533
+ "15": {
534
+ "pass@1": 0.77975,
535
+ "pass@2": 0.8448986220472454,
536
+ "pass@4": 0.8857677772778397,
537
+ "pass@8": 0.9115909825900337,
538
+ "pass@16": 0.9277541320088688,
539
+ "pass@32": 0.9388842588529385,
540
+ "pass@64": 0.9474485358545235,
541
+ "pass@128": 0.954
542
+ },
543
+ "16": {
544
+ "pass@1": 0.6545625,
545
+ "pass@2": 0.7510269438976384,
546
+ "pass@4": 0.8116648335208099,
547
+ "pass@8": 0.8497649349116291,
548
+ "pass@16": 0.8757048434440007,
549
+ "pass@32": 0.8956257423100381,
550
+ "pass@64": 0.9124470610164344,
551
+ "pass@128": 0.925
552
+ },
553
+ "17": {
554
+ "pass@1": 0.55796875,
555
+ "pass@2": 0.6625004921259854,
556
+ "pass@4": 0.7365261226096746,
557
+ "pass@8": 0.7870371231754127,
558
+ "pass@16": 0.8224065318779613,
559
+ "pass@32": 0.8483149142099418,
560
+ "pass@64": 0.8693774322654676,
561
+ "pass@128": 0.888
562
+ },
563
+ "18": {
564
+ "pass@1": 0.480296875,
565
+ "pass@2": 0.5879370078740163,
566
+ "pass@4": 0.6728501369516315,
567
+ "pass@8": 0.7364498415117591,
568
+ "pass@16": 0.7835286822486924,
569
+ "pass@32": 0.8195105952369934,
570
+ "pass@64": 0.8482734816022467,
571
+ "pass@128": 0.872
572
+ },
573
+ "19": {
574
+ "pass@1": 0.33475,
575
+ "pass@2": 0.44351464074803204,
576
+ "pass@4": 0.5406705727409072,
577
+ "pass@8": 0.6195248806878444,
578
+ "pass@16": 0.6802573730953311,
579
+ "pass@32": 0.7279540135363097,
580
+ "pass@64": 0.7678995342731354,
581
+ "pass@128": 0.803
582
+ },
583
+ "20": {
584
+ "pass@1": 0.307109375,
585
+ "pass@2": 0.4102106299212602,
586
+ "pass@4": 0.503785566179227,
587
+ "pass@8": 0.5801289441348498,
588
+ "pass@16": 0.6409392921053366,
589
+ "pass@32": 0.6898080203511475,
590
+ "pass@64": 0.7304009597057819,
591
+ "pass@128": 0.765
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.963046875,
596
+ "12": 0.9079453125,
597
+ "13": 0.8557421875,
598
+ "14": 0.837,
599
+ "15": 0.77975,
600
+ "16": 0.6545625,
601
+ "17": 0.55796875,
602
+ "18": 0.480296875,
603
+ "19": 0.33475,
604
+ "20": 0.307109375
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 297.081765625,
608
+ "12": 307.319046875,
609
+ "13": 326.3436640625,
610
+ "14": 353.279515625,
611
+ "15": 371.5872421875,
612
+ "16": 377.5440546875,
613
+ "17": 398.5268984375,
614
+ "18": 408.8486171875,
615
+ "19": 406.905546875,
616
+ "20": 423.87525
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 297.081765625,
621
+ "12": 307.319046875,
622
+ "13": 326.3436640625,
623
+ "14": 353.279515625,
624
+ "15": 371.5872421875,
625
+ "16": 377.5440546875,
626
+ "17": 398.5268984375,
627
+ "18": 408.8486171875,
628
+ "19": 406.905546875,
629
+ "20": 423.87525
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 290947,
636
+ "answer_accuracy": 0.6754898774145617,
637
+ "avg_response_len": 349.3103222511144,
638
+ "resp_tokens_sum": 150454942,
639
+ "pass_at_k": {
640
+ "pass@1": 0.6754898774145617,
641
+ "pass@2": 0.7447737390459964,
642
+ "pass@4": 0.7952060179698968,
643
+ "pass@8": 0.8316698977934396,
644
+ "pass@16": 0.8584897543756141,
645
+ "pass@32": 0.8792215528888129,
646
+ "pass@64": 0.8964337939825147,
647
+ "pass@128": 0.9105497771173848
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9636597025495751,
652
+ "pass@2": 0.9756313990319198,
653
+ "pass@4": 0.9833490821614721,
654
+ "pass@8": 0.9890875592461815,
655
+ "pass@16": 0.9928681364142148,
656
+ "pass@32": 0.9954215669121044,
657
+ "pass@64": 0.9969126772151184,
658
+ "pass@128": 0.9971671388101983
659
+ },
660
+ "12": {
661
+ "pass@1": 0.9084348516949152,
662
+ "pass@2": 0.9271233011922239,
663
+ "pass@4": 0.9376102685257564,
664
+ "pass@8": 0.9448320171203957,
665
+ "pass@16": 0.9500880057183615,
666
+ "pass@32": 0.9533178342586844,
667
+ "pass@64": 0.9561329927476948,
668
+ "pass@128": 0.96045197740113
669
+ },
670
+ "13": {
671
+ "pass@1": 0.8727468652037618,
672
+ "pass@2": 0.9105426036087183,
673
+ "pass@4": 0.9320865086809287,
674
+ "pass@8": 0.9476093057668198,
675
+ "pass@16": 0.9599091371962115,
676
+ "pass@32": 0.9694213182908559,
677
+ "pass@64": 0.9768613994766613,
678
+ "pass@128": 0.9811912225705329
679
+ },
680
+ "14": {
681
+ "pass@1": 0.8451129283489096,
682
+ "pass@2": 0.8927793926460124,
683
+ "pass@4": 0.9188406133812713,
684
+ "pass@8": 0.9352618975128104,
685
+ "pass@16": 0.9480306260712734,
686
+ "pass@32": 0.9589256381658317,
687
+ "pass@64": 0.9683198089214712,
688
+ "pass@128": 0.9750778816199377
689
+ },
690
+ "15": {
691
+ "pass@1": 0.7921107297687862,
692
+ "pass@2": 0.8578609303172365,
693
+ "pass@4": 0.8997546105147255,
694
+ "pass@8": 0.9264640318143393,
695
+ "pass@16": 0.9428917374738313,
696
+ "pass@32": 0.952393511786133,
697
+ "pass@64": 0.9574741921523455,
698
+ "pass@128": 0.9595375722543352
699
+ },
700
+ "16": {
701
+ "pass@1": 0.6568800403225806,
702
+ "pass@2": 0.7477683674117346,
703
+ "pass@4": 0.8092675075897764,
704
+ "pass@8": 0.8516551322611292,
705
+ "pass@16": 0.8812944796205525,
706
+ "pass@32": 0.902773745388599,
707
+ "pass@64": 0.9194705540558217,
708
+ "pass@128": 0.932258064516129
709
+ },
710
+ "17": {
711
+ "pass@1": 0.5657514450867052,
712
+ "pass@2": 0.6691021687679213,
713
+ "pass@4": 0.739191088198512,
714
+ "pass@8": 0.7856207032319886,
715
+ "pass@16": 0.8183619373072404,
716
+ "pass@32": 0.8430109049522699,
717
+ "pass@64": 0.8637531249758412,
718
+ "pass@128": 0.884393063583815
719
+ },
720
+ "18": {
721
+ "pass@1": 0.45412946428571427,
722
+ "pass@2": 0.5638899043869511,
723
+ "pass@4": 0.6530802185441107,
724
+ "pass@8": 0.720726478211569,
725
+ "pass@16": 0.7704510468029769,
726
+ "pass@32": 0.807289056099719,
727
+ "pass@64": 0.8378334239926626,
728
+ "pass@128": 0.8657142857142858
729
+ },
730
+ "19": {
731
+ "pass@1": 0.36369181681681684,
732
+ "pass@2": 0.4703841390603205,
733
+ "pass@4": 0.5581976540657639,
734
+ "pass@8": 0.627149829400672,
735
+ "pass@16": 0.6817550885633032,
736
+ "pass@32": 0.7263554189791901,
737
+ "pass@64": 0.763762224924364,
738
+ "pass@128": 0.7957957957957958
739
+ },
740
+ "20": {
741
+ "pass@1": 0.32453547297297297,
742
+ "pass@2": 0.4275736119978244,
743
+ "pass@4": 0.5169445100330928,
744
+ "pass@8": 0.585449717921024,
745
+ "pass@16": 0.6372318334285229,
746
+ "pass@32": 0.6821752409020172,
747
+ "pass@64": 0.7234653885399501,
748
+ "pass@128": 0.7537537537537538
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 291147,
755
+ "answer_accuracy": 0.6763562109723461,
756
+ "avg_response_len": 362.8061301293488,
757
+ "resp_tokens_sum": 156174978,
758
+ "pass_at_k": {
759
+ "pass@1": 0.6763562109723461,
760
+ "pass@2": 0.7465801048815182,
761
+ "pass@4": 0.7976332155904704,
762
+ "pass@8": 0.8348140204603103,
763
+ "pass@16": 0.8624812646405657,
764
+ "pass@32": 0.883998466128224,
765
+ "pass@64": 0.9017876930240121,
766
+ "pass@128": 0.9170383586083853
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9630082831325302,
771
+ "pass@2": 0.9713395906460487,
772
+ "pass@4": 0.9757507891521089,
773
+ "pass@8": 0.9784955531653872,
774
+ "pass@16": 0.9805383638724173,
775
+ "pass@32": 0.9824747306648515,
776
+ "pass@64": 0.9841054553733752,
777
+ "pass@128": 0.9849397590361446
778
+ },
779
+ "12": {
780
+ "pass@1": 0.9007885971786834,
781
+ "pass@2": 0.915418739046726,
782
+ "pass@4": 0.924029935129582,
783
+ "pass@8": 0.9303735376408099,
784
+ "pass@16": 0.9353211411280159,
785
+ "pass@32": 0.939039317249673,
786
+ "pass@64": 0.9419950990841026,
787
+ "pass@128": 0.9467084639498433
788
+ },
789
+ "13": {
790
+ "pass@1": 0.8611913515406162,
791
+ "pass@2": 0.8988553590727634,
792
+ "pass@4": 0.918313749121695,
793
+ "pass@8": 0.9307861443620137,
794
+ "pass@16": 0.9400313356135247,
795
+ "pass@32": 0.9484692530915426,
796
+ "pass@64": 0.9581816741685208,
797
+ "pass@128": 0.969187675070028
798
+ },
799
+ "14": {
800
+ "pass@1": 0.8469709429824561,
801
+ "pass@2": 0.8980640886172121,
802
+ "pass@4": 0.9245809564374623,
803
+ "pass@8": 0.9405203926008627,
804
+ "pass@16": 0.9525469091473052,
805
+ "pass@32": 0.963199459193292,
806
+ "pass@64": 0.9716362530225532,
807
+ "pass@128": 0.9766081871345029
808
+ },
809
+ "15": {
810
+ "pass@1": 0.7924715909090909,
811
+ "pass@2": 0.861036670842281,
812
+ "pass@4": 0.9029049320539471,
813
+ "pass@8": 0.9281268028368013,
814
+ "pass@16": 0.9426543876503707,
815
+ "pass@32": 0.9525476884583312,
816
+ "pass@64": 0.9611302965599587,
817
+ "pass@128": 0.9696969696969697
818
+ },
819
+ "16": {
820
+ "pass@1": 0.6614583333333334,
821
+ "pass@2": 0.757938160374184,
822
+ "pass@4": 0.8144365875205767,
823
+ "pass@8": 0.8475187224277707,
824
+ "pass@16": 0.8701097260916296,
825
+ "pass@32": 0.8890719227083439,
826
+ "pass@64": 0.9063072562149335,
827
+ "pass@128": 0.9173789173789174
828
+ },
829
+ "17": {
830
+ "pass@1": 0.5830260093167702,
831
+ "pass@2": 0.6846121986110436,
832
+ "pass@4": 0.7533166365656156,
833
+ "pass@8": 0.8007323980276663,
834
+ "pass@16": 0.8367999826994603,
835
+ "pass@32": 0.8654849657428961,
836
+ "pass@64": 0.8891736360856827,
837
+ "pass@128": 0.906832298136646
838
+ },
839
+ "18": {
840
+ "pass@1": 0.5104779411764706,
841
+ "pass@2": 0.615626447429365,
842
+ "pass@4": 0.6993681317408854,
843
+ "pass@8": 0.7636960252154447,
844
+ "pass@16": 0.8126314154686012,
845
+ "pass@32": 0.8493046683312366,
846
+ "pass@64": 0.8760405951252511,
847
+ "pass@128": 0.8970588235294118
848
+ },
849
+ "19": {
850
+ "pass@1": 0.33535621279761907,
851
+ "pass@2": 0.450107066694788,
852
+ "pass@4": 0.5589451405069903,
853
+ "pass@8": 0.6479033713249467,
854
+ "pass@16": 0.7114656294943115,
855
+ "pass@32": 0.7547122095604747,
856
+ "pass@64": 0.7871870069795173,
857
+ "pass@128": 0.8154761904761905
858
+ },
859
+ "20": {
860
+ "pass@1": 0.30761133982035926,
861
+ "pass@2": 0.4103107760856239,
862
+ "pass@4": 0.5020851449519407,
863
+ "pass@8": 0.577641649195555,
864
+ "pass@16": 0.640643393523646,
865
+ "pass@32": 0.6937003584791263,
866
+ "pass@64": 0.7400380698580312,
867
+ "pass@128": 0.7844311377245509
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 272712,
874
+ "answer_accuracy": 0.6511499083129584,
875
+ "avg_response_len": 389.90383605210883,
876
+ "resp_tokens_sum": 163297965,
877
+ "pass_at_k": {
878
+ "pass@1": 0.6511499083129584,
879
+ "pass@2": 0.7216727124564459,
880
+ "pass@4": 0.7748605393801656,
881
+ "pass@8": 0.8143913855344584,
882
+ "pass@16": 0.8438703995259332,
883
+ "pass@32": 0.8666738147613688,
884
+ "pass@64": 0.8852491163739664,
885
+ "pass@128": 0.9009779951100244
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.9624007936507937,
890
+ "pass@2": 0.9748172103487065,
891
+ "pass@4": 0.981273297087864,
892
+ "pass@8": 0.9846499737188555,
893
+ "pass@16": 0.9862178333308695,
894
+ "pass@32": 0.9878187753676408,
895
+ "pass@64": 0.9897012873390828,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.9143969801223242,
900
+ "pass@2": 0.9317818572563751,
901
+ "pass@4": 0.941430099907236,
902
+ "pass@8": 0.9493256684380422,
903
+ "pass@16": 0.9570131901299108,
904
+ "pass@32": 0.9641514897698327,
905
+ "pass@64": 0.9703591098246457,
906
+ "pass@128": 0.9755351681957186
907
+ },
908
+ "13": {
909
+ "pass@1": 0.8329957561728395,
910
+ "pass@2": 0.8691643579274818,
911
+ "pass@4": 0.8912119784216789,
912
+ "pass@8": 0.9091026647516887,
913
+ "pass@16": 0.9252978369245227,
914
+ "pass@32": 0.9383556719133557,
915
+ "pass@64": 0.9473490414410928,
916
+ "pass@128": 0.9537037037037037
917
+ },
918
+ "14": {
919
+ "pass@1": 0.8191533753709199,
920
+ "pass@2": 0.8682212201219658,
921
+ "pass@4": 0.8979430763847394,
922
+ "pass@8": 0.9180236634394026,
923
+ "pass@16": 0.9314134024114261,
924
+ "pass@32": 0.9406145618745281,
925
+ "pass@64": 0.9479964540280973,
926
+ "pass@128": 0.9554896142433235
927
+ },
928
+ "15": {
929
+ "pass@1": 0.7535927854938271,
930
+ "pass@2": 0.8146192554923689,
931
+ "pass@4": 0.8533767113022905,
932
+ "pass@8": 0.878865995821382,
933
+ "pass@16": 0.8964124781429048,
934
+ "pass@32": 0.910541254887923,
935
+ "pass@64": 0.922806998472301,
936
+ "pass@128": 0.9320987654320988
937
+ },
938
+ "16": {
939
+ "pass@1": 0.6453032817109144,
940
+ "pass@2": 0.7468509017954611,
941
+ "pass@4": 0.8109872092868917,
942
+ "pass@8": 0.8503621602906531,
943
+ "pass@16": 0.8763865513376642,
944
+ "pass@32": 0.8958750335367089,
945
+ "pass@64": 0.9123815407896373,
946
+ "pass@128": 0.9262536873156342
947
+ },
948
+ "17": {
949
+ "pass@1": 0.5255553463855421,
950
+ "pass@2": 0.6341747402997818,
951
+ "pass@4": 0.7174640033700606,
952
+ "pass@8": 0.7752305050970983,
953
+ "pass@16": 0.8126617684952755,
954
+ "pass@32": 0.8371897052025431,
955
+ "pass@64": 0.8560389765187822,
956
+ "pass@128": 0.8734939759036144
957
+ },
958
+ "18": {
959
+ "pass@1": 0.4767389112903226,
960
+ "pass@2": 0.5847179006858011,
961
+ "pass@4": 0.66608676022352,
962
+ "pass@8": 0.7243191147240607,
963
+ "pass@16": 0.7663746277687996,
964
+ "pass@32": 0.8006317366757167,
965
+ "pass@64": 0.829606389878158,
966
+ "pass@128": 0.8516129032258064
967
+ },
968
+ "19": {
969
+ "pass@1": 0.30501793806646527,
970
+ "pass@2": 0.40979077955134763,
971
+ "pass@4": 0.5044869689627164,
972
+ "pass@8": 0.5830466910339542,
973
+ "pass@16": 0.6470709277754211,
974
+ "pass@32": 0.7023998688940181,
975
+ "pass@64": 0.7524830182120951,
976
+ "pass@128": 0.797583081570997
977
+ },
978
+ "20": {
979
+ "pass@1": 0.2891798048048048,
980
+ "pass@2": 0.3927472009410985,
981
+ "pass@4": 0.49233214992270086,
982
+ "pass@8": 0.5773029346421416,
983
+ "pass@16": 0.6449435379481746,
984
+ "pass@32": 0.6935367729692132,
985
+ "pass@64": 0.7276704804486359,
986
+ "pass@128": 0.7567567567567568
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.8207002467105263,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.8207002467105263,
1002
+ "pass@2": 0.8587548500310641,
1003
+ "pass@4": 0.8865874391687733,
1004
+ "pass@8": 0.9069153433636171,
1005
+ "pass@16": 0.9219903964714486,
1006
+ "pass@32": 0.9336571744579908,
1007
+ "pass@64": 0.9432681168087804,
1008
+ "pass@128": 0.9513684210526315
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.942828125,
1013
+ "pass@2": 0.9543673720472442,
1014
+ "pass@4": 0.9623778480502434,
1015
+ "pass@8": 0.9682502041026957,
1016
+ "pass@16": 0.972860979589154,
1017
+ "pass@32": 0.9767782353474479,
1018
+ "pass@64": 0.9804701699275815,
1019
+ "pass@128": 0.984
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.999984375,
1023
+ "pass@2": 0.9999998769685039,
1024
+ "pass@4": 1.0,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.999953125,
1033
+ "pass@2": 0.9999992618110236,
1034
+ "pass@4": 0.9999999999062618,
1035
+ "pass@8": 1.0,
1036
+ "pass@16": 1.0,
1037
+ "pass@32": 1.0,
1038
+ "pass@64": 1.0,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.998390625,
1043
+ "pass@2": 0.9988737696850394,
1044
+ "pass@4": 0.9993528392388451,
1045
+ "pass@8": 0.9997678683581743,
1046
+ "pass@16": 0.9999654222866098,
1047
+ "pass@32": 0.9999993776062154,
1048
+ "pass@64": 0.9999999999893633,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.9933984375,
1053
+ "pass@2": 0.9961841781496062,
1054
+ "pass@4": 0.9975373475815523,
1055
+ "pass@8": 0.9983680305456933,
1056
+ "pass@16": 0.9990184921424964,
1057
+ "pass@32": 0.9995291759985511,
1058
+ "pass@64": 0.9998772846233093,
1059
+ "pass@128": 1.0
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.99665625,
1063
+ "pass@2": 0.9977063238188978,
1064
+ "pass@4": 0.9982420018747656,
1065
+ "pass@8": 0.9986003427777526,
1066
+ "pass@16": 0.9988616968167805,
1067
+ "pass@32": 0.9989862397601972,
1068
+ "pass@64": 0.998999972481556,
1069
+ "pass@128": 0.999
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9970390625,
1073
+ "pass@2": 0.9983011811023623,
1074
+ "pass@4": 0.9990445734908138,
1075
+ "pass@8": 0.9995488522357531,
1076
+ "pass@16": 0.9998741388348003,
1077
+ "pass@32": 0.9999900821873651,
1078
+ "pass@64": 0.9999999879305058,
1079
+ "pass@128": 1.0
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.9941171875,
1083
+ "pass@2": 0.9971940206692916,
1084
+ "pass@4": 0.9985053151481066,
1085
+ "pass@8": 0.9989375417785472,
1086
+ "pass@16": 0.9989989424149476,
1087
+ "pass@32": 0.9989999998940483,
1088
+ "pass@64": 0.999,
1089
+ "pass@128": 0.999
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.992765625,
1093
+ "pass@2": 0.9954905265748032,
1094
+ "pass@4": 0.9964489795650546,
1095
+ "pass@8": 0.9971813137583089,
1096
+ "pass@16": 0.9977513971639642,
1097
+ "pass@32": 0.9979791149485119,
1098
+ "pass@64": 0.9979999672478784,
1099
+ "pass@128": 0.998
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.963046875,
1103
+ "pass@2": 0.9739500492125989,
1104
+ "pass@4": 0.9801725765841772,
1105
+ "pass@8": 0.9841731737862502,
1106
+ "pass@16": 0.986679806459084,
1107
+ "pass@32": 0.9887283379415103,
1108
+ "pass@64": 0.9903890917527084,
1109
+ "pass@128": 0.991
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.9079453125,
1113
+ "pass@2": 0.9249128937007879,
1114
+ "pass@4": 0.9345272270341208,
1115
+ "pass@8": 0.9416891861472778,
1116
+ "pass@16": 0.9476419112166174,
1117
+ "pass@32": 0.9523055926849554,
1118
+ "pass@64": 0.9562749449531718,
1119
+ "pass@128": 0.961
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.8557421875,
1123
+ "pass@2": 0.8929637057086628,
1124
+ "pass@4": 0.9139262857142856,
1125
+ "pass@8": 0.9291272854564019,
1126
+ "pass@16": 0.9415987007431647,
1127
+ "pass@32": 0.9518761615883907,
1128
+ "pass@64": 0.9606307335381308,
1129
+ "pass@128": 0.968
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.837,
1133
+ "pass@2": 0.8863106545275599,
1134
+ "pass@4": 0.9137613407386579,
1135
+ "pass@8": 0.9312510179501858,
1136
+ "pass@16": 0.9439751905099076,
1137
+ "pass@32": 0.9542164522470538,
1138
+ "pass@64": 0.9626050622049743,
1139
+ "pass@128": 0.969
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.77975,
1143
+ "pass@2": 0.8448986220472454,
1144
+ "pass@4": 0.8857677772778397,
1145
+ "pass@8": 0.9115909825900337,
1146
+ "pass@16": 0.9277541320088688,
1147
+ "pass@32": 0.9388842588529385,
1148
+ "pass@64": 0.9474485358545235,
1149
+ "pass@128": 0.954
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.6545625,
1153
+ "pass@2": 0.7510269438976384,
1154
+ "pass@4": 0.8116648335208099,
1155
+ "pass@8": 0.8497649349116291,
1156
+ "pass@16": 0.8757048434440007,
1157
+ "pass@32": 0.8956257423100381,
1158
+ "pass@64": 0.9124470610164344,
1159
+ "pass@128": 0.925
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.55796875,
1163
+ "pass@2": 0.6625004921259854,
1164
+ "pass@4": 0.7365261226096746,
1165
+ "pass@8": 0.7870371231754127,
1166
+ "pass@16": 0.8224065318779613,
1167
+ "pass@32": 0.8483149142099418,
1168
+ "pass@64": 0.8693774322654676,
1169
+ "pass@128": 0.888
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.480296875,
1173
+ "pass@2": 0.5879370078740163,
1174
+ "pass@4": 0.6728501369516315,
1175
+ "pass@8": 0.7364498415117591,
1176
+ "pass@16": 0.7835286822486924,
1177
+ "pass@32": 0.8195105952369934,
1178
+ "pass@64": 0.8482734816022467,
1179
+ "pass@128": 0.872
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.33475,
1183
+ "pass@2": 0.44351464074803204,
1184
+ "pass@4": 0.5406705727409072,
1185
+ "pass@8": 0.6195248806878444,
1186
+ "pass@16": 0.6802573730953311,
1187
+ "pass@32": 0.7279540135363097,
1188
+ "pass@64": 0.7678995342731354,
1189
+ "pass@128": 0.803
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.307109375,
1193
+ "pass@2": 0.4102106299212602,
1194
+ "pass@4": 0.503785566179227,
1195
+ "pass@8": 0.5801289441348498,
1196
+ "pass@16": 0.6409392921053366,
1197
+ "pass@32": 0.6898080203511475,
1198
+ "pass@64": 0.7304009597057819,
1199
+ "pass@128": 0.765
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.942828125,
1204
+ "2": 0.999984375,
1205
+ "3": 0.999953125,
1206
+ "4": 0.998390625,
1207
+ "5": 0.9933984375,
1208
+ "6": 0.99665625,
1209
+ "7": 0.9970390625,
1210
+ "8": 0.9941171875,
1211
+ "9": 0.992765625,
1212
+ "11": 0.963046875,
1213
+ "12": 0.9079453125,
1214
+ "13": 0.8557421875,
1215
+ "14": 0.837,
1216
+ "15": 0.77975,
1217
+ "16": 0.6545625,
1218
+ "17": 0.55796875,
1219
+ "18": 0.480296875,
1220
+ "19": 0.33475,
1221
+ "20": 0.307109375
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 279.0269453125,
1225
+ "2": 115.345625,
1226
+ "3": 144.2695546875,
1227
+ "4": 159.5131875,
1228
+ "5": 172.7417421875,
1229
+ "6": 198.263265625,
1230
+ "7": 223.090140625,
1231
+ "8": 242.3848515625,
1232
+ "9": 258.0372109375,
1233
+ "11": 297.081765625,
1234
+ "12": 307.319046875,
1235
+ "13": 326.3436640625,
1236
+ "14": 353.279515625,
1237
+ "15": 371.5872421875,
1238
+ "16": 377.5440546875,
1239
+ "17": 398.5268984375,
1240
+ "18": 408.8486171875,
1241
+ "19": 406.905546875,
1242
+ "20": 423.87525
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 279.0269453125,
1247
+ "2": 115.345625,
1248
+ "3": 144.2695546875,
1249
+ "4": 159.5131875,
1250
+ "5": 172.7417421875,
1251
+ "6": 198.263265625,
1252
+ "7": 223.090140625,
1253
+ "8": 242.3848515625,
1254
+ "9": 258.0372109375,
1255
+ "11": 297.081765625,
1256
+ "12": 307.319046875,
1257
+ "13": 326.3436640625,
1258
+ "14": 353.279515625,
1259
+ "15": 371.5872421875,
1260
+ "16": 377.5440546875,
1261
+ "17": 398.5268984375,
1262
+ "18": 408.8486171875,
1263
+ "19": 406.905546875,
1264
+ "20": 423.87525
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 666691,
1271
+ "answer_accuracy": 0.8233517922067657,
1272
+ "avg_response_len": 274.56429813468225,
1273
+ "resp_tokens_sum": 222322400,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.8233517922067657,
1276
+ "pass@2": 0.8611208180960408,
1277
+ "pass@4": 0.8884471519832107,
1278
+ "pass@8": 0.9082678023063252,
1279
+ "pass@16": 0.9229493914421676,
1280
+ "pass@32": 0.9343577884724095,
1281
+ "pass@64": 0.9438576549065015,
1282
+ "pass@128": 0.9516282010749288
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9453585914454278,
1287
+ "pass@2": 0.9535728491626602,
1288
+ "pass@4": 0.95873854180838,
1289
+ "pass@8": 0.9629986555885872,
1290
+ "pass@16": 0.9679666317812168,
1291
+ "pass@32": 0.9740341139349422,
1292
+ "pass@64": 0.9803653794455086,
1293
+ "pass@128": 0.9852507374631269
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 1.0,
1297
+ "pass@2": 1.0,
1298
+ "pass@4": 1.0,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 1.0,
1307
+ "pass@2": 1.0,
1308
+ "pass@4": 1.0,
1309
+ "pass@8": 1.0,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.9971923828125,
1317
+ "pass@2": 0.9978869340551182,
1318
+ "pass@4": 0.9985996801181102,
1319
+ "pass@8": 0.9993905663162155,
1320
+ "pass@16": 0.9998951809167032,
1321
+ "pass@32": 0.9999980558628433,
1322
+ "pass@64": 0.9999999999667597,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9949900793650793,
1327
+ "pass@2": 0.9973585333083365,
1328
+ "pass@4": 0.9983299611358104,
1329
+ "pass@8": 0.9991563945664312,
1330
+ "pass@16": 0.9997971079638622,
1331
+ "pass@32": 0.9999918808807343,
1332
+ "pass@64": 0.9999999987391617,
1333
+ "pass@128": 1.0
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9971229416167665,
1337
+ "pass@2": 0.9976215138620398,
1338
+ "pass@4": 0.9981272295117304,
1339
+ "pass@8": 0.998847506991621,
1340
+ "pass@16": 0.9995863571069055,
1341
+ "pass@32": 0.999958801686582,
1342
+ "pass@64": 0.9999999176094492,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9964866863905325,
1347
+ "pass@2": 0.9979623363462704,
1348
+ "pass@4": 0.9990334570789599,
1349
+ "pass@8": 0.9997250598303445,
1350
+ "pass@16": 0.9999790093720811,
1351
+ "pass@32": 0.9999999405368915,
1352
+ "pass@64": 0.9999999999999963,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9981971153846154,
1357
+ "pass@2": 0.9998284657026045,
1358
+ "pass@4": 0.999995967751627,
1359
+ "pass@8": 0.9999999975724242,
1360
+ "pass@16": 1.0,
1361
+ "pass@32": 1.0,
1362
+ "pass@64": 1.0,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9949603873239437,
1367
+ "pass@2": 0.9973813352556282,
1368
+ "pass@4": 0.9982075685081618,
1369
+ "pass@8": 0.9989154500291274,
1370
+ "pass@16": 0.9996108261180917,
1371
+ "pass@32": 0.9999612387699112,
1372
+ "pass@64": 0.9999999224832565,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9636597025495751,
1377
+ "pass@2": 0.9756313990319198,
1378
+ "pass@4": 0.9833490821614721,
1379
+ "pass@8": 0.9890875592461815,
1380
+ "pass@16": 0.9928681364142148,
1381
+ "pass@32": 0.9954215669121044,
1382
+ "pass@64": 0.9969126772151184,
1383
+ "pass@128": 0.9971671388101983
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.9084348516949152,
1387
+ "pass@2": 0.9271233011922239,
1388
+ "pass@4": 0.9376102685257564,
1389
+ "pass@8": 0.9448320171203957,
1390
+ "pass@16": 0.9500880057183615,
1391
+ "pass@32": 0.9533178342586844,
1392
+ "pass@64": 0.9561329927476948,
1393
+ "pass@128": 0.96045197740113
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.8727468652037618,
1397
+ "pass@2": 0.9105426036087183,
1398
+ "pass@4": 0.9320865086809287,
1399
+ "pass@8": 0.9476093057668198,
1400
+ "pass@16": 0.9599091371962115,
1401
+ "pass@32": 0.9694213182908559,
1402
+ "pass@64": 0.9768613994766613,
1403
+ "pass@128": 0.9811912225705329
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.8451129283489096,
1407
+ "pass@2": 0.8927793926460124,
1408
+ "pass@4": 0.9188406133812713,
1409
+ "pass@8": 0.9352618975128104,
1410
+ "pass@16": 0.9480306260712734,
1411
+ "pass@32": 0.9589256381658317,
1412
+ "pass@64": 0.9683198089214712,
1413
+ "pass@128": 0.9750778816199377
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.7921107297687862,
1417
+ "pass@2": 0.8578609303172365,
1418
+ "pass@4": 0.8997546105147255,
1419
+ "pass@8": 0.9264640318143393,
1420
+ "pass@16": 0.9428917374738313,
1421
+ "pass@32": 0.952393511786133,
1422
+ "pass@64": 0.9574741921523455,
1423
+ "pass@128": 0.9595375722543352
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.6568800403225806,
1427
+ "pass@2": 0.7477683674117346,
1428
+ "pass@4": 0.8092675075897764,
1429
+ "pass@8": 0.8516551322611292,
1430
+ "pass@16": 0.8812944796205525,
1431
+ "pass@32": 0.902773745388599,
1432
+ "pass@64": 0.9194705540558217,
1433
+ "pass@128": 0.932258064516129
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.5657514450867052,
1437
+ "pass@2": 0.6691021687679213,
1438
+ "pass@4": 0.739191088198512,
1439
+ "pass@8": 0.7856207032319886,
1440
+ "pass@16": 0.8183619373072404,
1441
+ "pass@32": 0.8430109049522699,
1442
+ "pass@64": 0.8637531249758412,
1443
+ "pass@128": 0.884393063583815
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.45412946428571427,
1447
+ "pass@2": 0.5638899043869511,
1448
+ "pass@4": 0.6530802185441107,
1449
+ "pass@8": 0.720726478211569,
1450
+ "pass@16": 0.7704510468029769,
1451
+ "pass@32": 0.807289056099719,
1452
+ "pass@64": 0.8378334239926626,
1453
+ "pass@128": 0.8657142857142858
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.36369181681681684,
1457
+ "pass@2": 0.4703841390603205,
1458
+ "pass@4": 0.5581976540657639,
1459
+ "pass@8": 0.627149829400672,
1460
+ "pass@16": 0.6817550885633032,
1461
+ "pass@32": 0.7263554189791901,
1462
+ "pass@64": 0.763762224924364,
1463
+ "pass@128": 0.7957957957957958
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.32453547297297297,
1467
+ "pass@2": 0.4275736119978244,
1468
+ "pass@4": 0.5169445100330928,
1469
+ "pass@8": 0.585449717921024,
1470
+ "pass@16": 0.6372318334285229,
1471
+ "pass@32": 0.6821752409020172,
1472
+ "pass@64": 0.7234653885399501,
1473
+ "pass@128": 0.7537537537537538
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 677756,
1480
+ "answer_accuracy": 0.8254043257989088,
1481
+ "avg_response_len": 284.02363722720185,
1482
+ "resp_tokens_sum": 233217489,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.8254043257989088,
1485
+ "pass@2": 0.8633358853818162,
1486
+ "pass@4": 0.8908206644619501,
1487
+ "pass@8": 0.9108071320749088,
1488
+ "pass@16": 0.9256376776351245,
1489
+ "pass@32": 0.9371229822544132,
1490
+ "pass@64": 0.9466032938973412,
1491
+ "pass@128": 0.9547934528448948
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9309815950920245,
1496
+ "pass@2": 0.9449852966040289,
1497
+ "pass@4": 0.9559561028797782,
1498
+ "pass@8": 0.9638793675635439,
1499
+ "pass@16": 0.9687407182381751,
1500
+ "pass@32": 0.9718304058076516,
1501
+ "pass@64": 0.9746895660017418,
1502
+ "pass@128": 0.9785276073619632
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 1.0,
1506
+ "pass@2": 1.0,
1507
+ "pass@4": 1.0,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9999763257575758,
1516
+ "pass@2": 1.0,
1517
+ "pass@4": 1.0,
1518
+ "pass@8": 1.0,
1519
+ "pass@16": 1.0,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9999537721893491,
1526
+ "pass@2": 1.0,
1527
+ "pass@4": 1.0,
1528
+ "pass@8": 1.0,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.99375,
1536
+ "pass@2": 0.9960658450302408,
1537
+ "pass@4": 0.9975695505453124,
1538
+ "pass@8": 0.9984974444758158,
1539
+ "pass@16": 0.9992768151012691,
1540
+ "pass@32": 0.9998558695467676,
1541
+ "pass@64": 0.999998064534979,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9964183381088825,
1546
+ "pass@2": 0.9970275026509938,
1547
+ "pass@4": 0.9971312728100964,
1548
+ "pass@8": 0.9971346683194743,
1549
+ "pass@16": 0.997134670487106,
1550
+ "pass@32": 0.997134670487106,
1551
+ "pass@64": 0.997134670487106,
1552
+ "pass@128": 0.997134670487106
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9973582008670521,
1556
+ "pass@2": 0.9977495192526513,
1557
+ "pass@4": 0.9982556172530457,
1558
+ "pass@8": 0.9989655495774096,
1559
+ "pass@16": 0.9996567447729243,
1560
+ "pass@32": 0.999971393889872,
1561
+ "pass@64": 0.9999999651170723,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9936839788732394,
1566
+ "pass@2": 0.9963724769879118,
1567
+ "pass@4": 0.9971180363018003,
1568
+ "pass@8": 0.9971826215324542,
1569
+ "pass@16": 0.9971830985804359,
1570
+ "pass@32": 0.9971830985915493,
1571
+ "pass@64": 0.9971830985915493,
1572
+ "pass@128": 0.9971830985915493
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9924214071856288,
1576
+ "pass@2": 0.9938524771323495,
1577
+ "pass@4": 0.9940081237973997,
1578
+ "pass@8": 0.9940119737766329,
1579
+ "pass@16": 0.9940119760479041,
1580
+ "pass@32": 0.9940119760479041,
1581
+ "pass@64": 0.9940119760479041,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9630082831325302,
1586
+ "pass@2": 0.9713395906460487,
1587
+ "pass@4": 0.9757507891521089,
1588
+ "pass@8": 0.9784955531653872,
1589
+ "pass@16": 0.9805383638724173,
1590
+ "pass@32": 0.9824747306648515,
1591
+ "pass@64": 0.9841054553733752,
1592
+ "pass@128": 0.9849397590361446
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.9007885971786834,
1596
+ "pass@2": 0.915418739046726,
1597
+ "pass@4": 0.924029935129582,
1598
+ "pass@8": 0.9303735376408099,
1599
+ "pass@16": 0.9353211411280159,
1600
+ "pass@32": 0.939039317249673,
1601
+ "pass@64": 0.9419950990841026,
1602
+ "pass@128": 0.9467084639498433
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.8611913515406162,
1606
+ "pass@2": 0.8988553590727634,
1607
+ "pass@4": 0.918313749121695,
1608
+ "pass@8": 0.9307861443620137,
1609
+ "pass@16": 0.9400313356135247,
1610
+ "pass@32": 0.9484692530915426,
1611
+ "pass@64": 0.9581816741685208,
1612
+ "pass@128": 0.969187675070028
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.8469709429824561,
1616
+ "pass@2": 0.8980640886172121,
1617
+ "pass@4": 0.9245809564374623,
1618
+ "pass@8": 0.9405203926008627,
1619
+ "pass@16": 0.9525469091473052,
1620
+ "pass@32": 0.963199459193292,
1621
+ "pass@64": 0.9716362530225532,
1622
+ "pass@128": 0.9766081871345029
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.7924715909090909,
1626
+ "pass@2": 0.861036670842281,
1627
+ "pass@4": 0.9029049320539471,
1628
+ "pass@8": 0.9281268028368013,
1629
+ "pass@16": 0.9426543876503707,
1630
+ "pass@32": 0.9525476884583312,
1631
+ "pass@64": 0.9611302965599587,
1632
+ "pass@128": 0.9696969696969697
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.6614583333333334,
1636
+ "pass@2": 0.757938160374184,
1637
+ "pass@4": 0.8144365875205767,
1638
+ "pass@8": 0.8475187224277707,
1639
+ "pass@16": 0.8701097260916296,
1640
+ "pass@32": 0.8890719227083439,
1641
+ "pass@64": 0.9063072562149335,
1642
+ "pass@128": 0.9173789173789174
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.5830260093167702,
1646
+ "pass@2": 0.6846121986110436,
1647
+ "pass@4": 0.7533166365656156,
1648
+ "pass@8": 0.8007323980276663,
1649
+ "pass@16": 0.8367999826994603,
1650
+ "pass@32": 0.8654849657428961,
1651
+ "pass@64": 0.8891736360856827,
1652
+ "pass@128": 0.906832298136646
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.5104779411764706,
1656
+ "pass@2": 0.615626447429365,
1657
+ "pass@4": 0.6993681317408854,
1658
+ "pass@8": 0.7636960252154447,
1659
+ "pass@16": 0.8126314154686012,
1660
+ "pass@32": 0.8493046683312366,
1661
+ "pass@64": 0.8760405951252511,
1662
+ "pass@128": 0.8970588235294118
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.33535621279761907,
1666
+ "pass@2": 0.450107066694788,
1667
+ "pass@4": 0.5589451405069903,
1668
+ "pass@8": 0.6479033713249467,
1669
+ "pass@16": 0.7114656294943115,
1670
+ "pass@32": 0.7547122095604747,
1671
+ "pass@64": 0.7871870069795173,
1672
+ "pass@128": 0.8154761904761905
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.30761133982035926,
1676
+ "pass@2": 0.4103107760856239,
1677
+ "pass@4": 0.5020851449519407,
1678
+ "pass@8": 0.577641649195555,
1679
+ "pass@16": 0.640643393523646,
1680
+ "pass@32": 0.6937003584791263,
1681
+ "pass@64": 0.7400380698580312,
1682
+ "pass@128": 0.7844311377245509
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 651496,
1689
+ "answer_accuracy": 0.8131989934494328,
1690
+ "avg_response_len": 304.374299758348,
1691
+ "resp_tokens_sum": 243850079,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.8131989934494328,
1694
+ "pass@2": 0.8516683416824594,
1695
+ "pass@4": 0.8803690842367309,
1696
+ "pass@8": 0.9015596188301824,
1697
+ "pass@16": 0.9172829494592375,
1698
+ "pass@32": 0.9293968706923023,
1699
+ "pass@64": 0.9392539645433563,
1700
+ "pass@128": 0.9475954625339511
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9517957089552239,
1705
+ "pass@2": 0.9643014014572805,
1706
+ "pass@4": 0.9723098293683435,
1707
+ "pass@8": 0.9778178687535791,
1708
+ "pass@16": 0.9818233351333638,
1709
+ "pass@32": 0.9843700251647998,
1710
+ "pass@64": 0.9862015157581675,
1711
+ "pass@128": 0.9880597014925373
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9999553571428571,
1715
+ "pass@2": 0.9999996484814399,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9998861151603499,
1725
+ "pass@2": 0.99999784784555,
1726
+ "pass@4": 0.9999999997267104,
1727
+ "pass@8": 1.0,
1728
+ "pass@16": 1.0,
1729
+ "pass@32": 1.0,
1730
+ "pass@64": 1.0,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9979669225146199,
1735
+ "pass@2": 0.9986840666298292,
1736
+ "pass@4": 0.9994179578978066,
1737
+ "pass@8": 0.9998914828566827,
1738
+ "pass@16": 0.999996971910131,
1739
+ "pass@32": 0.9999999992108352,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9915670955882353,
1745
+ "pass@2": 0.9952162459471977,
1746
+ "pass@4": 0.996770337898939,
1747
+ "pass@8": 0.9975063173915031,
1748
+ "pass@16": 0.9980350056586529,
1749
+ "pass@32": 0.9987689956690733,
1750
+ "pass@64": 0.9996410375173692,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9964264589905363,
1755
+ "pass@2": 0.9985430276459921,
1756
+ "pass@4": 0.9995857823568582,
1757
+ "pass@8": 0.9999535526784058,
1758
+ "pass@16": 0.9999995379907698,
1759
+ "pass@32": 0.9999999999901541,
1760
+ "pass@64": 1.0,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9972804588607594,
1765
+ "pass@2": 0.9992676511262834,
1766
+ "pass@4": 0.9999203209764919,
1767
+ "pass@8": 0.9999990565168133,
1768
+ "pass@16": 0.9999999999228011,
1769
+ "pass@32": 1.0,
1770
+ "pass@64": 1.0,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9907563813813813,
1775
+ "pass@2": 0.99560153164976,
1776
+ "pass@4": 0.998587598565945,
1777
+ "pass@8": 0.9998129486244137,
1778
+ "pass@16": 0.999996824080759,
1779
+ "pass@32": 0.9999999996818268,
1780
+ "pass@64": 1.0,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9906300241157556,
1785
+ "pass@2": 0.9950913993467858,
1786
+ "pass@4": 0.9970629562582811,
1787
+ "pass@8": 0.9986055619182425,
1788
+ "pass@16": 0.99964486782007,
1789
+ "pass@32": 0.9999770906276318,
1790
+ "pass@64": 0.9999999831714547,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.9624007936507937,
1795
+ "pass@2": 0.9748172103487065,
1796
+ "pass@4": 0.981273297087864,
1797
+ "pass@8": 0.9846499737188555,
1798
+ "pass@16": 0.9862178333308695,
1799
+ "pass@32": 0.9878187753676408,
1800
+ "pass@64": 0.9897012873390828,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.9143969801223242,
1805
+ "pass@2": 0.9317818572563751,
1806
+ "pass@4": 0.941430099907236,
1807
+ "pass@8": 0.9493256684380422,
1808
+ "pass@16": 0.9570131901299108,
1809
+ "pass@32": 0.9641514897698327,
1810
+ "pass@64": 0.9703591098246457,
1811
+ "pass@128": 0.9755351681957186
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.8329957561728395,
1815
+ "pass@2": 0.8691643579274818,
1816
+ "pass@4": 0.8912119784216789,
1817
+ "pass@8": 0.9091026647516887,
1818
+ "pass@16": 0.9252978369245227,
1819
+ "pass@32": 0.9383556719133557,
1820
+ "pass@64": 0.9473490414410928,
1821
+ "pass@128": 0.9537037037037037
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.8191533753709199,
1825
+ "pass@2": 0.8682212201219658,
1826
+ "pass@4": 0.8979430763847394,
1827
+ "pass@8": 0.9180236634394026,
1828
+ "pass@16": 0.9314134024114261,
1829
+ "pass@32": 0.9406145618745281,
1830
+ "pass@64": 0.9479964540280973,
1831
+ "pass@128": 0.9554896142433235
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.7535927854938271,
1835
+ "pass@2": 0.8146192554923689,
1836
+ "pass@4": 0.8533767113022905,
1837
+ "pass@8": 0.878865995821382,
1838
+ "pass@16": 0.8964124781429048,
1839
+ "pass@32": 0.910541254887923,
1840
+ "pass@64": 0.922806998472301,
1841
+ "pass@128": 0.9320987654320988
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.6453032817109144,
1845
+ "pass@2": 0.7468509017954611,
1846
+ "pass@4": 0.8109872092868917,
1847
+ "pass@8": 0.8503621602906531,
1848
+ "pass@16": 0.8763865513376642,
1849
+ "pass@32": 0.8958750335367089,
1850
+ "pass@64": 0.9123815407896373,
1851
+ "pass@128": 0.9262536873156342
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.5255553463855421,
1855
+ "pass@2": 0.6341747402997818,
1856
+ "pass@4": 0.7174640033700606,
1857
+ "pass@8": 0.7752305050970983,
1858
+ "pass@16": 0.8126617684952755,
1859
+ "pass@32": 0.8371897052025431,
1860
+ "pass@64": 0.8560389765187822,
1861
+ "pass@128": 0.8734939759036144
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.4767389112903226,
1865
+ "pass@2": 0.5847179006858011,
1866
+ "pass@4": 0.66608676022352,
1867
+ "pass@8": 0.7243191147240607,
1868
+ "pass@16": 0.7663746277687996,
1869
+ "pass@32": 0.8006317366757167,
1870
+ "pass@64": 0.829606389878158,
1871
+ "pass@128": 0.8516129032258064
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.30501793806646527,
1875
+ "pass@2": 0.40979077955134763,
1876
+ "pass@4": 0.5044869689627164,
1877
+ "pass@8": 0.5830466910339542,
1878
+ "pass@16": 0.6470709277754211,
1879
+ "pass@32": 0.7023998688940181,
1880
+ "pass@64": 0.7524830182120951,
1881
+ "pass@128": 0.797583081570997
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.2891798048048048,
1885
+ "pass@2": 0.3927472009410985,
1886
+ "pass@4": 0.49233214992270086,
1887
+ "pass@8": 0.5773029346421416,
1888
+ "pass@16": 0.6449435379481746,
1889
+ "pass@32": 0.6935367729692132,
1890
+ "pass@64": 0.7276704804486359,
1891
+ "pass@128": 0.7567567567567568
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch4-op11-14/SDAR-100M-block16-epoch4-op11-14_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8240e70e3f31767a356d1233f48142ad393af64f291e3e2a8dd5e290e3951805
3
+ size 2693207340
results/composition/SDAR-100M-block16-epoch4-op11-14/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch4-op11-14,0.9905703125,0.6678171875,0.8207002467105263,0.0,0.0,0.0,199.1858359375,367.13116015625,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch4-op11-14/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch4-op11-14",
4
+ "id_acc": 0.9905703125,
5
+ "ood_acc": 0.6678171875,
6
+ "total_acc": 0.8207002467105263,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.1858359375,
11
+ "ood_avg_resp_len": 367.13116015625,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 666691,
19
+ "answer_accuracy": 0.8233517922067657,
20
+ "avg_response_len": 274.56429813468225,
21
+ "resp_tokens_sum": 222322400,
22
+ "pass_at_k": {
23
+ "pass@1": 0.8233517922067657,
24
+ "pass@2": 0.8611208180960408,
25
+ "pass@4": 0.8884471519832107,
26
+ "pass@8": 0.9082678023063252,
27
+ "pass@16": 0.9229493914421676,
28
+ "pass@32": 0.9343577884724095,
29
+ "pass@64": 0.9438576549065015,
30
+ "pass@128": 0.9516282010749288
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9453585914454278,
35
+ "pass@2": 0.9535728491626602,
36
+ "pass@4": 0.95873854180838,
37
+ "pass@8": 0.9629986555885872,
38
+ "pass@16": 0.9679666317812168,
39
+ "pass@32": 0.9740341139349422,
40
+ "pass@64": 0.9803653794455086,
41
+ "pass@128": 0.9852507374631269
42
+ },
43
+ "2": {
44
+ "pass@1": 1.0,
45
+ "pass@2": 1.0,
46
+ "pass@4": 1.0,
47
+ "pass@8": 1.0,
48
+ "pass@16": 1.0,
49
+ "pass@32": 1.0,
50
+ "pass@64": 1.0,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 1.0,
55
+ "pass@2": 1.0,
56
+ "pass@4": 1.0,
57
+ "pass@8": 1.0,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.9971923828125,
65
+ "pass@2": 0.9978869340551182,
66
+ "pass@4": 0.9985996801181102,
67
+ "pass@8": 0.9993905663162155,
68
+ "pass@16": 0.9998951809167032,
69
+ "pass@32": 0.9999980558628433,
70
+ "pass@64": 0.9999999999667597,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9949900793650793,
75
+ "pass@2": 0.9973585333083365,
76
+ "pass@4": 0.9983299611358104,
77
+ "pass@8": 0.9991563945664312,
78
+ "pass@16": 0.9997971079638622,
79
+ "pass@32": 0.9999918808807343,
80
+ "pass@64": 0.9999999987391617,
81
+ "pass@128": 1.0
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9971229416167665,
85
+ "pass@2": 0.9976215138620398,
86
+ "pass@4": 0.9981272295117304,
87
+ "pass@8": 0.998847506991621,
88
+ "pass@16": 0.9995863571069055,
89
+ "pass@32": 0.999958801686582,
90
+ "pass@64": 0.9999999176094492,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9964866863905325,
95
+ "pass@2": 0.9979623363462704,
96
+ "pass@4": 0.9990334570789599,
97
+ "pass@8": 0.9997250598303445,
98
+ "pass@16": 0.9999790093720811,
99
+ "pass@32": 0.9999999405368915,
100
+ "pass@64": 0.9999999999999963,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9981971153846154,
105
+ "pass@2": 0.9998284657026045,
106
+ "pass@4": 0.999995967751627,
107
+ "pass@8": 0.9999999975724242,
108
+ "pass@16": 1.0,
109
+ "pass@32": 1.0,
110
+ "pass@64": 1.0,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9949603873239437,
115
+ "pass@2": 0.9973813352556282,
116
+ "pass@4": 0.9982075685081618,
117
+ "pass@8": 0.9989154500291274,
118
+ "pass@16": 0.9996108261180917,
119
+ "pass@32": 0.9999612387699112,
120
+ "pass@64": 0.9999999224832565,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.9636597025495751,
125
+ "pass@2": 0.9756313990319198,
126
+ "pass@4": 0.9833490821614721,
127
+ "pass@8": 0.9890875592461815,
128
+ "pass@16": 0.9928681364142148,
129
+ "pass@32": 0.9954215669121044,
130
+ "pass@64": 0.9969126772151184,
131
+ "pass@128": 0.9971671388101983
132
+ },
133
+ "12": {
134
+ "pass@1": 0.9084348516949152,
135
+ "pass@2": 0.9271233011922239,
136
+ "pass@4": 0.9376102685257564,
137
+ "pass@8": 0.9448320171203957,
138
+ "pass@16": 0.9500880057183615,
139
+ "pass@32": 0.9533178342586844,
140
+ "pass@64": 0.9561329927476948,
141
+ "pass@128": 0.96045197740113
142
+ },
143
+ "13": {
144
+ "pass@1": 0.8727468652037618,
145
+ "pass@2": 0.9105426036087183,
146
+ "pass@4": 0.9320865086809287,
147
+ "pass@8": 0.9476093057668198,
148
+ "pass@16": 0.9599091371962115,
149
+ "pass@32": 0.9694213182908559,
150
+ "pass@64": 0.9768613994766613,
151
+ "pass@128": 0.9811912225705329
152
+ },
153
+ "14": {
154
+ "pass@1": 0.8451129283489096,
155
+ "pass@2": 0.8927793926460124,
156
+ "pass@4": 0.9188406133812713,
157
+ "pass@8": 0.9352618975128104,
158
+ "pass@16": 0.9480306260712734,
159
+ "pass@32": 0.9589256381658317,
160
+ "pass@64": 0.9683198089214712,
161
+ "pass@128": 0.9750778816199377
162
+ },
163
+ "15": {
164
+ "pass@1": 0.7921107297687862,
165
+ "pass@2": 0.8578609303172365,
166
+ "pass@4": 0.8997546105147255,
167
+ "pass@8": 0.9264640318143393,
168
+ "pass@16": 0.9428917374738313,
169
+ "pass@32": 0.952393511786133,
170
+ "pass@64": 0.9574741921523455,
171
+ "pass@128": 0.9595375722543352
172
+ },
173
+ "16": {
174
+ "pass@1": 0.6568800403225806,
175
+ "pass@2": 0.7477683674117346,
176
+ "pass@4": 0.8092675075897764,
177
+ "pass@8": 0.8516551322611292,
178
+ "pass@16": 0.8812944796205525,
179
+ "pass@32": 0.902773745388599,
180
+ "pass@64": 0.9194705540558217,
181
+ "pass@128": 0.932258064516129
182
+ },
183
+ "17": {
184
+ "pass@1": 0.5657514450867052,
185
+ "pass@2": 0.6691021687679213,
186
+ "pass@4": 0.739191088198512,
187
+ "pass@8": 0.7856207032319886,
188
+ "pass@16": 0.8183619373072404,
189
+ "pass@32": 0.8430109049522699,
190
+ "pass@64": 0.8637531249758412,
191
+ "pass@128": 0.884393063583815
192
+ },
193
+ "18": {
194
+ "pass@1": 0.45412946428571427,
195
+ "pass@2": 0.5638899043869511,
196
+ "pass@4": 0.6530802185441107,
197
+ "pass@8": 0.720726478211569,
198
+ "pass@16": 0.7704510468029769,
199
+ "pass@32": 0.807289056099719,
200
+ "pass@64": 0.8378334239926626,
201
+ "pass@128": 0.8657142857142858
202
+ },
203
+ "19": {
204
+ "pass@1": 0.36369181681681684,
205
+ "pass@2": 0.4703841390603205,
206
+ "pass@4": 0.5581976540657639,
207
+ "pass@8": 0.627149829400672,
208
+ "pass@16": 0.6817550885633032,
209
+ "pass@32": 0.7263554189791901,
210
+ "pass@64": 0.763762224924364,
211
+ "pass@128": 0.7957957957957958
212
+ },
213
+ "20": {
214
+ "pass@1": 0.32453547297297297,
215
+ "pass@2": 0.4275736119978244,
216
+ "pass@4": 0.5169445100330928,
217
+ "pass@8": 0.585449717921024,
218
+ "pass@16": 0.6372318334285229,
219
+ "pass@32": 0.6821752409020172,
220
+ "pass@64": 0.7234653885399501,
221
+ "pass@128": 0.7537537537537538
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 677756,
228
+ "answer_accuracy": 0.8254043257989088,
229
+ "avg_response_len": 284.02363722720185,
230
+ "resp_tokens_sum": 233217489,
231
+ "pass_at_k": {
232
+ "pass@1": 0.8254043257989088,
233
+ "pass@2": 0.8633358853818162,
234
+ "pass@4": 0.8908206644619501,
235
+ "pass@8": 0.9108071320749088,
236
+ "pass@16": 0.9256376776351245,
237
+ "pass@32": 0.9371229822544132,
238
+ "pass@64": 0.9466032938973412,
239
+ "pass@128": 0.9547934528448948
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9309815950920245,
244
+ "pass@2": 0.9449852966040289,
245
+ "pass@4": 0.9559561028797782,
246
+ "pass@8": 0.9638793675635439,
247
+ "pass@16": 0.9687407182381751,
248
+ "pass@32": 0.9718304058076516,
249
+ "pass@64": 0.9746895660017418,
250
+ "pass@128": 0.9785276073619632
251
+ },
252
+ "2": {
253
+ "pass@1": 1.0,
254
+ "pass@2": 1.0,
255
+ "pass@4": 1.0,
256
+ "pass@8": 1.0,
257
+ "pass@16": 1.0,
258
+ "pass@32": 1.0,
259
+ "pass@64": 1.0,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9999763257575758,
264
+ "pass@2": 1.0,
265
+ "pass@4": 1.0,
266
+ "pass@8": 1.0,
267
+ "pass@16": 1.0,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9999537721893491,
274
+ "pass@2": 1.0,
275
+ "pass@4": 1.0,
276
+ "pass@8": 1.0,
277
+ "pass@16": 1.0,
278
+ "pass@32": 1.0,
279
+ "pass@64": 1.0,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.99375,
284
+ "pass@2": 0.9960658450302408,
285
+ "pass@4": 0.9975695505453124,
286
+ "pass@8": 0.9984974444758158,
287
+ "pass@16": 0.9992768151012691,
288
+ "pass@32": 0.9998558695467676,
289
+ "pass@64": 0.999998064534979,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9964183381088825,
294
+ "pass@2": 0.9970275026509938,
295
+ "pass@4": 0.9971312728100964,
296
+ "pass@8": 0.9971346683194743,
297
+ "pass@16": 0.997134670487106,
298
+ "pass@32": 0.997134670487106,
299
+ "pass@64": 0.997134670487106,
300
+ "pass@128": 0.997134670487106
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9973582008670521,
304
+ "pass@2": 0.9977495192526513,
305
+ "pass@4": 0.9982556172530457,
306
+ "pass@8": 0.9989655495774096,
307
+ "pass@16": 0.9996567447729243,
308
+ "pass@32": 0.999971393889872,
309
+ "pass@64": 0.9999999651170723,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9936839788732394,
314
+ "pass@2": 0.9963724769879118,
315
+ "pass@4": 0.9971180363018003,
316
+ "pass@8": 0.9971826215324542,
317
+ "pass@16": 0.9971830985804359,
318
+ "pass@32": 0.9971830985915493,
319
+ "pass@64": 0.9971830985915493,
320
+ "pass@128": 0.9971830985915493
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9924214071856288,
324
+ "pass@2": 0.9938524771323495,
325
+ "pass@4": 0.9940081237973997,
326
+ "pass@8": 0.9940119737766329,
327
+ "pass@16": 0.9940119760479041,
328
+ "pass@32": 0.9940119760479041,
329
+ "pass@64": 0.9940119760479041,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9630082831325302,
334
+ "pass@2": 0.9713395906460487,
335
+ "pass@4": 0.9757507891521089,
336
+ "pass@8": 0.9784955531653872,
337
+ "pass@16": 0.9805383638724173,
338
+ "pass@32": 0.9824747306648515,
339
+ "pass@64": 0.9841054553733752,
340
+ "pass@128": 0.9849397590361446
341
+ },
342
+ "12": {
343
+ "pass@1": 0.9007885971786834,
344
+ "pass@2": 0.915418739046726,
345
+ "pass@4": 0.924029935129582,
346
+ "pass@8": 0.9303735376408099,
347
+ "pass@16": 0.9353211411280159,
348
+ "pass@32": 0.939039317249673,
349
+ "pass@64": 0.9419950990841026,
350
+ "pass@128": 0.9467084639498433
351
+ },
352
+ "13": {
353
+ "pass@1": 0.8611913515406162,
354
+ "pass@2": 0.8988553590727634,
355
+ "pass@4": 0.918313749121695,
356
+ "pass@8": 0.9307861443620137,
357
+ "pass@16": 0.9400313356135247,
358
+ "pass@32": 0.9484692530915426,
359
+ "pass@64": 0.9581816741685208,
360
+ "pass@128": 0.969187675070028
361
+ },
362
+ "14": {
363
+ "pass@1": 0.8469709429824561,
364
+ "pass@2": 0.8980640886172121,
365
+ "pass@4": 0.9245809564374623,
366
+ "pass@8": 0.9405203926008627,
367
+ "pass@16": 0.9525469091473052,
368
+ "pass@32": 0.963199459193292,
369
+ "pass@64": 0.9716362530225532,
370
+ "pass@128": 0.9766081871345029
371
+ },
372
+ "15": {
373
+ "pass@1": 0.7924715909090909,
374
+ "pass@2": 0.861036670842281,
375
+ "pass@4": 0.9029049320539471,
376
+ "pass@8": 0.9281268028368013,
377
+ "pass@16": 0.9426543876503707,
378
+ "pass@32": 0.9525476884583312,
379
+ "pass@64": 0.9611302965599587,
380
+ "pass@128": 0.9696969696969697
381
+ },
382
+ "16": {
383
+ "pass@1": 0.6614583333333334,
384
+ "pass@2": 0.757938160374184,
385
+ "pass@4": 0.8144365875205767,
386
+ "pass@8": 0.8475187224277707,
387
+ "pass@16": 0.8701097260916296,
388
+ "pass@32": 0.8890719227083439,
389
+ "pass@64": 0.9063072562149335,
390
+ "pass@128": 0.9173789173789174
391
+ },
392
+ "17": {
393
+ "pass@1": 0.5830260093167702,
394
+ "pass@2": 0.6846121986110436,
395
+ "pass@4": 0.7533166365656156,
396
+ "pass@8": 0.8007323980276663,
397
+ "pass@16": 0.8367999826994603,
398
+ "pass@32": 0.8654849657428961,
399
+ "pass@64": 0.8891736360856827,
400
+ "pass@128": 0.906832298136646
401
+ },
402
+ "18": {
403
+ "pass@1": 0.5104779411764706,
404
+ "pass@2": 0.615626447429365,
405
+ "pass@4": 0.6993681317408854,
406
+ "pass@8": 0.7636960252154447,
407
+ "pass@16": 0.8126314154686012,
408
+ "pass@32": 0.8493046683312366,
409
+ "pass@64": 0.8760405951252511,
410
+ "pass@128": 0.8970588235294118
411
+ },
412
+ "19": {
413
+ "pass@1": 0.33535621279761907,
414
+ "pass@2": 0.450107066694788,
415
+ "pass@4": 0.5589451405069903,
416
+ "pass@8": 0.6479033713249467,
417
+ "pass@16": 0.7114656294943115,
418
+ "pass@32": 0.7547122095604747,
419
+ "pass@64": 0.7871870069795173,
420
+ "pass@128": 0.8154761904761905
421
+ },
422
+ "20": {
423
+ "pass@1": 0.30761133982035926,
424
+ "pass@2": 0.4103107760856239,
425
+ "pass@4": 0.5020851449519407,
426
+ "pass@8": 0.577641649195555,
427
+ "pass@16": 0.640643393523646,
428
+ "pass@32": 0.6937003584791263,
429
+ "pass@64": 0.7400380698580312,
430
+ "pass@128": 0.7844311377245509
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 651496,
437
+ "answer_accuracy": 0.8131989934494328,
438
+ "avg_response_len": 304.374299758348,
439
+ "resp_tokens_sum": 243850079,
440
+ "pass_at_k": {
441
+ "pass@1": 0.8131989934494328,
442
+ "pass@2": 0.8516683416824594,
443
+ "pass@4": 0.8803690842367309,
444
+ "pass@8": 0.9015596188301824,
445
+ "pass@16": 0.9172829494592375,
446
+ "pass@32": 0.9293968706923023,
447
+ "pass@64": 0.9392539645433563,
448
+ "pass@128": 0.9475954625339511
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9517957089552239,
453
+ "pass@2": 0.9643014014572805,
454
+ "pass@4": 0.9723098293683435,
455
+ "pass@8": 0.9778178687535791,
456
+ "pass@16": 0.9818233351333638,
457
+ "pass@32": 0.9843700251647998,
458
+ "pass@64": 0.9862015157581675,
459
+ "pass@128": 0.9880597014925373
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9999553571428571,
463
+ "pass@2": 0.9999996484814399,
464
+ "pass@4": 1.0,
465
+ "pass@8": 1.0,
466
+ "pass@16": 1.0,
467
+ "pass@32": 1.0,
468
+ "pass@64": 1.0,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9998861151603499,
473
+ "pass@2": 0.99999784784555,
474
+ "pass@4": 0.9999999997267104,
475
+ "pass@8": 1.0,
476
+ "pass@16": 1.0,
477
+ "pass@32": 1.0,
478
+ "pass@64": 1.0,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9979669225146199,
483
+ "pass@2": 0.9986840666298292,
484
+ "pass@4": 0.9994179578978066,
485
+ "pass@8": 0.9998914828566827,
486
+ "pass@16": 0.999996971910131,
487
+ "pass@32": 0.9999999992108352,
488
+ "pass@64": 1.0,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9915670955882353,
493
+ "pass@2": 0.9952162459471977,
494
+ "pass@4": 0.996770337898939,
495
+ "pass@8": 0.9975063173915031,
496
+ "pass@16": 0.9980350056586529,
497
+ "pass@32": 0.9987689956690733,
498
+ "pass@64": 0.9996410375173692,
499
+ "pass@128": 1.0
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9964264589905363,
503
+ "pass@2": 0.9985430276459921,
504
+ "pass@4": 0.9995857823568582,
505
+ "pass@8": 0.9999535526784058,
506
+ "pass@16": 0.9999995379907698,
507
+ "pass@32": 0.9999999999901541,
508
+ "pass@64": 1.0,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9972804588607594,
513
+ "pass@2": 0.9992676511262834,
514
+ "pass@4": 0.9999203209764919,
515
+ "pass@8": 0.9999990565168133,
516
+ "pass@16": 0.9999999999228011,
517
+ "pass@32": 1.0,
518
+ "pass@64": 1.0,
519
+ "pass@128": 1.0
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9907563813813813,
523
+ "pass@2": 0.99560153164976,
524
+ "pass@4": 0.998587598565945,
525
+ "pass@8": 0.9998129486244137,
526
+ "pass@16": 0.999996824080759,
527
+ "pass@32": 0.9999999996818268,
528
+ "pass@64": 1.0,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9906300241157556,
533
+ "pass@2": 0.9950913993467858,
534
+ "pass@4": 0.9970629562582811,
535
+ "pass@8": 0.9986055619182425,
536
+ "pass@16": 0.99964486782007,
537
+ "pass@32": 0.9999770906276318,
538
+ "pass@64": 0.9999999831714547,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.9624007936507937,
543
+ "pass@2": 0.9748172103487065,
544
+ "pass@4": 0.981273297087864,
545
+ "pass@8": 0.9846499737188555,
546
+ "pass@16": 0.9862178333308695,
547
+ "pass@32": 0.9878187753676408,
548
+ "pass@64": 0.9897012873390828,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.9143969801223242,
553
+ "pass@2": 0.9317818572563751,
554
+ "pass@4": 0.941430099907236,
555
+ "pass@8": 0.9493256684380422,
556
+ "pass@16": 0.9570131901299108,
557
+ "pass@32": 0.9641514897698327,
558
+ "pass@64": 0.9703591098246457,
559
+ "pass@128": 0.9755351681957186
560
+ },
561
+ "13": {
562
+ "pass@1": 0.8329957561728395,
563
+ "pass@2": 0.8691643579274818,
564
+ "pass@4": 0.8912119784216789,
565
+ "pass@8": 0.9091026647516887,
566
+ "pass@16": 0.9252978369245227,
567
+ "pass@32": 0.9383556719133557,
568
+ "pass@64": 0.9473490414410928,
569
+ "pass@128": 0.9537037037037037
570
+ },
571
+ "14": {
572
+ "pass@1": 0.8191533753709199,
573
+ "pass@2": 0.8682212201219658,
574
+ "pass@4": 0.8979430763847394,
575
+ "pass@8": 0.9180236634394026,
576
+ "pass@16": 0.9314134024114261,
577
+ "pass@32": 0.9406145618745281,
578
+ "pass@64": 0.9479964540280973,
579
+ "pass@128": 0.9554896142433235
580
+ },
581
+ "15": {
582
+ "pass@1": 0.7535927854938271,
583
+ "pass@2": 0.8146192554923689,
584
+ "pass@4": 0.8533767113022905,
585
+ "pass@8": 0.878865995821382,
586
+ "pass@16": 0.8964124781429048,
587
+ "pass@32": 0.910541254887923,
588
+ "pass@64": 0.922806998472301,
589
+ "pass@128": 0.9320987654320988
590
+ },
591
+ "16": {
592
+ "pass@1": 0.6453032817109144,
593
+ "pass@2": 0.7468509017954611,
594
+ "pass@4": 0.8109872092868917,
595
+ "pass@8": 0.8503621602906531,
596
+ "pass@16": 0.8763865513376642,
597
+ "pass@32": 0.8958750335367089,
598
+ "pass@64": 0.9123815407896373,
599
+ "pass@128": 0.9262536873156342
600
+ },
601
+ "17": {
602
+ "pass@1": 0.5255553463855421,
603
+ "pass@2": 0.6341747402997818,
604
+ "pass@4": 0.7174640033700606,
605
+ "pass@8": 0.7752305050970983,
606
+ "pass@16": 0.8126617684952755,
607
+ "pass@32": 0.8371897052025431,
608
+ "pass@64": 0.8560389765187822,
609
+ "pass@128": 0.8734939759036144
610
+ },
611
+ "18": {
612
+ "pass@1": 0.4767389112903226,
613
+ "pass@2": 0.5847179006858011,
614
+ "pass@4": 0.66608676022352,
615
+ "pass@8": 0.7243191147240607,
616
+ "pass@16": 0.7663746277687996,
617
+ "pass@32": 0.8006317366757167,
618
+ "pass@64": 0.829606389878158,
619
+ "pass@128": 0.8516129032258064
620
+ },
621
+ "19": {
622
+ "pass@1": 0.30501793806646527,
623
+ "pass@2": 0.40979077955134763,
624
+ "pass@4": 0.5044869689627164,
625
+ "pass@8": 0.5830466910339542,
626
+ "pass@16": 0.6470709277754211,
627
+ "pass@32": 0.7023998688940181,
628
+ "pass@64": 0.7524830182120951,
629
+ "pass@128": 0.797583081570997
630
+ },
631
+ "20": {
632
+ "pass@1": 0.2891798048048048,
633
+ "pass@2": 0.3927472009410985,
634
+ "pass@4": 0.49233214992270086,
635
+ "pass@8": 0.5773029346421416,
636
+ "pass@16": 0.6449435379481746,
637
+ "pass@32": 0.6935367729692132,
638
+ "pass@64": 0.7276704804486359,
639
+ "pass@128": 0.7567567567567568
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9fea81e38268b0cf4fddf50b43439227beea5c926207e592d283cd10c6270c2
3
+ size 1790456324
results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch4-op17-20",
3
+ "id": {
4
+ "answer_accuracy": 0.9890277777777777,
5
+ "avg_response_len": 199.14139930555555,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9890277777777777,
11
+ "pass@2": 0.9928172162073456,
12
+ "pass@4": 0.9948839220097478,
13
+ "pass@8": 0.9963080761191235,
14
+ "pass@16": 0.9973152803179801,
15
+ "pass@32": 0.9980268388493161,
16
+ "pass@64": 0.99847883565242,
17
+ "pass@128": 0.9987777777777778
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9391875,
22
+ "pass@2": 0.9553868110236222,
23
+ "pass@4": 0.9664699572553431,
24
+ "pass@8": 0.9749212520185588,
25
+ "pass@16": 0.9816938886763031,
26
+ "pass@32": 0.9868280196969069,
27
+ "pass@64": 0.989992463600238,
28
+ "pass@128": 0.992
29
+ },
30
+ "2": {
31
+ "pass@1": 0.999953125,
32
+ "pass@2": 0.9999991387795276,
33
+ "pass@4": 0.9999999999062616,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 0.99996875,
42
+ "pass@2": 0.9999998769685039,
43
+ "pass@4": 1.0,
44
+ "pass@8": 1.0,
45
+ "pass@16": 1.0,
46
+ "pass@32": 1.0,
47
+ "pass@64": 1.0,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.99815625,
52
+ "pass@2": 0.9986737204724409,
53
+ "pass@4": 0.9989233598612672,
54
+ "pass@8": 0.9990519583151254,
55
+ "pass@16": 0.9991249279427927,
56
+ "pass@32": 0.9992499999993945,
57
+ "pass@64": 0.9995,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.992640625,
62
+ "pass@2": 0.9958212352362207,
63
+ "pass@4": 0.997663693007124,
64
+ "pass@8": 0.9989601583261803,
65
+ "pass@16": 0.9996727318798748,
66
+ "pass@32": 0.9999476997271504,
67
+ "pass@64": 0.9999993318674036,
68
+ "pass@128": 1.0
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9955546875,
72
+ "pass@2": 0.9968681102362207,
73
+ "pass@4": 0.9975768131796027,
74
+ "pass@8": 0.9981352577610507,
75
+ "pass@16": 0.9986862301248822,
76
+ "pass@32": 0.9992675062194463,
77
+ "pass@64": 0.9998183933036234,
78
+ "pass@128": 1.0
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9961640625,
82
+ "pass@2": 0.9984712106299214,
83
+ "pass@4": 0.9994211929133856,
84
+ "pass@8": 0.9998762765811555,
85
+ "pass@16": 0.9999913086119966,
86
+ "pass@32": 0.9999999700015125,
87
+ "pass@64": 0.9999999999999963,
88
+ "pass@128": 1.0
89
+ },
90
+ "8": {
91
+ "pass@1": 0.9908046875,
92
+ "pass@2": 0.995634965551181,
93
+ "pass@4": 0.9976798744844393,
94
+ "pass@8": 0.998685442665731,
95
+ "pass@16": 0.998977556450126,
96
+ "pass@32": 0.9989998945869978,
97
+ "pass@64": 0.9989999999999175,
98
+ "pass@128": 0.999
99
+ },
100
+ "9": {
101
+ "pass@1": 0.9888203125,
102
+ "pass@2": 0.9944998769685044,
103
+ "pass@4": 0.9962204074803149,
104
+ "pass@8": 0.9971423394043094,
105
+ "pass@16": 0.9976908791758443,
106
+ "pass@32": 0.997948459412436,
107
+ "pass@64": 0.9979993321006008,
108
+ "pass@128": 0.998
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9391875,
113
+ "2": 0.999953125,
114
+ "3": 0.99996875,
115
+ "4": 0.99815625,
116
+ "5": 0.992640625,
117
+ "6": 0.9955546875,
118
+ "7": 0.9961640625,
119
+ "8": 0.9908046875,
120
+ "9": 0.9888203125
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 278.9036640625,
124
+ "2": 115.269109375,
125
+ "3": 144.2801953125,
126
+ "4": 159.3698984375,
127
+ "5": 173.026875,
128
+ "6": 198.258796875,
129
+ "7": 223.091875,
130
+ "8": 242.26153125,
131
+ "9": 257.8106484375
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 278.9036640625,
136
+ "2": 115.269109375,
137
+ "3": 144.2801953125,
138
+ "4": 159.3698984375,
139
+ "5": 173.026875,
140
+ "6": 198.258796875,
141
+ "7": 223.091875,
142
+ "8": 242.26153125,
143
+ "9": 257.8106484375
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 375474,
150
+ "answer_accuracy": 0.9906756585612969,
151
+ "avg_response_len": 189.4462069402229,
152
+ "resp_tokens_sum": 71801628,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9906756585612969,
155
+ "pass@2": 0.994166370825988,
156
+ "pass@4": 0.9958945190108589,
157
+ "pass@8": 0.9970284643001964,
158
+ "pass@16": 0.997750665029614,
159
+ "pass@32": 0.9982530045544682,
160
+ "pass@64": 0.9987353591726881,
161
+ "pass@128": 0.9993245525160419
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9463956489675516,
166
+ "pass@2": 0.9599948029173347,
167
+ "pass@4": 0.9696439955514411,
168
+ "pass@8": 0.9772987780870966,
169
+ "pass@16": 0.9829576313879089,
170
+ "pass@32": 0.9869532680938505,
171
+ "pass@64": 0.9904289041602642,
172
+ "pass@128": 0.9941002949852508
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9999513239875389,
176
+ "pass@2": 0.9999996167243114,
177
+ "pass@4": 1.0,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 1.0,
186
+ "pass@2": 1.0,
187
+ "pass@4": 1.0,
188
+ "pass@8": 1.0,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.9962890625,
196
+ "pass@2": 0.9968965305118112,
197
+ "pass@4": 0.9969725551884139,
198
+ "pass@8": 0.9970703124996394,
199
+ "pass@16": 0.997265625,
200
+ "pass@32": 0.99765625,
201
+ "pass@64": 0.9984375,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9941220238095239,
206
+ "pass@2": 0.9976346706661666,
207
+ "pass@4": 0.9991279667422525,
208
+ "pass@8": 0.999810090374339,
209
+ "pass@16": 0.9999911414698637,
210
+ "pass@32": 0.9999999920410901,
211
+ "pass@64": 1.0,
212
+ "pass@128": 1.0
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9975673652694611,
216
+ "pass@2": 0.9990827891461174,
217
+ "pass@4": 0.9997700759897528,
218
+ "pass@8": 0.9999845197580833,
219
+ "pass@16": 0.9999999528630369,
220
+ "pass@32": 0.9999999999999486,
221
+ "pass@64": 1.0,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9960012943786982,
226
+ "pass@2": 0.9984763022410661,
227
+ "pass@4": 0.9996465231971745,
228
+ "pass@8": 0.9999815406455984,
229
+ "pass@16": 0.9999999678464966,
230
+ "pass@32": 0.9999999999999909,
231
+ "pass@64": 1.0,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9963441506410257,
236
+ "pass@2": 0.9996159209065214,
237
+ "pass@4": 0.9999852656639074,
238
+ "pass@8": 0.9999999592877328,
239
+ "pass@16": 0.9999999999999242,
240
+ "pass@32": 1.0,
241
+ "pass@64": 1.0,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9913292253521127,
246
+ "pass@2": 0.9970926444493733,
247
+ "pass@4": 0.9988132281880259,
248
+ "pass@8": 0.9997344814601936,
249
+ "pass@16": 0.9999856314389629,
250
+ "pass@32": 0.9999999749550461,
251
+ "pass@64": 0.9999999999999997,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 386021,
259
+ "answer_accuracy": 0.9881353415792923,
260
+ "avg_response_len": 197.20948609518348,
261
+ "resp_tokens_sum": 77041069,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9881353415792923,
264
+ "pass@2": 0.991507118089597,
265
+ "pass@4": 0.9934450351819555,
266
+ "pass@8": 0.9948278390202879,
267
+ "pass@16": 0.995978093148729,
268
+ "pass@32": 0.9968872544058474,
269
+ "pass@64": 0.9974407893814518,
270
+ "pass@128": 0.9977064220183486
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.924079754601227,
275
+ "pass@2": 0.9425480653108548,
276
+ "pass@4": 0.9558939233439372,
277
+ "pass@8": 0.9660136894297928,
278
+ "pass@16": 0.9745849213182081,
279
+ "pass@32": 0.9815053815983932,
280
+ "pass@64": 0.9856196459168264,
281
+ "pass@128": 0.9877300613496932
282
+ },
283
+ "2": {
284
+ "pass@1": 0.9999050151975684,
285
+ "pass@2": 0.9999977562645096,
286
+ "pass@4": 0.9999999997150811,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9999763257575758,
295
+ "pass@2": 1.0,
296
+ "pass@4": 1.0,
297
+ "pass@8": 1.0,
298
+ "pass@16": 1.0,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9998844304733728,
305
+ "pass@2": 0.9999978160089457,
306
+ "pass@4": 0.9999999997226678,
307
+ "pass@8": 1.0,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9920969202898551,
315
+ "pass@2": 0.9947952327969871,
316
+ "pass@4": 0.9963463162213417,
317
+ "pass@8": 0.9977460600531718,
318
+ "pass@16": 0.9990916148451592,
319
+ "pass@32": 0.9998484564379228,
320
+ "pass@64": 0.999998063383779,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9952542979942693,
325
+ "pass@2": 0.9966728617196489,
326
+ "pass@4": 0.9972907742728435,
327
+ "pass@8": 0.997639465969732,
328
+ "pass@16": 0.9980869583411133,
329
+ "pass@32": 0.9988007408816018,
330
+ "pass@64": 0.9996502944295288,
331
+ "pass@128": 1.0
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9967937138728323,
335
+ "pass@2": 0.9983024498202175,
336
+ "pass@4": 0.9990612040546953,
337
+ "pass@8": 0.9997078494159238,
338
+ "pass@16": 0.999975444312545,
339
+ "pass@32": 0.9999999133144741,
340
+ "pass@64": 0.9999999999999895,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9918573943661971,
345
+ "pass@2": 0.9956231978485083,
346
+ "pass@4": 0.9969242494336096,
347
+ "pass@8": 0.9971736180162805,
348
+ "pass@16": 0.9971830890857465,
349
+ "pass@32": 0.9971830985915483,
350
+ "pass@64": 0.9971830985915493,
351
+ "pass@128": 0.9971830985915493
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9910179640718563,
355
+ "pass@2": 0.9937379178650573,
356
+ "pass@4": 0.9939987547402381,
357
+ "pass@8": 0.9940119380249567,
358
+ "pass@16": 0.9940119760478334,
359
+ "pass@32": 0.9940119760479041,
360
+ "pass@64": 0.9940119760479041,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 377865,
368
+ "answer_accuracy": 0.9883060972547707,
369
+ "avg_response_len": 210.72615448192167,
370
+ "resp_tokens_sum": 80568195,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9883060972547707,
373
+ "pass@2": 0.9928184122668052,
374
+ "pass@4": 0.9953523200272051,
375
+ "pass@8": 0.9971064069599305,
376
+ "pass@16": 0.9982499710141318,
377
+ "pass@32": 0.9989670246774082,
378
+ "pass@64": 0.9992851801704917,
379
+ "pass@128": 0.9993304318714429
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9465951492537313,
384
+ "pass@2": 0.9632176225173348,
385
+ "pass@4": 0.9735499216515845,
386
+ "pass@8": 0.9811835925161799,
387
+ "pass@16": 0.9873330366748251,
388
+ "pass@32": 0.9918809176478053,
389
+ "pass@64": 0.993806150809024,
390
+ "pass@128": 0.9940298507462687
391
+ },
392
+ "2": {
393
+ "pass@1": 1.0,
394
+ "pass@2": 1.0,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9999316690962099,
404
+ "pass@2": 0.9999996413075917,
405
+ "pass@4": 1.0,
406
+ "pass@8": 1.0,
407
+ "pass@16": 1.0,
408
+ "pass@32": 1.0,
409
+ "pass@64": 1.0,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9981953581871345,
414
+ "pass@2": 0.9990279792328591,
415
+ "pass@4": 0.9996846265927285,
416
+ "pass@8": 0.9999691763603531,
417
+ "pass@16": 0.9999997893064115,
418
+ "pass@32": 0.9999999999982297,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9918198529411765,
424
+ "pass@2": 0.9951822313571099,
425
+ "pass@4": 0.9976438364322108,
426
+ "pass@8": 0.9994046739409391,
427
+ "pass@16": 0.9999673976331993,
428
+ "pass@32": 0.9999999551856581,
429
+ "pass@64": 0.9999999999999997,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.993764787066246,
434
+ "pass@2": 0.9947496212027123,
435
+ "pass@4": 0.9955808440940151,
436
+ "pass@8": 0.9967326641590364,
437
+ "pass@16": 0.9979618214750134,
438
+ "pass@32": 0.9990096140434843,
439
+ "pass@64": 0.9998121152924854,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9956487341772152,
444
+ "pass@2": 0.9986505469450812,
445
+ "pass@4": 0.9995743400943553,
446
+ "pass@8": 0.9999481010918784,
447
+ "pass@16": 0.9999994169232284,
448
+ "pass@32": 0.9999999999832521,
449
+ "pass@64": 1.0,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9844923048048048,
454
+ "pass@2": 0.9919176065829609,
455
+ "pass@4": 0.9963254145596664,
456
+ "pass@8": 0.9990655284449819,
457
+ "pass@16": 0.9999326120862155,
458
+ "pass@32": 0.9999996834444386,
459
+ "pass@64": 0.999999999999752,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9835962620578779,
464
+ "pass@2": 0.9923585968554577,
465
+ "pass@4": 0.9956467118660005,
466
+ "pass@8": 0.9975453735871543,
467
+ "pass@16": 0.9990224437782512,
468
+ "pass@32": 0.9998343032263491,
469
+ "pass@64": 0.9999978524135079,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.6918046875,
478
+ "avg_response_len": 372.07711328125,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.6918046875,
484
+ "pass@2": 0.7649430733267603,
485
+ "pass@4": 0.8173947985658003,
486
+ "pass@8": 0.8545082829889343,
487
+ "pass@16": 0.8807741388344542,
488
+ "pass@32": 0.9002820731595781,
489
+ "pass@64": 0.9154065233838675,
490
+ "pass@128": 0.9274
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.948625,
495
+ "pass@2": 0.9650340797244101,
496
+ "pass@4": 0.974049004686914,
497
+ "pass@8": 0.9799215853539738,
498
+ "pass@16": 0.9840942258978155,
499
+ "pass@32": 0.9873766462931601,
500
+ "pass@64": 0.9897657015488682,
501
+ "pass@128": 0.991
502
+ },
503
+ "12": {
504
+ "pass@1": 0.8919921875,
505
+ "pass@2": 0.9173727854330723,
506
+ "pass@4": 0.9306753659542553,
507
+ "pass@8": 0.9402165770631085,
508
+ "pass@16": 0.9484162200957359,
509
+ "pass@32": 0.9556793185220008,
510
+ "pass@64": 0.9622626730700722,
511
+ "pass@128": 0.969
512
+ },
513
+ "13": {
514
+ "pass@1": 0.8227109375,
515
+ "pass@2": 0.8682181348425212,
516
+ "pass@4": 0.8953016328271474,
517
+ "pass@8": 0.9137342715759691,
518
+ "pass@16": 0.9279086259612404,
519
+ "pass@32": 0.9409428563904914,
520
+ "pass@64": 0.9531583365904523,
521
+ "pass@128": 0.963
522
+ },
523
+ "14": {
524
+ "pass@1": 0.8139765625,
525
+ "pass@2": 0.8710821850393708,
526
+ "pass@4": 0.9045889305399336,
527
+ "pass@8": 0.9250908670684431,
528
+ "pass@16": 0.9386299476507689,
529
+ "pass@32": 0.9487427582732499,
530
+ "pass@64": 0.957440936100577,
531
+ "pass@128": 0.965
532
+ },
533
+ "15": {
534
+ "pass@1": 0.7839609375,
535
+ "pass@2": 0.8494299950787403,
536
+ "pass@4": 0.8907248334270719,
537
+ "pass@8": 0.9178499015041497,
538
+ "pass@16": 0.9354141739723325,
539
+ "pass@32": 0.9468501950584634,
540
+ "pass@64": 0.9547623055339345,
541
+ "pass@128": 0.962
542
+ },
543
+ "16": {
544
+ "pass@1": 0.66109375,
545
+ "pass@2": 0.7585896899606303,
546
+ "pass@4": 0.8241256755718038,
547
+ "pass@8": 0.8659143995718789,
548
+ "pass@16": 0.8935886268502504,
549
+ "pass@32": 0.9145088226574314,
550
+ "pass@64": 0.9303470680153623,
551
+ "pass@128": 0.941
552
+ },
553
+ "17": {
554
+ "pass@1": 0.605640625,
555
+ "pass@2": 0.7034212598425199,
556
+ "pass@4": 0.7721566392950879,
557
+ "pass@8": 0.8196972052982548,
558
+ "pass@16": 0.8530516738610301,
559
+ "pass@32": 0.876275545986001,
560
+ "pass@64": 0.8930550155935169,
561
+ "pass@128": 0.906
562
+ },
563
+ "18": {
564
+ "pass@1": 0.56478125,
565
+ "pass@2": 0.6675775098425195,
566
+ "pass@4": 0.7450145286839147,
567
+ "pass@8": 0.7996635802319876,
568
+ "pass@16": 0.8369695037285299,
569
+ "pass@32": 0.8625344989141525,
570
+ "pass@64": 0.8809200646871671,
571
+ "pass@128": 0.896
572
+ },
573
+ "19": {
574
+ "pass@1": 0.413765625,
575
+ "pass@2": 0.5293097933070865,
576
+ "pass@4": 0.6272639149793771,
577
+ "pass@8": 0.7019809692420499,
578
+ "pass@16": 0.7557623698652861,
579
+ "pass@32": 0.7956271976826763,
580
+ "pass@64": 0.8254726377576067,
581
+ "pass@128": 0.848
582
+ },
583
+ "20": {
584
+ "pass@1": 0.4115,
585
+ "pass@2": 0.5193953001968503,
586
+ "pass@4": 0.6100474596925382,
587
+ "pass@8": 0.6810134729795795,
588
+ "pass@16": 0.7339060204614524,
589
+ "pass@32": 0.7742828918181295,
590
+ "pass@64": 0.8068804949410628,
591
+ "pass@128": 0.833
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.948625,
596
+ "12": 0.8919921875,
597
+ "13": 0.8227109375,
598
+ "14": 0.8139765625,
599
+ "15": 0.7839609375,
600
+ "16": 0.66109375,
601
+ "17": 0.605640625,
602
+ "18": 0.56478125,
603
+ "19": 0.413765625,
604
+ "20": 0.4115
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 296.949046875,
608
+ "12": 306.7407578125,
609
+ "13": 325.3369921875,
610
+ "14": 353.14478125,
611
+ "15": 373.4759765625,
612
+ "16": 378.2594296875,
613
+ "17": 402.084828125,
614
+ "18": 419.432109375,
615
+ "19": 421.6563125,
616
+ "20": 443.6908984375
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 296.949046875,
621
+ "12": 306.7407578125,
622
+ "13": 325.3369921875,
623
+ "14": 353.14478125,
624
+ "15": 373.4759765625,
625
+ "16": 378.2594296875,
626
+ "17": 402.084828125,
627
+ "18": 419.432109375,
628
+ "19": 421.6563125,
629
+ "20": 443.6908984375
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 301159,
636
+ "answer_accuracy": 0.6991990156017831,
637
+ "avg_response_len": 353.8343819650817,
638
+ "resp_tokens_sum": 152403545,
639
+ "pass_at_k": {
640
+ "pass@1": 0.6991990156017831,
641
+ "pass@2": 0.7698041221583997,
642
+ "pass@4": 0.8194136314684324,
643
+ "pass@8": 0.8540800124080026,
644
+ "pass@16": 0.8791774558830416,
645
+ "pass@32": 0.8984448726143027,
646
+ "pass@64": 0.9136238088950139,
647
+ "pass@128": 0.9260029717682021
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9385844546742209,
652
+ "pass@2": 0.9559055815172538,
653
+ "pass@4": 0.9662483132420059,
654
+ "pass@8": 0.9743145176794873,
655
+ "pass@16": 0.9814528957415205,
656
+ "pass@32": 0.9870929428344121,
657
+ "pass@64": 0.9904799574420998,
658
+ "pass@128": 0.9915014164305949
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8917284604519774,
662
+ "pass@2": 0.9156980542951199,
663
+ "pass@4": 0.9281065293215466,
664
+ "pass@8": 0.935889004485147,
665
+ "pass@16": 0.9418117371185404,
666
+ "pass@32": 0.947600022930526,
667
+ "pass@64": 0.9542037056880797,
668
+ "pass@128": 0.963276836158192
669
+ },
670
+ "13": {
671
+ "pass@1": 0.8402233542319749,
672
+ "pass@2": 0.8852274269987415,
673
+ "pass@4": 0.9120427055748123,
674
+ "pass@8": 0.9300158215455117,
675
+ "pass@16": 0.9440746211517979,
676
+ "pass@32": 0.9571960077579881,
677
+ "pass@64": 0.9683369401551648,
678
+ "pass@128": 0.9749216300940439
679
+ },
680
+ "14": {
681
+ "pass@1": 0.8298773364485982,
682
+ "pass@2": 0.8854661092305048,
683
+ "pass@4": 0.9165421769708681,
684
+ "pass@8": 0.9342494936091128,
685
+ "pass@16": 0.9469401578089754,
686
+ "pass@32": 0.9577873713575069,
687
+ "pass@64": 0.9667747235355996,
688
+ "pass@128": 0.9750778816199377
689
+ },
690
+ "15": {
691
+ "pass@1": 0.7909365968208093,
692
+ "pass@2": 0.8548815057348322,
693
+ "pass@4": 0.8965234123552469,
694
+ "pass@8": 0.9232631518428708,
695
+ "pass@16": 0.93956332213618,
696
+ "pass@32": 0.9494509870773286,
697
+ "pass@64": 0.9560635736921728,
698
+ "pass@128": 0.9624277456647399
699
+ },
700
+ "16": {
701
+ "pass@1": 0.6512096774193549,
702
+ "pass@2": 0.7479818865887731,
703
+ "pass@4": 0.8152397825271837,
704
+ "pass@8": 0.8589674413513186,
705
+ "pass@16": 0.8885187025219686,
706
+ "pass@32": 0.9122664817583825,
707
+ "pass@64": 0.9310470742858282,
708
+ "pass@128": 0.9419354838709677
709
+ },
710
+ "17": {
711
+ "pass@1": 0.6124909682080925,
712
+ "pass@2": 0.7055230474261528,
713
+ "pass@4": 0.768419161297034,
714
+ "pass@8": 0.8132989938840093,
715
+ "pass@16": 0.8486630253769624,
716
+ "pass@32": 0.8756942867659188,
717
+ "pass@64": 0.8953656362346946,
718
+ "pass@128": 0.9075144508670521
719
+ },
720
+ "18": {
721
+ "pass@1": 0.5464732142857143,
722
+ "pass@2": 0.6480627812148483,
723
+ "pass@4": 0.7236771966896989,
724
+ "pass@8": 0.777945603229985,
725
+ "pass@16": 0.8163166798894307,
726
+ "pass@32": 0.8431802277020857,
727
+ "pass@64": 0.8622731277447071,
728
+ "pass@128": 0.8771428571428571
729
+ },
730
+ "19": {
731
+ "pass@1": 0.4315643768768769,
732
+ "pass@2": 0.5412484334728428,
733
+ "pass@4": 0.6303853890448375,
734
+ "pass@8": 0.6978895463021549,
735
+ "pass@16": 0.7471806522306299,
736
+ "pass@32": 0.7842191914351475,
737
+ "pass@64": 0.8144645683656517,
738
+ "pass@128": 0.8438438438438438
739
+ },
740
+ "20": {
741
+ "pass@1": 0.4473066816816817,
742
+ "pass@2": 0.550585157007401,
743
+ "pass@4": 0.6322523291007539,
744
+ "pass@8": 0.6917831399797562,
745
+ "pass@16": 0.7350186424328486,
746
+ "pass@32": 0.7688185050754536,
747
+ "pass@64": 0.7971501932040521,
748
+ "pass@128": 0.8228228228228228
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 300394,
755
+ "answer_accuracy": 0.6978376821290514,
756
+ "avg_response_len": 367.61922715953017,
757
+ "resp_tokens_sum": 158246843,
758
+ "pass_at_k": {
759
+ "pass@1": 0.6978376821290514,
760
+ "pass@2": 0.7712399262703721,
761
+ "pass@4": 0.8234056467183366,
762
+ "pass@8": 0.8598974993623416,
763
+ "pass@16": 0.8856925425332236,
764
+ "pass@32": 0.9053746000361343,
765
+ "pass@64": 0.9216224010043141,
766
+ "pass@128": 0.9351769253642581
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9533838478915663,
771
+ "pass@2": 0.9686280802343229,
772
+ "pass@4": 0.9774122846842942,
773
+ "pass@8": 0.9826945678160793,
774
+ "pass@16": 0.9851505016991625,
775
+ "pass@32": 0.9862401453810479,
776
+ "pass@64": 0.9872047186881181,
777
+ "pass@128": 0.9879518072289156
778
+ },
779
+ "12": {
780
+ "pass@1": 0.885384012539185,
781
+ "pass@2": 0.9101669525831221,
782
+ "pass@4": 0.9239799511738148,
783
+ "pass@8": 0.9333532233276649,
784
+ "pass@16": 0.9404962983721141,
785
+ "pass@32": 0.947095869663446,
786
+ "pass@64": 0.9545776664683873,
787
+ "pass@128": 0.9623824451410659
788
+ },
789
+ "13": {
790
+ "pass@1": 0.8328956582633054,
791
+ "pass@2": 0.8768782119146881,
792
+ "pass@4": 0.9001525788793208,
793
+ "pass@8": 0.9144132901889223,
794
+ "pass@16": 0.9252804791130302,
795
+ "pass@32": 0.9358350221164605,
796
+ "pass@64": 0.9474943865376289,
797
+ "pass@128": 0.9607843137254902
798
+ },
799
+ "14": {
800
+ "pass@1": 0.8158351608187134,
801
+ "pass@2": 0.8764760182115391,
802
+ "pass@4": 0.9120771337245117,
803
+ "pass@8": 0.9326000038402523,
804
+ "pass@16": 0.9449230559731131,
805
+ "pass@32": 0.9539728291687525,
806
+ "pass@64": 0.9628130945239121,
807
+ "pass@128": 0.9707602339181286
808
+ },
809
+ "15": {
810
+ "pass@1": 0.7901515151515152,
811
+ "pass@2": 0.8613483506323071,
812
+ "pass@4": 0.9039200494824501,
813
+ "pass@8": 0.9292158194336342,
814
+ "pass@16": 0.9452248133475442,
815
+ "pass@32": 0.9574867632372372,
816
+ "pass@64": 0.9678628579175225,
817
+ "pass@128": 0.9787878787878788
818
+ },
819
+ "16": {
820
+ "pass@1": 0.6649528133903134,
821
+ "pass@2": 0.7624254800681963,
822
+ "pass@4": 0.8277979675617472,
823
+ "pass@8": 0.8686706781122179,
824
+ "pass@16": 0.8961598925297205,
825
+ "pass@32": 0.9179229188572494,
826
+ "pass@64": 0.9337860632473033,
827
+ "pass@128": 0.9430199430199431
828
+ },
829
+ "17": {
830
+ "pass@1": 0.6393390916149069,
831
+ "pass@2": 0.7354016634469608,
832
+ "pass@4": 0.8004865514675568,
833
+ "pass@8": 0.8453401336925654,
834
+ "pass@16": 0.8772971858982275,
835
+ "pass@32": 0.9004220002674332,
836
+ "pass@64": 0.9174945886997983,
837
+ "pass@128": 0.9316770186335404
838
+ },
839
+ "18": {
840
+ "pass@1": 0.5892693014705882,
841
+ "pass@2": 0.6895900880037054,
842
+ "pass@4": 0.7655650637420315,
843
+ "pass@8": 0.8203016454124229,
844
+ "pass@16": 0.859618198955504,
845
+ "pass@32": 0.8878937646484227,
846
+ "pass@64": 0.9097340634600377,
847
+ "pass@128": 0.9294117647058824
848
+ },
849
+ "19": {
850
+ "pass@1": 0.4204799107142857,
851
+ "pass@2": 0.5389387361736032,
852
+ "pass@4": 0.6410350558858714,
853
+ "pass@8": 0.7190375968217035,
854
+ "pass@16": 0.7736645475875377,
855
+ "pass@32": 0.8123192037757934,
856
+ "pass@64": 0.8394299819521862,
857
+ "pass@128": 0.8571428571428571
858
+ },
859
+ "20": {
860
+ "pass@1": 0.3888005239520958,
861
+ "pass@2": 0.4932697351360271,
862
+ "pass@4": 0.581708334175294,
863
+ "pass@8": 0.6533021743606491,
864
+ "pass@16": 0.7090402348266315,
865
+ "pass@32": 0.754332532451074,
866
+ "pass@64": 0.7954232145576876,
867
+ "pass@128": 0.8293413173652695
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 283957,
874
+ "answer_accuracy": 0.6779994078545232,
875
+ "avg_response_len": 395.4202251107885,
876
+ "resp_tokens_sum": 165608317,
877
+ "pass_at_k": {
878
+ "pass@1": 0.6779994078545232,
879
+ "pass@2": 0.7534718796324861,
880
+ "pass@4": 0.8091405641360095,
881
+ "pass@8": 0.8494096264611646,
882
+ "pass@16": 0.8773610112343615,
883
+ "pass@32": 0.8969373335351778,
884
+ "pass@64": 0.9108511559686301,
885
+ "pass@128": 0.9208435207823961
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.9548611111111112,
890
+ "pass@2": 0.9714758311461071,
891
+ "pass@4": 0.9792459415787311,
892
+ "pass@8": 0.9832824257084963,
893
+ "pass@16": 0.9859409115458316,
894
+ "pass@32": 0.9888924101781106,
895
+ "pass@64": 0.9916644760868305,
896
+ "pass@128": 0.9936507936507937
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8987241972477065,
900
+ "pass@2": 0.9262153404849626,
901
+ "pass@4": 0.9399879209479548,
902
+ "pass@8": 0.9515969150881995,
903
+ "pass@16": 0.9632921895262051,
904
+ "pass@32": 0.9727991681405372,
905
+ "pass@64": 0.978484053984944,
906
+ "pass@128": 0.981651376146789
907
+ },
908
+ "13": {
909
+ "pass@1": 0.7942467206790124,
910
+ "pass@2": 0.8419292098036356,
911
+ "pass@4": 0.87347388607674,
912
+ "pass@8": 0.896955802177486,
913
+ "pass@16": 0.9148879344767462,
914
+ "pass@32": 0.9305686019139384,
915
+ "pass@64": 0.9444548045895704,
916
+ "pass@128": 0.9537037037037037
917
+ },
918
+ "14": {
919
+ "pass@1": 0.7969445474777448,
920
+ "pass@2": 0.8519073167597371,
921
+ "pass@4": 0.8856038931706228,
922
+ "pass@8": 0.9087465231648425,
923
+ "pass@16": 0.9243278096477272,
924
+ "pass@32": 0.9348198946343538,
925
+ "pass@64": 0.9430984318055536,
926
+ "pass@128": 0.9495548961424333
927
+ },
928
+ "15": {
929
+ "pass@1": 0.7702064043209876,
930
+ "pass@2": 0.8314692542772435,
931
+ "pass@4": 0.8710929519226761,
932
+ "pass@8": 0.9004926868932659,
933
+ "pass@16": 0.9209909756435962,
934
+ "pass@32": 0.9332392643870973,
935
+ "pass@64": 0.9400295244557415,
936
+ "pass@128": 0.9444444444444444
937
+ },
938
+ "16": {
939
+ "pass@1": 0.6661366150442478,
940
+ "pass@2": 0.7643184708382698,
941
+ "pass@4": 0.8284491338472064,
942
+ "pass@8": 0.8694132293085024,
943
+ "pass@16": 0.8955625569041571,
944
+ "pass@32": 0.9130243917210557,
945
+ "pass@64": 0.926146214710774,
946
+ "pass@128": 0.9380530973451328
947
+ },
948
+ "17": {
949
+ "pass@1": 0.5658179593373494,
950
+ "pass@2": 0.6702137042263541,
951
+ "pass@4": 0.7485751202221707,
952
+ "pass@8": 0.8014946697752476,
953
+ "pass@16": 0.8341101603053381,
954
+ "pass@32": 0.8534621646954216,
955
+ "pass@64": 0.8669435177559548,
956
+ "pass@128": 0.8795180722891566
957
+ },
958
+ "18": {
959
+ "pass@1": 0.55859375,
960
+ "pass@2": 0.6654674403098803,
961
+ "pass@4": 0.7465657682910599,
962
+ "pass@8": 0.801548579552477,
963
+ "pass@16": 0.8354467036205091,
964
+ "pass@32": 0.8565727072192222,
965
+ "pass@64": 0.8703706077422788,
966
+ "pass@128": 0.8806451612903226
967
+ },
968
+ "19": {
969
+ "pass@1": 0.3890436178247734,
970
+ "pass@2": 0.5075246211670672,
971
+ "pass@4": 0.6101444158604039,
972
+ "pass@8": 0.6887828332064626,
973
+ "pass@16": 0.7462233374111003,
974
+ "pass@32": 0.7901599832812849,
975
+ "pass@64": 0.8223790409544078,
976
+ "pass@128": 0.8429003021148036
977
+ },
978
+ "20": {
979
+ "pass@1": 0.39846096096096095,
980
+ "pass@2": 0.5144094635974552,
981
+ "pass@4": 0.6162668182805975,
982
+ "pass@8": 0.6980383217113035,
983
+ "pass@16": 0.7577338561538128,
984
+ "pass@32": 0.799757548916951,
985
+ "pass@64": 0.828102483308844,
986
+ "pass@128": 0.8468468468468469
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.832594572368421,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.832594572368421,
1002
+ "pass@2": 0.8728834567964937,
1003
+ "pass@4": 0.9014685938813386,
1004
+ "pass@8": 0.9216766060506145,
1005
+ "pass@16": 0.9359778374318961,
1006
+ "pass@32": 0.9465822253284079,
1007
+ "pass@64": 0.954756566037391,
1008
+ "pass@128": 0.9612105263157895
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9391875,
1013
+ "pass@2": 0.9553868110236222,
1014
+ "pass@4": 0.9664699572553431,
1015
+ "pass@8": 0.9749212520185588,
1016
+ "pass@16": 0.9816938886763031,
1017
+ "pass@32": 0.9868280196969069,
1018
+ "pass@64": 0.989992463600238,
1019
+ "pass@128": 0.992
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.999953125,
1023
+ "pass@2": 0.9999991387795276,
1024
+ "pass@4": 0.9999999999062616,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.99996875,
1033
+ "pass@2": 0.9999998769685039,
1034
+ "pass@4": 1.0,
1035
+ "pass@8": 1.0,
1036
+ "pass@16": 1.0,
1037
+ "pass@32": 1.0,
1038
+ "pass@64": 1.0,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.99815625,
1043
+ "pass@2": 0.9986737204724409,
1044
+ "pass@4": 0.9989233598612672,
1045
+ "pass@8": 0.9990519583151254,
1046
+ "pass@16": 0.9991249279427927,
1047
+ "pass@32": 0.9992499999993945,
1048
+ "pass@64": 0.9995,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.992640625,
1053
+ "pass@2": 0.9958212352362207,
1054
+ "pass@4": 0.997663693007124,
1055
+ "pass@8": 0.9989601583261803,
1056
+ "pass@16": 0.9996727318798748,
1057
+ "pass@32": 0.9999476997271504,
1058
+ "pass@64": 0.9999993318674036,
1059
+ "pass@128": 1.0
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9955546875,
1063
+ "pass@2": 0.9968681102362207,
1064
+ "pass@4": 0.9975768131796027,
1065
+ "pass@8": 0.9981352577610507,
1066
+ "pass@16": 0.9986862301248822,
1067
+ "pass@32": 0.9992675062194463,
1068
+ "pass@64": 0.9998183933036234,
1069
+ "pass@128": 1.0
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9961640625,
1073
+ "pass@2": 0.9984712106299214,
1074
+ "pass@4": 0.9994211929133856,
1075
+ "pass@8": 0.9998762765811555,
1076
+ "pass@16": 0.9999913086119966,
1077
+ "pass@32": 0.9999999700015125,
1078
+ "pass@64": 0.9999999999999963,
1079
+ "pass@128": 1.0
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.9908046875,
1083
+ "pass@2": 0.995634965551181,
1084
+ "pass@4": 0.9976798744844393,
1085
+ "pass@8": 0.998685442665731,
1086
+ "pass@16": 0.998977556450126,
1087
+ "pass@32": 0.9989998945869978,
1088
+ "pass@64": 0.9989999999999175,
1089
+ "pass@128": 0.999
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.9888203125,
1093
+ "pass@2": 0.9944998769685044,
1094
+ "pass@4": 0.9962204074803149,
1095
+ "pass@8": 0.9971423394043094,
1096
+ "pass@16": 0.9976908791758443,
1097
+ "pass@32": 0.997948459412436,
1098
+ "pass@64": 0.9979993321006008,
1099
+ "pass@128": 0.998
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.948625,
1103
+ "pass@2": 0.9650340797244101,
1104
+ "pass@4": 0.974049004686914,
1105
+ "pass@8": 0.9799215853539738,
1106
+ "pass@16": 0.9840942258978155,
1107
+ "pass@32": 0.9873766462931601,
1108
+ "pass@64": 0.9897657015488682,
1109
+ "pass@128": 0.991
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.8919921875,
1113
+ "pass@2": 0.9173727854330723,
1114
+ "pass@4": 0.9306753659542553,
1115
+ "pass@8": 0.9402165770631085,
1116
+ "pass@16": 0.9484162200957359,
1117
+ "pass@32": 0.9556793185220008,
1118
+ "pass@64": 0.9622626730700722,
1119
+ "pass@128": 0.969
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.8227109375,
1123
+ "pass@2": 0.8682181348425212,
1124
+ "pass@4": 0.8953016328271474,
1125
+ "pass@8": 0.9137342715759691,
1126
+ "pass@16": 0.9279086259612404,
1127
+ "pass@32": 0.9409428563904914,
1128
+ "pass@64": 0.9531583365904523,
1129
+ "pass@128": 0.963
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.8139765625,
1133
+ "pass@2": 0.8710821850393708,
1134
+ "pass@4": 0.9045889305399336,
1135
+ "pass@8": 0.9250908670684431,
1136
+ "pass@16": 0.9386299476507689,
1137
+ "pass@32": 0.9487427582732499,
1138
+ "pass@64": 0.957440936100577,
1139
+ "pass@128": 0.965
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.7839609375,
1143
+ "pass@2": 0.8494299950787403,
1144
+ "pass@4": 0.8907248334270719,
1145
+ "pass@8": 0.9178499015041497,
1146
+ "pass@16": 0.9354141739723325,
1147
+ "pass@32": 0.9468501950584634,
1148
+ "pass@64": 0.9547623055339345,
1149
+ "pass@128": 0.962
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.66109375,
1153
+ "pass@2": 0.7585896899606303,
1154
+ "pass@4": 0.8241256755718038,
1155
+ "pass@8": 0.8659143995718789,
1156
+ "pass@16": 0.8935886268502504,
1157
+ "pass@32": 0.9145088226574314,
1158
+ "pass@64": 0.9303470680153623,
1159
+ "pass@128": 0.941
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.605640625,
1163
+ "pass@2": 0.7034212598425199,
1164
+ "pass@4": 0.7721566392950879,
1165
+ "pass@8": 0.8196972052982548,
1166
+ "pass@16": 0.8530516738610301,
1167
+ "pass@32": 0.876275545986001,
1168
+ "pass@64": 0.8930550155935169,
1169
+ "pass@128": 0.906
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.56478125,
1173
+ "pass@2": 0.6675775098425195,
1174
+ "pass@4": 0.7450145286839147,
1175
+ "pass@8": 0.7996635802319876,
1176
+ "pass@16": 0.8369695037285299,
1177
+ "pass@32": 0.8625344989141525,
1178
+ "pass@64": 0.8809200646871671,
1179
+ "pass@128": 0.896
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.413765625,
1183
+ "pass@2": 0.5293097933070865,
1184
+ "pass@4": 0.6272639149793771,
1185
+ "pass@8": 0.7019809692420499,
1186
+ "pass@16": 0.7557623698652861,
1187
+ "pass@32": 0.7956271976826763,
1188
+ "pass@64": 0.8254726377576067,
1189
+ "pass@128": 0.848
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.4115,
1193
+ "pass@2": 0.5193953001968503,
1194
+ "pass@4": 0.6100474596925382,
1195
+ "pass@8": 0.6810134729795795,
1196
+ "pass@16": 0.7339060204614524,
1197
+ "pass@32": 0.7742828918181295,
1198
+ "pass@64": 0.8068804949410628,
1199
+ "pass@128": 0.833
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9391875,
1204
+ "2": 0.999953125,
1205
+ "3": 0.99996875,
1206
+ "4": 0.99815625,
1207
+ "5": 0.992640625,
1208
+ "6": 0.9955546875,
1209
+ "7": 0.9961640625,
1210
+ "8": 0.9908046875,
1211
+ "9": 0.9888203125,
1212
+ "11": 0.948625,
1213
+ "12": 0.8919921875,
1214
+ "13": 0.8227109375,
1215
+ "14": 0.8139765625,
1216
+ "15": 0.7839609375,
1217
+ "16": 0.66109375,
1218
+ "17": 0.605640625,
1219
+ "18": 0.56478125,
1220
+ "19": 0.413765625,
1221
+ "20": 0.4115
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 278.9036640625,
1225
+ "2": 115.269109375,
1226
+ "3": 144.2801953125,
1227
+ "4": 159.3698984375,
1228
+ "5": 173.026875,
1229
+ "6": 198.258796875,
1230
+ "7": 223.091875,
1231
+ "8": 242.26153125,
1232
+ "9": 257.8106484375,
1233
+ "11": 296.949046875,
1234
+ "12": 306.7407578125,
1235
+ "13": 325.3369921875,
1236
+ "14": 353.14478125,
1237
+ "15": 373.4759765625,
1238
+ "16": 378.2594296875,
1239
+ "17": 402.084828125,
1240
+ "18": 419.432109375,
1241
+ "19": 421.6563125,
1242
+ "20": 443.6908984375
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 278.9036640625,
1247
+ "2": 115.269109375,
1248
+ "3": 144.2801953125,
1249
+ "4": 159.3698984375,
1250
+ "5": 173.026875,
1251
+ "6": 198.258796875,
1252
+ "7": 223.091875,
1253
+ "8": 242.26153125,
1254
+ "9": 257.8106484375,
1255
+ "11": 296.949046875,
1256
+ "12": 306.7407578125,
1257
+ "13": 325.3369921875,
1258
+ "14": 353.14478125,
1259
+ "15": 373.4759765625,
1260
+ "16": 378.2594296875,
1261
+ "17": 402.084828125,
1262
+ "18": 419.432109375,
1263
+ "19": 421.6563125,
1264
+ "20": 443.6908984375
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 676633,
1271
+ "answer_accuracy": 0.8356299893297502,
1272
+ "avg_response_len": 276.8894900509801,
1273
+ "resp_tokens_sum": 224205173,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.8356299893297502,
1276
+ "pass@2": 0.8748209761426936,
1277
+ "pass@4": 0.9020187386472343,
1278
+ "pass@8": 0.9209896497859279,
1279
+ "pass@16": 0.9346778150804824,
1280
+ "pass@32": 0.9451618942195561,
1281
+ "pass@64": 0.9534618266585658,
1282
+ "pass@128": 0.9603224786595005
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9463956489675516,
1287
+ "pass@2": 0.9599948029173347,
1288
+ "pass@4": 0.9696439955514411,
1289
+ "pass@8": 0.9772987780870966,
1290
+ "pass@16": 0.9829576313879089,
1291
+ "pass@32": 0.9869532680938505,
1292
+ "pass@64": 0.9904289041602642,
1293
+ "pass@128": 0.9941002949852508
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9999513239875389,
1297
+ "pass@2": 0.9999996167243114,
1298
+ "pass@4": 1.0,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 1.0,
1307
+ "pass@2": 1.0,
1308
+ "pass@4": 1.0,
1309
+ "pass@8": 1.0,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.9962890625,
1317
+ "pass@2": 0.9968965305118112,
1318
+ "pass@4": 0.9969725551884139,
1319
+ "pass@8": 0.9970703124996394,
1320
+ "pass@16": 0.997265625,
1321
+ "pass@32": 0.99765625,
1322
+ "pass@64": 0.9984375,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9941220238095239,
1327
+ "pass@2": 0.9976346706661666,
1328
+ "pass@4": 0.9991279667422525,
1329
+ "pass@8": 0.999810090374339,
1330
+ "pass@16": 0.9999911414698637,
1331
+ "pass@32": 0.9999999920410901,
1332
+ "pass@64": 1.0,
1333
+ "pass@128": 1.0
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9975673652694611,
1337
+ "pass@2": 0.9990827891461174,
1338
+ "pass@4": 0.9997700759897528,
1339
+ "pass@8": 0.9999845197580833,
1340
+ "pass@16": 0.9999999528630369,
1341
+ "pass@32": 0.9999999999999486,
1342
+ "pass@64": 1.0,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9960012943786982,
1347
+ "pass@2": 0.9984763022410661,
1348
+ "pass@4": 0.9996465231971745,
1349
+ "pass@8": 0.9999815406455984,
1350
+ "pass@16": 0.9999999678464966,
1351
+ "pass@32": 0.9999999999999909,
1352
+ "pass@64": 1.0,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9963441506410257,
1357
+ "pass@2": 0.9996159209065214,
1358
+ "pass@4": 0.9999852656639074,
1359
+ "pass@8": 0.9999999592877328,
1360
+ "pass@16": 0.9999999999999242,
1361
+ "pass@32": 1.0,
1362
+ "pass@64": 1.0,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9913292253521127,
1367
+ "pass@2": 0.9970926444493733,
1368
+ "pass@4": 0.9988132281880259,
1369
+ "pass@8": 0.9997344814601936,
1370
+ "pass@16": 0.9999856314389629,
1371
+ "pass@32": 0.9999999749550461,
1372
+ "pass@64": 0.9999999999999997,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9385844546742209,
1377
+ "pass@2": 0.9559055815172538,
1378
+ "pass@4": 0.9662483132420059,
1379
+ "pass@8": 0.9743145176794873,
1380
+ "pass@16": 0.9814528957415205,
1381
+ "pass@32": 0.9870929428344121,
1382
+ "pass@64": 0.9904799574420998,
1383
+ "pass@128": 0.9915014164305949
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8917284604519774,
1387
+ "pass@2": 0.9156980542951199,
1388
+ "pass@4": 0.9281065293215466,
1389
+ "pass@8": 0.935889004485147,
1390
+ "pass@16": 0.9418117371185404,
1391
+ "pass@32": 0.947600022930526,
1392
+ "pass@64": 0.9542037056880797,
1393
+ "pass@128": 0.963276836158192
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.8402233542319749,
1397
+ "pass@2": 0.8852274269987415,
1398
+ "pass@4": 0.9120427055748123,
1399
+ "pass@8": 0.9300158215455117,
1400
+ "pass@16": 0.9440746211517979,
1401
+ "pass@32": 0.9571960077579881,
1402
+ "pass@64": 0.9683369401551648,
1403
+ "pass@128": 0.9749216300940439
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.8298773364485982,
1407
+ "pass@2": 0.8854661092305048,
1408
+ "pass@4": 0.9165421769708681,
1409
+ "pass@8": 0.9342494936091128,
1410
+ "pass@16": 0.9469401578089754,
1411
+ "pass@32": 0.9577873713575069,
1412
+ "pass@64": 0.9667747235355996,
1413
+ "pass@128": 0.9750778816199377
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.7909365968208093,
1417
+ "pass@2": 0.8548815057348322,
1418
+ "pass@4": 0.8965234123552469,
1419
+ "pass@8": 0.9232631518428708,
1420
+ "pass@16": 0.93956332213618,
1421
+ "pass@32": 0.9494509870773286,
1422
+ "pass@64": 0.9560635736921728,
1423
+ "pass@128": 0.9624277456647399
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.6512096774193549,
1427
+ "pass@2": 0.7479818865887731,
1428
+ "pass@4": 0.8152397825271837,
1429
+ "pass@8": 0.8589674413513186,
1430
+ "pass@16": 0.8885187025219686,
1431
+ "pass@32": 0.9122664817583825,
1432
+ "pass@64": 0.9310470742858282,
1433
+ "pass@128": 0.9419354838709677
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.6124909682080925,
1437
+ "pass@2": 0.7055230474261528,
1438
+ "pass@4": 0.768419161297034,
1439
+ "pass@8": 0.8132989938840093,
1440
+ "pass@16": 0.8486630253769624,
1441
+ "pass@32": 0.8756942867659188,
1442
+ "pass@64": 0.8953656362346946,
1443
+ "pass@128": 0.9075144508670521
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.5464732142857143,
1447
+ "pass@2": 0.6480627812148483,
1448
+ "pass@4": 0.7236771966896989,
1449
+ "pass@8": 0.777945603229985,
1450
+ "pass@16": 0.8163166798894307,
1451
+ "pass@32": 0.8431802277020857,
1452
+ "pass@64": 0.8622731277447071,
1453
+ "pass@128": 0.8771428571428571
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.4315643768768769,
1457
+ "pass@2": 0.5412484334728428,
1458
+ "pass@4": 0.6303853890448375,
1459
+ "pass@8": 0.6978895463021549,
1460
+ "pass@16": 0.7471806522306299,
1461
+ "pass@32": 0.7842191914351475,
1462
+ "pass@64": 0.8144645683656517,
1463
+ "pass@128": 0.8438438438438438
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.4473066816816817,
1467
+ "pass@2": 0.550585157007401,
1468
+ "pass@4": 0.6322523291007539,
1469
+ "pass@8": 0.6917831399797562,
1470
+ "pass@16": 0.7350186424328486,
1471
+ "pass@32": 0.7688185050754536,
1472
+ "pass@64": 0.7971501932040521,
1473
+ "pass@128": 0.8228228228228228
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 686415,
1480
+ "answer_accuracy": 0.8359496784879189,
1481
+ "avg_response_len": 286.5450993764614,
1482
+ "resp_tokens_sum": 235287912,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.8359496784879189,
1485
+ "pass@2": 0.8760342317157656,
1486
+ "pass@4": 0.9043035755711732,
1487
+ "pass@8": 0.9240919493445696,
1488
+ "pass@16": 0.9381619892173285,
1489
+ "pass@32": 0.9489126547729012,
1490
+ "pass@64": 0.9576937527310582,
1491
+ "pass@128": 0.9649259547934529
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.924079754601227,
1496
+ "pass@2": 0.9425480653108548,
1497
+ "pass@4": 0.9558939233439372,
1498
+ "pass@8": 0.9660136894297928,
1499
+ "pass@16": 0.9745849213182081,
1500
+ "pass@32": 0.9815053815983932,
1501
+ "pass@64": 0.9856196459168264,
1502
+ "pass@128": 0.9877300613496932
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.9999050151975684,
1506
+ "pass@2": 0.9999977562645096,
1507
+ "pass@4": 0.9999999997150811,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9999763257575758,
1516
+ "pass@2": 1.0,
1517
+ "pass@4": 1.0,
1518
+ "pass@8": 1.0,
1519
+ "pass@16": 1.0,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9998844304733728,
1526
+ "pass@2": 0.9999978160089457,
1527
+ "pass@4": 0.9999999997226678,
1528
+ "pass@8": 1.0,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9920969202898551,
1536
+ "pass@2": 0.9947952327969871,
1537
+ "pass@4": 0.9963463162213417,
1538
+ "pass@8": 0.9977460600531718,
1539
+ "pass@16": 0.9990916148451592,
1540
+ "pass@32": 0.9998484564379228,
1541
+ "pass@64": 0.999998063383779,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9952542979942693,
1546
+ "pass@2": 0.9966728617196489,
1547
+ "pass@4": 0.9972907742728435,
1548
+ "pass@8": 0.997639465969732,
1549
+ "pass@16": 0.9980869583411133,
1550
+ "pass@32": 0.9988007408816018,
1551
+ "pass@64": 0.9996502944295288,
1552
+ "pass@128": 1.0
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9967937138728323,
1556
+ "pass@2": 0.9983024498202175,
1557
+ "pass@4": 0.9990612040546953,
1558
+ "pass@8": 0.9997078494159238,
1559
+ "pass@16": 0.999975444312545,
1560
+ "pass@32": 0.9999999133144741,
1561
+ "pass@64": 0.9999999999999895,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9918573943661971,
1566
+ "pass@2": 0.9956231978485083,
1567
+ "pass@4": 0.9969242494336096,
1568
+ "pass@8": 0.9971736180162805,
1569
+ "pass@16": 0.9971830890857465,
1570
+ "pass@32": 0.9971830985915483,
1571
+ "pass@64": 0.9971830985915493,
1572
+ "pass@128": 0.9971830985915493
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9910179640718563,
1576
+ "pass@2": 0.9937379178650573,
1577
+ "pass@4": 0.9939987547402381,
1578
+ "pass@8": 0.9940119380249567,
1579
+ "pass@16": 0.9940119760478334,
1580
+ "pass@32": 0.9940119760479041,
1581
+ "pass@64": 0.9940119760479041,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9533838478915663,
1586
+ "pass@2": 0.9686280802343229,
1587
+ "pass@4": 0.9774122846842942,
1588
+ "pass@8": 0.9826945678160793,
1589
+ "pass@16": 0.9851505016991625,
1590
+ "pass@32": 0.9862401453810479,
1591
+ "pass@64": 0.9872047186881181,
1592
+ "pass@128": 0.9879518072289156
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.885384012539185,
1596
+ "pass@2": 0.9101669525831221,
1597
+ "pass@4": 0.9239799511738148,
1598
+ "pass@8": 0.9333532233276649,
1599
+ "pass@16": 0.9404962983721141,
1600
+ "pass@32": 0.947095869663446,
1601
+ "pass@64": 0.9545776664683873,
1602
+ "pass@128": 0.9623824451410659
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.8328956582633054,
1606
+ "pass@2": 0.8768782119146881,
1607
+ "pass@4": 0.9001525788793208,
1608
+ "pass@8": 0.9144132901889223,
1609
+ "pass@16": 0.9252804791130302,
1610
+ "pass@32": 0.9358350221164605,
1611
+ "pass@64": 0.9474943865376289,
1612
+ "pass@128": 0.9607843137254902
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.8158351608187134,
1616
+ "pass@2": 0.8764760182115391,
1617
+ "pass@4": 0.9120771337245117,
1618
+ "pass@8": 0.9326000038402523,
1619
+ "pass@16": 0.9449230559731131,
1620
+ "pass@32": 0.9539728291687525,
1621
+ "pass@64": 0.9628130945239121,
1622
+ "pass@128": 0.9707602339181286
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.7901515151515152,
1626
+ "pass@2": 0.8613483506323071,
1627
+ "pass@4": 0.9039200494824501,
1628
+ "pass@8": 0.9292158194336342,
1629
+ "pass@16": 0.9452248133475442,
1630
+ "pass@32": 0.9574867632372372,
1631
+ "pass@64": 0.9678628579175225,
1632
+ "pass@128": 0.9787878787878788
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.6649528133903134,
1636
+ "pass@2": 0.7624254800681963,
1637
+ "pass@4": 0.8277979675617472,
1638
+ "pass@8": 0.8686706781122179,
1639
+ "pass@16": 0.8961598925297205,
1640
+ "pass@32": 0.9179229188572494,
1641
+ "pass@64": 0.9337860632473033,
1642
+ "pass@128": 0.9430199430199431
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.6393390916149069,
1646
+ "pass@2": 0.7354016634469608,
1647
+ "pass@4": 0.8004865514675568,
1648
+ "pass@8": 0.8453401336925654,
1649
+ "pass@16": 0.8772971858982275,
1650
+ "pass@32": 0.9004220002674332,
1651
+ "pass@64": 0.9174945886997983,
1652
+ "pass@128": 0.9316770186335404
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.5892693014705882,
1656
+ "pass@2": 0.6895900880037054,
1657
+ "pass@4": 0.7655650637420315,
1658
+ "pass@8": 0.8203016454124229,
1659
+ "pass@16": 0.859618198955504,
1660
+ "pass@32": 0.8878937646484227,
1661
+ "pass@64": 0.9097340634600377,
1662
+ "pass@128": 0.9294117647058824
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.4204799107142857,
1666
+ "pass@2": 0.5389387361736032,
1667
+ "pass@4": 0.6410350558858714,
1668
+ "pass@8": 0.7190375968217035,
1669
+ "pass@16": 0.7736645475875377,
1670
+ "pass@32": 0.8123192037757934,
1671
+ "pass@64": 0.8394299819521862,
1672
+ "pass@128": 0.8571428571428571
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.3888005239520958,
1676
+ "pass@2": 0.4932697351360271,
1677
+ "pass@4": 0.581708334175294,
1678
+ "pass@8": 0.6533021743606491,
1679
+ "pass@16": 0.7090402348266315,
1680
+ "pass@32": 0.754332532451074,
1681
+ "pass@64": 0.7954232145576876,
1682
+ "pass@128": 0.8293413173652695
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 661822,
1689
+ "answer_accuracy": 0.8260879333759387,
1690
+ "avg_response_len": 307.27815945039146,
1691
+ "resp_tokens_sum": 246176512,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.8260879333759387,
1694
+ "pass@2": 0.8676958919313628,
1695
+ "pass@4": 0.8980069189605794,
1696
+ "pass@8": 0.9198953723230876,
1697
+ "pass@16": 0.93505318616042,
1698
+ "pass@32": 0.94562924717024,
1699
+ "pass@64": 0.9530547716086694,
1700
+ "pass@128": 0.9583000479309793
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9465951492537313,
1705
+ "pass@2": 0.9632176225173348,
1706
+ "pass@4": 0.9735499216515845,
1707
+ "pass@8": 0.9811835925161799,
1708
+ "pass@16": 0.9873330366748251,
1709
+ "pass@32": 0.9918809176478053,
1710
+ "pass@64": 0.993806150809024,
1711
+ "pass@128": 0.9940298507462687
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 1.0,
1715
+ "pass@2": 1.0,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9999316690962099,
1725
+ "pass@2": 0.9999996413075917,
1726
+ "pass@4": 1.0,
1727
+ "pass@8": 1.0,
1728
+ "pass@16": 1.0,
1729
+ "pass@32": 1.0,
1730
+ "pass@64": 1.0,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9981953581871345,
1735
+ "pass@2": 0.9990279792328591,
1736
+ "pass@4": 0.9996846265927285,
1737
+ "pass@8": 0.9999691763603531,
1738
+ "pass@16": 0.9999997893064115,
1739
+ "pass@32": 0.9999999999982297,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9918198529411765,
1745
+ "pass@2": 0.9951822313571099,
1746
+ "pass@4": 0.9976438364322108,
1747
+ "pass@8": 0.9994046739409391,
1748
+ "pass@16": 0.9999673976331993,
1749
+ "pass@32": 0.9999999551856581,
1750
+ "pass@64": 0.9999999999999997,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.993764787066246,
1755
+ "pass@2": 0.9947496212027123,
1756
+ "pass@4": 0.9955808440940151,
1757
+ "pass@8": 0.9967326641590364,
1758
+ "pass@16": 0.9979618214750134,
1759
+ "pass@32": 0.9990096140434843,
1760
+ "pass@64": 0.9998121152924854,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9956487341772152,
1765
+ "pass@2": 0.9986505469450812,
1766
+ "pass@4": 0.9995743400943553,
1767
+ "pass@8": 0.9999481010918784,
1768
+ "pass@16": 0.9999994169232284,
1769
+ "pass@32": 0.9999999999832521,
1770
+ "pass@64": 1.0,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9844923048048048,
1775
+ "pass@2": 0.9919176065829609,
1776
+ "pass@4": 0.9963254145596664,
1777
+ "pass@8": 0.9990655284449819,
1778
+ "pass@16": 0.9999326120862155,
1779
+ "pass@32": 0.9999996834444386,
1780
+ "pass@64": 0.999999999999752,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9835962620578779,
1785
+ "pass@2": 0.9923585968554577,
1786
+ "pass@4": 0.9956467118660005,
1787
+ "pass@8": 0.9975453735871543,
1788
+ "pass@16": 0.9990224437782512,
1789
+ "pass@32": 0.9998343032263491,
1790
+ "pass@64": 0.9999978524135079,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.9548611111111112,
1795
+ "pass@2": 0.9714758311461071,
1796
+ "pass@4": 0.9792459415787311,
1797
+ "pass@8": 0.9832824257084963,
1798
+ "pass@16": 0.9859409115458316,
1799
+ "pass@32": 0.9888924101781106,
1800
+ "pass@64": 0.9916644760868305,
1801
+ "pass@128": 0.9936507936507937
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8987241972477065,
1805
+ "pass@2": 0.9262153404849626,
1806
+ "pass@4": 0.9399879209479548,
1807
+ "pass@8": 0.9515969150881995,
1808
+ "pass@16": 0.9632921895262051,
1809
+ "pass@32": 0.9727991681405372,
1810
+ "pass@64": 0.978484053984944,
1811
+ "pass@128": 0.981651376146789
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.7942467206790124,
1815
+ "pass@2": 0.8419292098036356,
1816
+ "pass@4": 0.87347388607674,
1817
+ "pass@8": 0.896955802177486,
1818
+ "pass@16": 0.9148879344767462,
1819
+ "pass@32": 0.9305686019139384,
1820
+ "pass@64": 0.9444548045895704,
1821
+ "pass@128": 0.9537037037037037
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.7969445474777448,
1825
+ "pass@2": 0.8519073167597371,
1826
+ "pass@4": 0.8856038931706228,
1827
+ "pass@8": 0.9087465231648425,
1828
+ "pass@16": 0.9243278096477272,
1829
+ "pass@32": 0.9348198946343538,
1830
+ "pass@64": 0.9430984318055536,
1831
+ "pass@128": 0.9495548961424333
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.7702064043209876,
1835
+ "pass@2": 0.8314692542772435,
1836
+ "pass@4": 0.8710929519226761,
1837
+ "pass@8": 0.9004926868932659,
1838
+ "pass@16": 0.9209909756435962,
1839
+ "pass@32": 0.9332392643870973,
1840
+ "pass@64": 0.9400295244557415,
1841
+ "pass@128": 0.9444444444444444
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.6661366150442478,
1845
+ "pass@2": 0.7643184708382698,
1846
+ "pass@4": 0.8284491338472064,
1847
+ "pass@8": 0.8694132293085024,
1848
+ "pass@16": 0.8955625569041571,
1849
+ "pass@32": 0.9130243917210557,
1850
+ "pass@64": 0.926146214710774,
1851
+ "pass@128": 0.9380530973451328
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.5658179593373494,
1855
+ "pass@2": 0.6702137042263541,
1856
+ "pass@4": 0.7485751202221707,
1857
+ "pass@8": 0.8014946697752476,
1858
+ "pass@16": 0.8341101603053381,
1859
+ "pass@32": 0.8534621646954216,
1860
+ "pass@64": 0.8669435177559548,
1861
+ "pass@128": 0.8795180722891566
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.55859375,
1865
+ "pass@2": 0.6654674403098803,
1866
+ "pass@4": 0.7465657682910599,
1867
+ "pass@8": 0.801548579552477,
1868
+ "pass@16": 0.8354467036205091,
1869
+ "pass@32": 0.8565727072192222,
1870
+ "pass@64": 0.8703706077422788,
1871
+ "pass@128": 0.8806451612903226
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.3890436178247734,
1875
+ "pass@2": 0.5075246211670672,
1876
+ "pass@4": 0.6101444158604039,
1877
+ "pass@8": 0.6887828332064626,
1878
+ "pass@16": 0.7462233374111003,
1879
+ "pass@32": 0.7901599832812849,
1880
+ "pass@64": 0.8223790409544078,
1881
+ "pass@128": 0.8429003021148036
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.39846096096096095,
1885
+ "pass@2": 0.5144094635974552,
1886
+ "pass@4": 0.6162668182805975,
1887
+ "pass@8": 0.6980383217113035,
1888
+ "pass@16": 0.7577338561538128,
1889
+ "pass@32": 0.799757548916951,
1890
+ "pass@64": 0.828102483308844,
1891
+ "pass@128": 0.8468468468468469
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch4-op17-20/SDAR-100M-block16-epoch4-op17-20_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b21f23cf7c95aaee22d5a011127f91f6e887a3096a79c85a40541c898f663946
3
+ size 2710541884
results/composition/SDAR-100M-block16-epoch4-op17-20/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch4-op17-20,0.9890277777777777,0.6918046875,0.832594572368421,0.0,0.0,0.0,199.14139930555555,372.07711328125,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch4-op17-20/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch4-op17-20",
4
+ "id_acc": 0.9890277777777777,
5
+ "ood_acc": 0.6918046875,
6
+ "total_acc": 0.832594572368421,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.14139930555555,
11
+ "ood_avg_resp_len": 372.07711328125,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 676633,
19
+ "answer_accuracy": 0.8356299893297502,
20
+ "avg_response_len": 276.8894900509801,
21
+ "resp_tokens_sum": 224205173,
22
+ "pass_at_k": {
23
+ "pass@1": 0.8356299893297502,
24
+ "pass@2": 0.8748209761426936,
25
+ "pass@4": 0.9020187386472343,
26
+ "pass@8": 0.9209896497859279,
27
+ "pass@16": 0.9346778150804824,
28
+ "pass@32": 0.9451618942195561,
29
+ "pass@64": 0.9534618266585658,
30
+ "pass@128": 0.9603224786595005
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9463956489675516,
35
+ "pass@2": 0.9599948029173347,
36
+ "pass@4": 0.9696439955514411,
37
+ "pass@8": 0.9772987780870966,
38
+ "pass@16": 0.9829576313879089,
39
+ "pass@32": 0.9869532680938505,
40
+ "pass@64": 0.9904289041602642,
41
+ "pass@128": 0.9941002949852508
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9999513239875389,
45
+ "pass@2": 0.9999996167243114,
46
+ "pass@4": 1.0,
47
+ "pass@8": 1.0,
48
+ "pass@16": 1.0,
49
+ "pass@32": 1.0,
50
+ "pass@64": 1.0,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 1.0,
55
+ "pass@2": 1.0,
56
+ "pass@4": 1.0,
57
+ "pass@8": 1.0,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.9962890625,
65
+ "pass@2": 0.9968965305118112,
66
+ "pass@4": 0.9969725551884139,
67
+ "pass@8": 0.9970703124996394,
68
+ "pass@16": 0.997265625,
69
+ "pass@32": 0.99765625,
70
+ "pass@64": 0.9984375,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9941220238095239,
75
+ "pass@2": 0.9976346706661666,
76
+ "pass@4": 0.9991279667422525,
77
+ "pass@8": 0.999810090374339,
78
+ "pass@16": 0.9999911414698637,
79
+ "pass@32": 0.9999999920410901,
80
+ "pass@64": 1.0,
81
+ "pass@128": 1.0
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9975673652694611,
85
+ "pass@2": 0.9990827891461174,
86
+ "pass@4": 0.9997700759897528,
87
+ "pass@8": 0.9999845197580833,
88
+ "pass@16": 0.9999999528630369,
89
+ "pass@32": 0.9999999999999486,
90
+ "pass@64": 1.0,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9960012943786982,
95
+ "pass@2": 0.9984763022410661,
96
+ "pass@4": 0.9996465231971745,
97
+ "pass@8": 0.9999815406455984,
98
+ "pass@16": 0.9999999678464966,
99
+ "pass@32": 0.9999999999999909,
100
+ "pass@64": 1.0,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9963441506410257,
105
+ "pass@2": 0.9996159209065214,
106
+ "pass@4": 0.9999852656639074,
107
+ "pass@8": 0.9999999592877328,
108
+ "pass@16": 0.9999999999999242,
109
+ "pass@32": 1.0,
110
+ "pass@64": 1.0,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9913292253521127,
115
+ "pass@2": 0.9970926444493733,
116
+ "pass@4": 0.9988132281880259,
117
+ "pass@8": 0.9997344814601936,
118
+ "pass@16": 0.9999856314389629,
119
+ "pass@32": 0.9999999749550461,
120
+ "pass@64": 0.9999999999999997,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.9385844546742209,
125
+ "pass@2": 0.9559055815172538,
126
+ "pass@4": 0.9662483132420059,
127
+ "pass@8": 0.9743145176794873,
128
+ "pass@16": 0.9814528957415205,
129
+ "pass@32": 0.9870929428344121,
130
+ "pass@64": 0.9904799574420998,
131
+ "pass@128": 0.9915014164305949
132
+ },
133
+ "12": {
134
+ "pass@1": 0.8917284604519774,
135
+ "pass@2": 0.9156980542951199,
136
+ "pass@4": 0.9281065293215466,
137
+ "pass@8": 0.935889004485147,
138
+ "pass@16": 0.9418117371185404,
139
+ "pass@32": 0.947600022930526,
140
+ "pass@64": 0.9542037056880797,
141
+ "pass@128": 0.963276836158192
142
+ },
143
+ "13": {
144
+ "pass@1": 0.8402233542319749,
145
+ "pass@2": 0.8852274269987415,
146
+ "pass@4": 0.9120427055748123,
147
+ "pass@8": 0.9300158215455117,
148
+ "pass@16": 0.9440746211517979,
149
+ "pass@32": 0.9571960077579881,
150
+ "pass@64": 0.9683369401551648,
151
+ "pass@128": 0.9749216300940439
152
+ },
153
+ "14": {
154
+ "pass@1": 0.8298773364485982,
155
+ "pass@2": 0.8854661092305048,
156
+ "pass@4": 0.9165421769708681,
157
+ "pass@8": 0.9342494936091128,
158
+ "pass@16": 0.9469401578089754,
159
+ "pass@32": 0.9577873713575069,
160
+ "pass@64": 0.9667747235355996,
161
+ "pass@128": 0.9750778816199377
162
+ },
163
+ "15": {
164
+ "pass@1": 0.7909365968208093,
165
+ "pass@2": 0.8548815057348322,
166
+ "pass@4": 0.8965234123552469,
167
+ "pass@8": 0.9232631518428708,
168
+ "pass@16": 0.93956332213618,
169
+ "pass@32": 0.9494509870773286,
170
+ "pass@64": 0.9560635736921728,
171
+ "pass@128": 0.9624277456647399
172
+ },
173
+ "16": {
174
+ "pass@1": 0.6512096774193549,
175
+ "pass@2": 0.7479818865887731,
176
+ "pass@4": 0.8152397825271837,
177
+ "pass@8": 0.8589674413513186,
178
+ "pass@16": 0.8885187025219686,
179
+ "pass@32": 0.9122664817583825,
180
+ "pass@64": 0.9310470742858282,
181
+ "pass@128": 0.9419354838709677
182
+ },
183
+ "17": {
184
+ "pass@1": 0.6124909682080925,
185
+ "pass@2": 0.7055230474261528,
186
+ "pass@4": 0.768419161297034,
187
+ "pass@8": 0.8132989938840093,
188
+ "pass@16": 0.8486630253769624,
189
+ "pass@32": 0.8756942867659188,
190
+ "pass@64": 0.8953656362346946,
191
+ "pass@128": 0.9075144508670521
192
+ },
193
+ "18": {
194
+ "pass@1": 0.5464732142857143,
195
+ "pass@2": 0.6480627812148483,
196
+ "pass@4": 0.7236771966896989,
197
+ "pass@8": 0.777945603229985,
198
+ "pass@16": 0.8163166798894307,
199
+ "pass@32": 0.8431802277020857,
200
+ "pass@64": 0.8622731277447071,
201
+ "pass@128": 0.8771428571428571
202
+ },
203
+ "19": {
204
+ "pass@1": 0.4315643768768769,
205
+ "pass@2": 0.5412484334728428,
206
+ "pass@4": 0.6303853890448375,
207
+ "pass@8": 0.6978895463021549,
208
+ "pass@16": 0.7471806522306299,
209
+ "pass@32": 0.7842191914351475,
210
+ "pass@64": 0.8144645683656517,
211
+ "pass@128": 0.8438438438438438
212
+ },
213
+ "20": {
214
+ "pass@1": 0.4473066816816817,
215
+ "pass@2": 0.550585157007401,
216
+ "pass@4": 0.6322523291007539,
217
+ "pass@8": 0.6917831399797562,
218
+ "pass@16": 0.7350186424328486,
219
+ "pass@32": 0.7688185050754536,
220
+ "pass@64": 0.7971501932040521,
221
+ "pass@128": 0.8228228228228228
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 686415,
228
+ "answer_accuracy": 0.8359496784879189,
229
+ "avg_response_len": 286.5450993764614,
230
+ "resp_tokens_sum": 235287912,
231
+ "pass_at_k": {
232
+ "pass@1": 0.8359496784879189,
233
+ "pass@2": 0.8760342317157656,
234
+ "pass@4": 0.9043035755711732,
235
+ "pass@8": 0.9240919493445696,
236
+ "pass@16": 0.9381619892173285,
237
+ "pass@32": 0.9489126547729012,
238
+ "pass@64": 0.9576937527310582,
239
+ "pass@128": 0.9649259547934529
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.924079754601227,
244
+ "pass@2": 0.9425480653108548,
245
+ "pass@4": 0.9558939233439372,
246
+ "pass@8": 0.9660136894297928,
247
+ "pass@16": 0.9745849213182081,
248
+ "pass@32": 0.9815053815983932,
249
+ "pass@64": 0.9856196459168264,
250
+ "pass@128": 0.9877300613496932
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9999050151975684,
254
+ "pass@2": 0.9999977562645096,
255
+ "pass@4": 0.9999999997150811,
256
+ "pass@8": 1.0,
257
+ "pass@16": 1.0,
258
+ "pass@32": 1.0,
259
+ "pass@64": 1.0,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9999763257575758,
264
+ "pass@2": 1.0,
265
+ "pass@4": 1.0,
266
+ "pass@8": 1.0,
267
+ "pass@16": 1.0,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9998844304733728,
274
+ "pass@2": 0.9999978160089457,
275
+ "pass@4": 0.9999999997226678,
276
+ "pass@8": 1.0,
277
+ "pass@16": 1.0,
278
+ "pass@32": 1.0,
279
+ "pass@64": 1.0,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9920969202898551,
284
+ "pass@2": 0.9947952327969871,
285
+ "pass@4": 0.9963463162213417,
286
+ "pass@8": 0.9977460600531718,
287
+ "pass@16": 0.9990916148451592,
288
+ "pass@32": 0.9998484564379228,
289
+ "pass@64": 0.999998063383779,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9952542979942693,
294
+ "pass@2": 0.9966728617196489,
295
+ "pass@4": 0.9972907742728435,
296
+ "pass@8": 0.997639465969732,
297
+ "pass@16": 0.9980869583411133,
298
+ "pass@32": 0.9988007408816018,
299
+ "pass@64": 0.9996502944295288,
300
+ "pass@128": 1.0
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9967937138728323,
304
+ "pass@2": 0.9983024498202175,
305
+ "pass@4": 0.9990612040546953,
306
+ "pass@8": 0.9997078494159238,
307
+ "pass@16": 0.999975444312545,
308
+ "pass@32": 0.9999999133144741,
309
+ "pass@64": 0.9999999999999895,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9918573943661971,
314
+ "pass@2": 0.9956231978485083,
315
+ "pass@4": 0.9969242494336096,
316
+ "pass@8": 0.9971736180162805,
317
+ "pass@16": 0.9971830890857465,
318
+ "pass@32": 0.9971830985915483,
319
+ "pass@64": 0.9971830985915493,
320
+ "pass@128": 0.9971830985915493
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9910179640718563,
324
+ "pass@2": 0.9937379178650573,
325
+ "pass@4": 0.9939987547402381,
326
+ "pass@8": 0.9940119380249567,
327
+ "pass@16": 0.9940119760478334,
328
+ "pass@32": 0.9940119760479041,
329
+ "pass@64": 0.9940119760479041,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9533838478915663,
334
+ "pass@2": 0.9686280802343229,
335
+ "pass@4": 0.9774122846842942,
336
+ "pass@8": 0.9826945678160793,
337
+ "pass@16": 0.9851505016991625,
338
+ "pass@32": 0.9862401453810479,
339
+ "pass@64": 0.9872047186881181,
340
+ "pass@128": 0.9879518072289156
341
+ },
342
+ "12": {
343
+ "pass@1": 0.885384012539185,
344
+ "pass@2": 0.9101669525831221,
345
+ "pass@4": 0.9239799511738148,
346
+ "pass@8": 0.9333532233276649,
347
+ "pass@16": 0.9404962983721141,
348
+ "pass@32": 0.947095869663446,
349
+ "pass@64": 0.9545776664683873,
350
+ "pass@128": 0.9623824451410659
351
+ },
352
+ "13": {
353
+ "pass@1": 0.8328956582633054,
354
+ "pass@2": 0.8768782119146881,
355
+ "pass@4": 0.9001525788793208,
356
+ "pass@8": 0.9144132901889223,
357
+ "pass@16": 0.9252804791130302,
358
+ "pass@32": 0.9358350221164605,
359
+ "pass@64": 0.9474943865376289,
360
+ "pass@128": 0.9607843137254902
361
+ },
362
+ "14": {
363
+ "pass@1": 0.8158351608187134,
364
+ "pass@2": 0.8764760182115391,
365
+ "pass@4": 0.9120771337245117,
366
+ "pass@8": 0.9326000038402523,
367
+ "pass@16": 0.9449230559731131,
368
+ "pass@32": 0.9539728291687525,
369
+ "pass@64": 0.9628130945239121,
370
+ "pass@128": 0.9707602339181286
371
+ },
372
+ "15": {
373
+ "pass@1": 0.7901515151515152,
374
+ "pass@2": 0.8613483506323071,
375
+ "pass@4": 0.9039200494824501,
376
+ "pass@8": 0.9292158194336342,
377
+ "pass@16": 0.9452248133475442,
378
+ "pass@32": 0.9574867632372372,
379
+ "pass@64": 0.9678628579175225,
380
+ "pass@128": 0.9787878787878788
381
+ },
382
+ "16": {
383
+ "pass@1": 0.6649528133903134,
384
+ "pass@2": 0.7624254800681963,
385
+ "pass@4": 0.8277979675617472,
386
+ "pass@8": 0.8686706781122179,
387
+ "pass@16": 0.8961598925297205,
388
+ "pass@32": 0.9179229188572494,
389
+ "pass@64": 0.9337860632473033,
390
+ "pass@128": 0.9430199430199431
391
+ },
392
+ "17": {
393
+ "pass@1": 0.6393390916149069,
394
+ "pass@2": 0.7354016634469608,
395
+ "pass@4": 0.8004865514675568,
396
+ "pass@8": 0.8453401336925654,
397
+ "pass@16": 0.8772971858982275,
398
+ "pass@32": 0.9004220002674332,
399
+ "pass@64": 0.9174945886997983,
400
+ "pass@128": 0.9316770186335404
401
+ },
402
+ "18": {
403
+ "pass@1": 0.5892693014705882,
404
+ "pass@2": 0.6895900880037054,
405
+ "pass@4": 0.7655650637420315,
406
+ "pass@8": 0.8203016454124229,
407
+ "pass@16": 0.859618198955504,
408
+ "pass@32": 0.8878937646484227,
409
+ "pass@64": 0.9097340634600377,
410
+ "pass@128": 0.9294117647058824
411
+ },
412
+ "19": {
413
+ "pass@1": 0.4204799107142857,
414
+ "pass@2": 0.5389387361736032,
415
+ "pass@4": 0.6410350558858714,
416
+ "pass@8": 0.7190375968217035,
417
+ "pass@16": 0.7736645475875377,
418
+ "pass@32": 0.8123192037757934,
419
+ "pass@64": 0.8394299819521862,
420
+ "pass@128": 0.8571428571428571
421
+ },
422
+ "20": {
423
+ "pass@1": 0.3888005239520958,
424
+ "pass@2": 0.4932697351360271,
425
+ "pass@4": 0.581708334175294,
426
+ "pass@8": 0.6533021743606491,
427
+ "pass@16": 0.7090402348266315,
428
+ "pass@32": 0.754332532451074,
429
+ "pass@64": 0.7954232145576876,
430
+ "pass@128": 0.8293413173652695
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 661822,
437
+ "answer_accuracy": 0.8260879333759387,
438
+ "avg_response_len": 307.27815945039146,
439
+ "resp_tokens_sum": 246176512,
440
+ "pass_at_k": {
441
+ "pass@1": 0.8260879333759387,
442
+ "pass@2": 0.8676958919313628,
443
+ "pass@4": 0.8980069189605794,
444
+ "pass@8": 0.9198953723230876,
445
+ "pass@16": 0.93505318616042,
446
+ "pass@32": 0.94562924717024,
447
+ "pass@64": 0.9530547716086694,
448
+ "pass@128": 0.9583000479309793
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9465951492537313,
453
+ "pass@2": 0.9632176225173348,
454
+ "pass@4": 0.9735499216515845,
455
+ "pass@8": 0.9811835925161799,
456
+ "pass@16": 0.9873330366748251,
457
+ "pass@32": 0.9918809176478053,
458
+ "pass@64": 0.993806150809024,
459
+ "pass@128": 0.9940298507462687
460
+ },
461
+ "2": {
462
+ "pass@1": 1.0,
463
+ "pass@2": 1.0,
464
+ "pass@4": 1.0,
465
+ "pass@8": 1.0,
466
+ "pass@16": 1.0,
467
+ "pass@32": 1.0,
468
+ "pass@64": 1.0,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9999316690962099,
473
+ "pass@2": 0.9999996413075917,
474
+ "pass@4": 1.0,
475
+ "pass@8": 1.0,
476
+ "pass@16": 1.0,
477
+ "pass@32": 1.0,
478
+ "pass@64": 1.0,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9981953581871345,
483
+ "pass@2": 0.9990279792328591,
484
+ "pass@4": 0.9996846265927285,
485
+ "pass@8": 0.9999691763603531,
486
+ "pass@16": 0.9999997893064115,
487
+ "pass@32": 0.9999999999982297,
488
+ "pass@64": 1.0,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9918198529411765,
493
+ "pass@2": 0.9951822313571099,
494
+ "pass@4": 0.9976438364322108,
495
+ "pass@8": 0.9994046739409391,
496
+ "pass@16": 0.9999673976331993,
497
+ "pass@32": 0.9999999551856581,
498
+ "pass@64": 0.9999999999999997,
499
+ "pass@128": 1.0
500
+ },
501
+ "6": {
502
+ "pass@1": 0.993764787066246,
503
+ "pass@2": 0.9947496212027123,
504
+ "pass@4": 0.9955808440940151,
505
+ "pass@8": 0.9967326641590364,
506
+ "pass@16": 0.9979618214750134,
507
+ "pass@32": 0.9990096140434843,
508
+ "pass@64": 0.9998121152924854,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.9956487341772152,
513
+ "pass@2": 0.9986505469450812,
514
+ "pass@4": 0.9995743400943553,
515
+ "pass@8": 0.9999481010918784,
516
+ "pass@16": 0.9999994169232284,
517
+ "pass@32": 0.9999999999832521,
518
+ "pass@64": 1.0,
519
+ "pass@128": 1.0
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9844923048048048,
523
+ "pass@2": 0.9919176065829609,
524
+ "pass@4": 0.9963254145596664,
525
+ "pass@8": 0.9990655284449819,
526
+ "pass@16": 0.9999326120862155,
527
+ "pass@32": 0.9999996834444386,
528
+ "pass@64": 0.999999999999752,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9835962620578779,
533
+ "pass@2": 0.9923585968554577,
534
+ "pass@4": 0.9956467118660005,
535
+ "pass@8": 0.9975453735871543,
536
+ "pass@16": 0.9990224437782512,
537
+ "pass@32": 0.9998343032263491,
538
+ "pass@64": 0.9999978524135079,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.9548611111111112,
543
+ "pass@2": 0.9714758311461071,
544
+ "pass@4": 0.9792459415787311,
545
+ "pass@8": 0.9832824257084963,
546
+ "pass@16": 0.9859409115458316,
547
+ "pass@32": 0.9888924101781106,
548
+ "pass@64": 0.9916644760868305,
549
+ "pass@128": 0.9936507936507937
550
+ },
551
+ "12": {
552
+ "pass@1": 0.8987241972477065,
553
+ "pass@2": 0.9262153404849626,
554
+ "pass@4": 0.9399879209479548,
555
+ "pass@8": 0.9515969150881995,
556
+ "pass@16": 0.9632921895262051,
557
+ "pass@32": 0.9727991681405372,
558
+ "pass@64": 0.978484053984944,
559
+ "pass@128": 0.981651376146789
560
+ },
561
+ "13": {
562
+ "pass@1": 0.7942467206790124,
563
+ "pass@2": 0.8419292098036356,
564
+ "pass@4": 0.87347388607674,
565
+ "pass@8": 0.896955802177486,
566
+ "pass@16": 0.9148879344767462,
567
+ "pass@32": 0.9305686019139384,
568
+ "pass@64": 0.9444548045895704,
569
+ "pass@128": 0.9537037037037037
570
+ },
571
+ "14": {
572
+ "pass@1": 0.7969445474777448,
573
+ "pass@2": 0.8519073167597371,
574
+ "pass@4": 0.8856038931706228,
575
+ "pass@8": 0.9087465231648425,
576
+ "pass@16": 0.9243278096477272,
577
+ "pass@32": 0.9348198946343538,
578
+ "pass@64": 0.9430984318055536,
579
+ "pass@128": 0.9495548961424333
580
+ },
581
+ "15": {
582
+ "pass@1": 0.7702064043209876,
583
+ "pass@2": 0.8314692542772435,
584
+ "pass@4": 0.8710929519226761,
585
+ "pass@8": 0.9004926868932659,
586
+ "pass@16": 0.9209909756435962,
587
+ "pass@32": 0.9332392643870973,
588
+ "pass@64": 0.9400295244557415,
589
+ "pass@128": 0.9444444444444444
590
+ },
591
+ "16": {
592
+ "pass@1": 0.6661366150442478,
593
+ "pass@2": 0.7643184708382698,
594
+ "pass@4": 0.8284491338472064,
595
+ "pass@8": 0.8694132293085024,
596
+ "pass@16": 0.8955625569041571,
597
+ "pass@32": 0.9130243917210557,
598
+ "pass@64": 0.926146214710774,
599
+ "pass@128": 0.9380530973451328
600
+ },
601
+ "17": {
602
+ "pass@1": 0.5658179593373494,
603
+ "pass@2": 0.6702137042263541,
604
+ "pass@4": 0.7485751202221707,
605
+ "pass@8": 0.8014946697752476,
606
+ "pass@16": 0.8341101603053381,
607
+ "pass@32": 0.8534621646954216,
608
+ "pass@64": 0.8669435177559548,
609
+ "pass@128": 0.8795180722891566
610
+ },
611
+ "18": {
612
+ "pass@1": 0.55859375,
613
+ "pass@2": 0.6654674403098803,
614
+ "pass@4": 0.7465657682910599,
615
+ "pass@8": 0.801548579552477,
616
+ "pass@16": 0.8354467036205091,
617
+ "pass@32": 0.8565727072192222,
618
+ "pass@64": 0.8703706077422788,
619
+ "pass@128": 0.8806451612903226
620
+ },
621
+ "19": {
622
+ "pass@1": 0.3890436178247734,
623
+ "pass@2": 0.5075246211670672,
624
+ "pass@4": 0.6101444158604039,
625
+ "pass@8": 0.6887828332064626,
626
+ "pass@16": 0.7462233374111003,
627
+ "pass@32": 0.7901599832812849,
628
+ "pass@64": 0.8223790409544078,
629
+ "pass@128": 0.8429003021148036
630
+ },
631
+ "20": {
632
+ "pass@1": 0.39846096096096095,
633
+ "pass@2": 0.5144094635974552,
634
+ "pass@4": 0.6162668182805975,
635
+ "pass@8": 0.6980383217113035,
636
+ "pass@16": 0.7577338561538128,
637
+ "pass@32": 0.799757548916951,
638
+ "pass@64": 0.828102483308844,
639
+ "pass@128": 0.8468468468468469
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb818e9d3c62454f3489e1980f7b823cc05ceb82fbc59af63f720aa2d349199c
3
+ size 1789415021
results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch4-op7-10",
3
+ "id": {
4
+ "answer_accuracy": 0.9928402777777777,
5
+ "avg_response_len": 200.1375373263889,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9928402777777777,
11
+ "pass@2": 0.9949716207349067,
12
+ "pass@4": 0.9962314358309377,
13
+ "pass@8": 0.9970589677382524,
14
+ "pass@16": 0.997637285152463,
15
+ "pass@32": 0.9980942875801967,
16
+ "pass@64": 0.9984440103790677,
17
+ "pass@128": 0.9986666666666667
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.954203125,
22
+ "pass@2": 0.9668100393700787,
23
+ "pass@4": 0.9751169003562058,
24
+ "pass@8": 0.9805623138162619,
25
+ "pass@16": 0.9841740075635769,
26
+ "pass@32": 0.9868633714733398,
27
+ "pass@64": 0.9886806402008643,
28
+ "pass@128": 0.99
29
+ },
30
+ "2": {
31
+ "pass@1": 1.0,
32
+ "pass@2": 1.0,
33
+ "pass@4": 1.0,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 0.9999921875,
42
+ "pass@2": 1.0,
43
+ "pass@4": 1.0,
44
+ "pass@8": 1.0,
45
+ "pass@16": 1.0,
46
+ "pass@32": 1.0,
47
+ "pass@64": 1.0,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9990234375,
52
+ "pass@2": 0.999410063976378,
53
+ "pass@4": 0.9996751137982751,
54
+ "pass@8": 0.9998989158969318,
55
+ "pass@16": 0.9999915037321406,
56
+ "pass@32": 0.9999999700068081,
57
+ "pass@64": 0.9999999999999963,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.9949921875,
62
+ "pass@2": 0.9967944143700785,
63
+ "pass@4": 0.9977303270528682,
64
+ "pass@8": 0.9982286507773027,
65
+ "pass@16": 0.9986409291833077,
66
+ "pass@32": 0.999127515746347,
67
+ "pass@64": 0.9996924094487892,
68
+ "pass@128": 1.0
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9986875,
72
+ "pass@2": 0.9990652066929134,
73
+ "pass@4": 0.9992026385451819,
74
+ "pass@8": 0.99937067318584,
75
+ "pass@16": 0.9996167406956445,
76
+ "pass@32": 0.9998739053984705,
77
+ "pass@64": 0.9999934280775878,
78
+ "pass@128": 1.0
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9977421875,
82
+ "pass@2": 0.9986300442913385,
83
+ "pass@4": 0.9992237436257967,
84
+ "pass@8": 0.999713898666317,
85
+ "pass@16": 0.9999638315713223,
86
+ "pass@32": 0.9999996347541911,
87
+ "pass@64": 0.9999999999983727,
88
+ "pass@128": 1.0
89
+ },
90
+ "8": {
91
+ "pass@1": 0.9968125,
92
+ "pass@2": 0.998474655511811,
93
+ "pass@4": 0.9989533392388451,
94
+ "pass@8": 0.9989996318701388,
95
+ "pass@16": 0.9989999999891657,
96
+ "pass@32": 0.999,
97
+ "pass@64": 0.999,
98
+ "pass@128": 0.999
99
+ },
100
+ "9": {
101
+ "pass@1": 0.994109375,
102
+ "pass@2": 0.9955601624015749,
103
+ "pass@4": 0.9961808598612675,
104
+ "pass@8": 0.9967566254314778,
105
+ "pass@16": 0.9973485536370097,
106
+ "pass@32": 0.9979841908426146,
107
+ "pass@64": 0.9986296156860006,
108
+ "pass@128": 0.999
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.954203125,
113
+ "2": 1.0,
114
+ "3": 0.9999921875,
115
+ "4": 0.9990234375,
116
+ "5": 0.9949921875,
117
+ "6": 0.9986875,
118
+ "7": 0.9977421875,
119
+ "8": 0.9968125,
120
+ "9": 0.994109375
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 280.9476015625,
124
+ "2": 115.7252734375,
125
+ "3": 144.8482265625,
126
+ "4": 160.08678125,
127
+ "5": 173.194703125,
128
+ "6": 199.14746875,
129
+ "7": 224.20671875,
130
+ "8": 243.602875,
131
+ "9": 259.4781875
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 280.9476015625,
136
+ "2": 115.7252734375,
137
+ "3": 144.8482265625,
138
+ "4": 160.08678125,
139
+ "5": 173.194703125,
140
+ "6": 199.14746875,
141
+ "7": 224.20671875,
142
+ "8": 243.602875,
143
+ "9": 259.4781875
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 376709,
150
+ "answer_accuracy": 0.993934164978048,
151
+ "avg_response_len": 190.66135279466397,
152
+ "resp_tokens_sum": 72262178,
153
+ "pass_at_k": {
154
+ "pass@1": 0.993934164978048,
155
+ "pass@2": 0.9959355978109121,
156
+ "pass@4": 0.9969234930043573,
157
+ "pass@8": 0.9975032519300019,
158
+ "pass@16": 0.9979521898023057,
159
+ "pass@32": 0.998386677156877,
160
+ "pass@64": 0.9988549563110108,
161
+ "pass@128": 0.9993245525160419
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9639564896755162,
166
+ "pass@2": 0.9751933663159359,
167
+ "pass@4": 0.9805886808396739,
168
+ "pass@8": 0.9832729426842688,
169
+ "pass@16": 0.9853094435850058,
170
+ "pass@32": 0.9875159779599025,
171
+ "pass@64": 0.9903780082743677,
172
+ "pass@128": 0.9941002949852508
173
+ },
174
+ "2": {
175
+ "pass@1": 1.0,
176
+ "pass@2": 1.0,
177
+ "pass@4": 1.0,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9999761085626911,
186
+ "pass@2": 1.0,
187
+ "pass@4": 1.0,
188
+ "pass@8": 1.0,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.997314453125,
196
+ "pass@2": 0.9981799028051181,
197
+ "pass@4": 0.9989848275801462,
198
+ "pass@8": 0.9996841121782726,
199
+ "pass@16": 0.9999734491629393,
200
+ "pass@32": 0.9999999062712751,
201
+ "pass@64": 0.9999999999999887,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9964285714285714,
206
+ "pass@2": 0.9987411729783777,
207
+ "pass@4": 0.9997791192767571,
208
+ "pass@8": 0.9999910052051663,
209
+ "pass@16": 0.999999987524699,
210
+ "pass@32": 0.999999999999998,
211
+ "pass@64": 1.0,
212
+ "pass@128": 1.0
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9970761601796407,
216
+ "pass@2": 0.9973235124239709,
217
+ "pass@4": 0.9976157328200742,
218
+ "pass@8": 0.9981157895218378,
219
+ "pass@16": 0.998852517052828,
220
+ "pass@32": 0.9996224712529058,
221
+ "pass@64": 0.9999803235855924,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9972725591715976,
226
+ "pass@2": 0.9980547919675722,
227
+ "pass@4": 0.9987735604794961,
228
+ "pass@8": 0.9995079830474547,
229
+ "pass@16": 0.999928825622786,
230
+ "pass@32": 0.9999991140832827,
231
+ "pass@64": 0.9999999999952738,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9991486378205128,
236
+ "pass@2": 0.9999641158136482,
237
+ "pass@4": 0.999999888836011,
238
+ "pass@8": 0.9999999999996302,
239
+ "pass@16": 1.0,
240
+ "pass@32": 1.0,
241
+ "pass@64": 1.0,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9955325704225352,
246
+ "pass@2": 0.9969474326272597,
247
+ "pass@4": 0.9973978846834287,
248
+ "pass@8": 0.9976821039095084,
249
+ "pass@16": 0.998119293157022,
250
+ "pass@32": 0.9988210100498001,
251
+ "pass@64": 0.9996562049462128,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 387445,
259
+ "answer_accuracy": 0.9917804923001311,
260
+ "avg_response_len": 198.17119921363042,
261
+ "resp_tokens_sum": 77416768,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9917804923001311,
264
+ "pass@2": 0.9938671701659427,
265
+ "pass@4": 0.9952572525235622,
266
+ "pass@8": 0.996152824229983,
267
+ "pass@16": 0.9966572318246322,
268
+ "pass@32": 0.9970099127946118,
269
+ "pass@64": 0.9972736781091658,
270
+ "pass@128": 0.9973787680209698
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9412384969325154,
275
+ "pass@2": 0.9570337030819772,
276
+ "pass@4": 0.9684386097635029,
277
+ "pass@8": 0.9755421990555402,
278
+ "pass@16": 0.9790090135040668,
279
+ "pass@32": 0.9808182150392867,
280
+ "pass@64": 0.9815547734367644,
281
+ "pass@128": 0.9815950920245399
282
+ },
283
+ "2": {
284
+ "pass@1": 1.0,
285
+ "pass@2": 1.0,
286
+ "pass@4": 1.0,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 1.0,
295
+ "pass@2": 1.0,
296
+ "pass@4": 1.0,
297
+ "pass@8": 1.0,
298
+ "pass@16": 1.0,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9999075443786982,
305
+ "pass@2": 0.9999978160089457,
306
+ "pass@4": 0.9999999997226678,
307
+ "pass@8": 1.0,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9963768115942029,
315
+ "pass@2": 0.9971339010612804,
316
+ "pass@4": 0.9974446123582379,
317
+ "pass@8": 0.9977680501821345,
318
+ "pass@16": 0.9983124881128967,
319
+ "pass@32": 0.999097406302473,
320
+ "pass@64": 0.9998273650576287,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9997313753581661,
325
+ "pass@2": 0.9999841363400491,
326
+ "pass@4": 0.9999999435958757,
327
+ "pass@8": 0.9999999999999097,
328
+ "pass@16": 1.0,
329
+ "pass@32": 1.0,
330
+ "pass@64": 1.0,
331
+ "pass@128": 1.0
332
+ },
333
+ "7": {
334
+ "pass@1": 0.997674313583815,
335
+ "pass@2": 0.998275069978608,
336
+ "pass@4": 0.9989801556597332,
337
+ "pass@8": 0.9996539139236084,
338
+ "pass@16": 0.9999649956979594,
339
+ "pass@32": 0.9999998098093686,
340
+ "pass@64": 0.9999999999999141,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9957306338028169,
345
+ "pass@2": 0.9968081124542529,
346
+ "pass@4": 0.9971466248937193,
347
+ "pass@8": 0.9971827493473799,
348
+ "pass@16": 0.9971830985789736,
349
+ "pass@32": 0.9971830985915493,
350
+ "pass@64": 0.9971830985915493,
351
+ "pass@128": 0.9971830985915493
352
+ },
353
+ "9": {
354
+ "pass@1": 0.993310254491018,
355
+ "pass@2": 0.9940558106228489,
356
+ "pass@4": 0.994196379457059,
357
+ "pass@8": 0.9943759135003243,
358
+ "pass@16": 0.9947162761091989,
359
+ "pass@32": 0.995326276580697,
360
+ "pass@64": 0.9962633787543024,
361
+ "pass@128": 0.9970059880239521
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 379598,
368
+ "answer_accuracy": 0.9928387596250419,
369
+ "avg_response_len": 211.5403650192501,
370
+ "resp_tokens_sum": 80879497,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9928387596250419,
373
+ "pass@2": 0.9951445189653858,
374
+ "pass@4": 0.9965407850654927,
375
+ "pass@8": 0.9975444128321481,
376
+ "pass@16": 0.9983265017873332,
377
+ "pass@32": 0.9989124149016076,
378
+ "pass@64": 0.9992324413075104,
379
+ "pass@128": 0.9993304318714429
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9569496268656716,
384
+ "pass@2": 0.9678403014455282,
385
+ "pass@4": 0.976078659010907,
386
+ "pass@8": 0.9827045652363846,
387
+ "pass@16": 0.9880512471221918,
388
+ "pass@32": 0.9920857219167919,
389
+ "pass@64": 0.9938974306133376,
390
+ "pass@128": 0.9940298507462687
391
+ },
392
+ "2": {
393
+ "pass@1": 1.0,
394
+ "pass@2": 1.0,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 1.0,
404
+ "pass@2": 1.0,
405
+ "pass@4": 1.0,
406
+ "pass@8": 1.0,
407
+ "pass@16": 1.0,
408
+ "pass@32": 1.0,
409
+ "pass@64": 1.0,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9997487207602339,
414
+ "pass@2": 0.9999802142330892,
415
+ "pass@4": 0.9999999095507799,
416
+ "pass@8": 0.9999999999996626,
417
+ "pass@16": 1.0,
418
+ "pass@32": 1.0,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9922564338235295,
424
+ "pass@2": 0.994646320634553,
425
+ "pass@4": 0.9961220976973466,
426
+ "pass@8": 0.9970632553671733,
427
+ "pass@16": 0.997715072688465,
428
+ "pass@32": 0.9983497369764539,
429
+ "pass@64": 0.9992704955997272,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9992360015772871,
434
+ "pass@2": 0.9998886118631859,
435
+ "pass@4": 0.9999968563669289,
436
+ "pass@8": 0.9999999983772165,
437
+ "pass@16": 1.0,
438
+ "pass@32": 1.0,
439
+ "pass@64": 1.0,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.998318829113924,
444
+ "pass@2": 0.9996340202332306,
445
+ "pass@4": 0.9999719820425929,
446
+ "pass@8": 0.9999998296794577,
447
+ "pass@16": 0.999999999997869,
448
+ "pass@32": 1.0,
449
+ "pass@64": 1.0,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.995777027027027,
454
+ "pass@2": 0.9988557701402191,
455
+ "pass@4": 0.9998988651193375,
456
+ "pass@8": 0.9999992668226257,
457
+ "pass@16": 0.9999999999808709,
458
+ "pass@32": 1.0,
459
+ "pass@64": 1.0,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9933430466237942,
464
+ "pass@2": 0.9955922285236853,
465
+ "pass@4": 0.996922894083577,
466
+ "pass@8": 0.99825698853535,
467
+ "pass@16": 0.9992957181536803,
468
+ "pass@32": 0.9998834594758289,
469
+ "pass@64": 0.9999990174474539,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.4674421875,
478
+ "avg_response_len": 332.59690703125,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.4674421875,
484
+ "pass@2": 0.5340498277559019,
485
+ "pass@4": 0.5905295806805306,
486
+ "pass@8": 0.6377652123780675,
487
+ "pass@16": 0.6765026264659709,
488
+ "pass@32": 0.7087523766471188,
489
+ "pass@64": 0.7369760709987055,
490
+ "pass@128": 0.7626
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.9424921875,
495
+ "pass@2": 0.9608925935039377,
496
+ "pass@4": 0.9707150841769778,
497
+ "pass@8": 0.9776590164210477,
498
+ "pass@16": 0.983031112952659,
499
+ "pass@32": 0.9868108166417683,
500
+ "pass@64": 0.9895059512740915,
501
+ "pass@128": 0.992
502
+ },
503
+ "12": {
504
+ "pass@1": 0.850125,
505
+ "pass@2": 0.8868829970472448,
506
+ "pass@4": 0.9069662009748772,
507
+ "pass@8": 0.9210538138715969,
508
+ "pass@16": 0.932209399638482,
509
+ "pass@32": 0.941670117173944,
510
+ "pass@64": 0.9509294871368753,
511
+ "pass@128": 0.961
512
+ },
513
+ "13": {
514
+ "pass@1": 0.723671875,
515
+ "pass@2": 0.7991273375984257,
516
+ "pass@4": 0.848705464191976,
517
+ "pass@8": 0.8832906602998529,
518
+ "pass@16": 0.907676298226515,
519
+ "pass@32": 0.9258390025587503,
520
+ "pass@64": 0.9408623838730613,
521
+ "pass@128": 0.954
522
+ },
523
+ "14": {
524
+ "pass@1": 0.621359375,
525
+ "pass@2": 0.7092471702755899,
526
+ "pass@4": 0.7710694283839518,
527
+ "pass@8": 0.8143357774146883,
528
+ "pass@16": 0.8459435216064888,
529
+ "pass@32": 0.8706776031668482,
530
+ "pass@64": 0.8916396601771912,
531
+ "pass@128": 0.911
532
+ },
533
+ "15": {
534
+ "pass@1": 0.4664140625,
535
+ "pass@2": 0.5631503444881897,
536
+ "pass@4": 0.6409323717660281,
537
+ "pass@8": 0.700385082200824,
538
+ "pass@16": 0.7464661637312656,
539
+ "pass@32": 0.7838249784646978,
540
+ "pass@64": 0.8150213816108757,
541
+ "pass@128": 0.842
542
+ },
543
+ "16": {
544
+ "pass@1": 0.300421875,
545
+ "pass@2": 0.38325036909448806,
546
+ "pass@4": 0.45934014332583345,
547
+ "pass@8": 0.5256147490839751,
548
+ "pass@16": 0.5812667481374817,
549
+ "pass@32": 0.6290118259853492,
550
+ "pass@64": 0.6714562961595262,
551
+ "pass@128": 0.71
552
+ },
553
+ "17": {
554
+ "pass@1": 0.2392421875,
555
+ "pass@2": 0.3153343996062992,
556
+ "pass@4": 0.390559457255343,
557
+ "pass@8": 0.45835220794408055,
558
+ "pass@16": 0.515216747490821,
559
+ "pass@32": 0.5633337200265438,
560
+ "pass@64": 0.6057651819056846,
561
+ "pass@128": 0.644
562
+ },
563
+ "18": {
564
+ "pass@1": 0.20425,
565
+ "pass@2": 0.27352423720472446,
566
+ "pass@4": 0.3419236596362953,
567
+ "pass@8": 0.4029165957544186,
568
+ "pass@16": 0.45499798232417954,
569
+ "pass@32": 0.5004840348220753,
570
+ "pass@64": 0.5415533976453777,
571
+ "pass@128": 0.578
572
+ },
573
+ "19": {
574
+ "pass@1": 0.169625,
575
+ "pass@2": 0.23175504429133828,
576
+ "pass@4": 0.29506946662917083,
577
+ "pass@8": 0.353428445257125,
578
+ "pass@16": 0.40308684493744273,
579
+ "pass@32": 0.4453773587288245,
580
+ "pass@64": 0.4839624351859198,
581
+ "pass@128": 0.52
582
+ },
583
+ "20": {
584
+ "pass@1": 0.1568203125,
585
+ "pass@2": 0.217333784448819,
586
+ "pass@4": 0.2800145304649416,
587
+ "pass@8": 0.3406157755330686,
588
+ "pass@16": 0.39513144561425667,
589
+ "pass@32": 0.4404943089023678,
590
+ "pass@64": 0.47906453501838553,
591
+ "pass@128": 0.514
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.9424921875,
596
+ "12": 0.850125,
597
+ "13": 0.723671875,
598
+ "14": 0.621359375,
599
+ "15": 0.4664140625,
600
+ "16": 0.300421875,
601
+ "17": 0.2392421875,
602
+ "18": 0.20425,
603
+ "19": 0.169625,
604
+ "20": 0.1568203125
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 298.1274375,
608
+ "12": 307.074171875,
609
+ "13": 322.5293828125,
610
+ "14": 340.953328125,
611
+ "15": 345.885015625,
612
+ "16": 332.6650234375,
613
+ "17": 347.350203125,
614
+ "18": 347.9971171875,
615
+ "19": 336.1864375,
616
+ "20": 347.200953125
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 298.1274375,
621
+ "12": 307.074171875,
622
+ "13": 322.5293828125,
623
+ "14": 340.953328125,
624
+ "15": 345.885015625,
625
+ "16": 332.6650234375,
626
+ "17": 347.350203125,
627
+ "18": 347.9971171875,
628
+ "19": 336.1864375,
629
+ "20": 347.200953125
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 200217,
636
+ "answer_accuracy": 0.46484258915304605,
637
+ "avg_response_len": 315.6339617384844,
638
+ "resp_tokens_sum": 135949860,
639
+ "pass_at_k": {
640
+ "pass@1": 0.46484258915304605,
641
+ "pass@2": 0.5302466260486013,
642
+ "pass@4": 0.5859201473515673,
643
+ "pass@8": 0.6324395587767444,
644
+ "pass@16": 0.6708254517301923,
645
+ "pass@32": 0.7030991125764335,
646
+ "pass@64": 0.7314564132966396,
647
+ "pass@128": 0.7575037147102526
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9345122167138811,
652
+ "pass@2": 0.9542368143695208,
653
+ "pass@4": 0.9657462063877991,
654
+ "pass@8": 0.9749512748470323,
655
+ "pass@16": 0.9827357531203197,
656
+ "pass@32": 0.9884344587547554,
657
+ "pass@64": 0.9925691216469926,
658
+ "pass@128": 0.9971671388101983
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8454493290960452,
662
+ "pass@2": 0.8801169285777846,
663
+ "pass@4": 0.8989873583069065,
664
+ "pass@8": 0.9130351649779792,
665
+ "pass@16": 0.9250958046726186,
666
+ "pass@32": 0.9364960935539219,
667
+ "pass@64": 0.9474650567648585,
668
+ "pass@128": 0.9576271186440678
669
+ },
670
+ "13": {
671
+ "pass@1": 0.74047315830721,
672
+ "pass@2": 0.8119554217164869,
673
+ "pass@4": 0.8599147251969675,
674
+ "pass@8": 0.8945134207641785,
675
+ "pass@16": 0.9205661933194598,
676
+ "pass@32": 0.9404403427759488,
677
+ "pass@64": 0.9559823521026871,
678
+ "pass@128": 0.9686520376175548
679
+ },
680
+ "14": {
681
+ "pass@1": 0.6125632788161994,
682
+ "pass@2": 0.7013274370201388,
683
+ "pass@4": 0.7658476230190846,
684
+ "pass@8": 0.8097702298192307,
685
+ "pass@16": 0.8407102562368179,
686
+ "pass@32": 0.8644964958281782,
687
+ "pass@64": 0.8838021812976606,
688
+ "pass@128": 0.9003115264797508
689
+ },
690
+ "15": {
691
+ "pass@1": 0.48457821531791906,
692
+ "pass@2": 0.5827305027081152,
693
+ "pass@4": 0.6590647869702699,
694
+ "pass@8": 0.7158310076759379,
695
+ "pass@16": 0.7589664128271321,
696
+ "pass@32": 0.7930825505983045,
697
+ "pass@64": 0.8204516163076292,
698
+ "pass@128": 0.8410404624277457
699
+ },
700
+ "16": {
701
+ "pass@1": 0.2755040322580645,
702
+ "pass@2": 0.3586213328676657,
703
+ "pass@4": 0.43517604109970115,
704
+ "pass@8": 0.5017510566263869,
705
+ "pass@16": 0.5583572473734497,
706
+ "pass@32": 0.6078831915021241,
707
+ "pass@64": 0.6517453480905581,
708
+ "pass@128": 0.6903225806451613
709
+ },
710
+ "17": {
711
+ "pass@1": 0.21780166184971098,
712
+ "pass@2": 0.29274348857584986,
713
+ "pass@4": 0.36758594012453627,
714
+ "pass@8": 0.4327260544001389,
715
+ "pass@16": 0.4852625887317101,
716
+ "pass@32": 0.5308733536032256,
717
+ "pass@64": 0.5731955827266514,
718
+ "pass@128": 0.615606936416185
719
+ },
720
+ "18": {
721
+ "pass@1": 0.18082589285714284,
722
+ "pass@2": 0.24475007030371193,
723
+ "pass@4": 0.30927755396646855,
724
+ "pass@8": 0.3685482161042458,
725
+ "pass@16": 0.4204588630253465,
726
+ "pass@32": 0.46585847061159913,
727
+ "pass@64": 0.5068792519157992,
728
+ "pass@128": 0.5457142857142857
729
+ },
730
+ "19": {
731
+ "pass@1": 0.1753472222222222,
732
+ "pass@2": 0.23986560379276908,
733
+ "pass@4": 0.3042127763308865,
734
+ "pass@8": 0.3648899684541569,
735
+ "pass@16": 0.41793983675736257,
736
+ "pass@32": 0.46237397780199607,
737
+ "pass@64": 0.5029501083188042,
738
+ "pass@128": 0.5405405405405406
739
+ },
740
+ "20": {
741
+ "pass@1": 0.15636730480480482,
742
+ "pass@2": 0.2165424528859568,
743
+ "pass@4": 0.27821547475484476,
744
+ "pass@8": 0.3365076147164066,
745
+ "pass@16": 0.3888313889531125,
746
+ "pass@32": 0.4337362063462613,
747
+ "pass@64": 0.4738117632886543,
748
+ "pass@128": 0.5135135135135135
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 201848,
755
+ "answer_accuracy": 0.46890796907523047,
756
+ "avg_response_len": 328.5204500260184,
757
+ "resp_tokens_sum": 141416227,
758
+ "pass_at_k": {
759
+ "pass@1": 0.46890796907523047,
760
+ "pass@2": 0.5365449638961293,
761
+ "pass@4": 0.5941743382935776,
762
+ "pass@8": 0.6423905408438726,
763
+ "pass@16": 0.6812675313224216,
764
+ "pass@32": 0.7127574655170806,
765
+ "pass@64": 0.7399460641942278,
766
+ "pass@128": 0.7647933392804044
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9478068524096386,
771
+ "pass@2": 0.9611561403092685,
772
+ "pass@4": 0.9686419691702693,
773
+ "pass@8": 0.9750669382908692,
774
+ "pass@16": 0.9806081356024587,
775
+ "pass@32": 0.9842428892059527,
776
+ "pass@64": 0.9862549466314855,
777
+ "pass@128": 0.9879518072289156
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8410315438871473,
781
+ "pass@2": 0.8755083244884359,
782
+ "pass@4": 0.8937380000305601,
783
+ "pass@8": 0.905965360565369,
784
+ "pass@16": 0.9151980908995769,
785
+ "pass@32": 0.9235809759059825,
786
+ "pass@64": 0.9339121914379174,
787
+ "pass@128": 0.9467084639498433
788
+ },
789
+ "13": {
790
+ "pass@1": 0.7262123599439776,
791
+ "pass@2": 0.8062757091025381,
792
+ "pass@4": 0.8566690912585506,
793
+ "pass@8": 0.8882637919932588,
794
+ "pass@16": 0.9082264797954909,
795
+ "pass@32": 0.9222958616323473,
796
+ "pass@64": 0.9343518871725501,
797
+ "pass@128": 0.9467787114845938
798
+ },
799
+ "14": {
800
+ "pass@1": 0.6329952485380117,
801
+ "pass@2": 0.7221707792282543,
802
+ "pass@4": 0.7828848704767167,
803
+ "pass@8": 0.8258118272832482,
804
+ "pass@16": 0.8579023706600551,
805
+ "pass@32": 0.8832258032244631,
806
+ "pass@64": 0.9049950902372479,
807
+ "pass@128": 0.9269005847953217
808
+ },
809
+ "15": {
810
+ "pass@1": 0.4446022727272727,
811
+ "pass@2": 0.5437966028394173,
812
+ "pass@4": 0.6285262524002679,
813
+ "pass@8": 0.6960194300230498,
814
+ "pass@16": 0.7475932370874276,
815
+ "pass@32": 0.785985388651888,
816
+ "pass@64": 0.8147437752003887,
817
+ "pass@128": 0.8424242424242424
818
+ },
819
+ "16": {
820
+ "pass@1": 0.3059116809116809,
821
+ "pass@2": 0.39158751990936996,
822
+ "pass@4": 0.4727661555660242,
823
+ "pass@8": 0.5448737019577484,
824
+ "pass@16": 0.6040431432838912,
825
+ "pass@32": 0.6528764342680178,
826
+ "pass@64": 0.694521859735016,
827
+ "pass@128": 0.7293447293447294
828
+ },
829
+ "17": {
830
+ "pass@1": 0.26052989130434784,
831
+ "pass@2": 0.34320820291485293,
832
+ "pass@4": 0.42293633977041684,
833
+ "pass@8": 0.4927356348756277,
834
+ "pass@16": 0.5482837994495754,
835
+ "pass@32": 0.5921106992309274,
836
+ "pass@64": 0.6290502589670679,
837
+ "pass@128": 0.6614906832298136
838
+ },
839
+ "18": {
840
+ "pass@1": 0.2153262867647059,
841
+ "pass@2": 0.28677049559981455,
842
+ "pass@4": 0.35674156300315396,
843
+ "pass@8": 0.4197868754911729,
844
+ "pass@16": 0.47482275196634255,
845
+ "pass@32": 0.5236327311772606,
846
+ "pass@64": 0.5672130434247589,
847
+ "pass@128": 0.6029411764705882
848
+ },
849
+ "19": {
850
+ "pass@1": 0.16901506696428573,
851
+ "pass@2": 0.22576206294525675,
852
+ "pass@4": 0.28495160454273577,
853
+ "pass@8": 0.34010383127747634,
854
+ "pass@16": 0.38708927582856617,
855
+ "pass@32": 0.4276740630065036,
856
+ "pass@64": 0.4649332236307543,
857
+ "pass@128": 0.5
858
+ },
859
+ "20": {
860
+ "pass@1": 0.1504491017964072,
861
+ "pass@2": 0.2108255192371163,
862
+ "pass@4": 0.2735428047167756,
863
+ "pass@8": 0.3338204070406974,
864
+ "pass@16": 0.38676384570770866,
865
+ "pass@32": 0.4292259673414519,
866
+ "pass@64": 0.4663031080186876,
867
+ "pass@128": 0.5
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 196261,
874
+ "answer_accuracy": 0.4686091266809291,
875
+ "avg_response_len": 354.23182017878975,
876
+ "resp_tokens_sum": 148357954,
877
+ "pass_at_k": {
878
+ "pass@1": 0.4686091266809291,
879
+ "pass@2": 0.5353965975925558,
880
+ "pass@4": 0.5915239032047982,
881
+ "pass@8": 0.6384882700608202,
882
+ "pass@16": 0.6774437383099655,
883
+ "pass@32": 0.7104498460016812,
884
+ "pass@64": 0.7396000199751166,
885
+ "pass@128": 0.7655867970660146
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.9458333333333333,
890
+ "pass@2": 0.9680735220597424,
891
+ "pass@4": 0.9784683795477946,
892
+ "pass@8": 0.9834253742459571,
893
+ "pass@16": 0.9859158447021265,
894
+ "pass@32": 0.9876978205871854,
895
+ "pass@64": 0.9894997112731746,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8640577217125383,
900
+ "pass@2": 0.9053041248284324,
901
+ "pass@4": 0.9285084223990344,
902
+ "pass@8": 0.9444538698747363,
903
+ "pass@16": 0.9565054855884125,
904
+ "pass@32": 0.9649179472227764,
905
+ "pass@64": 0.971280941814739,
906
+ "pass@128": 0.9785932721712538
907
+ },
908
+ "13": {
909
+ "pass@1": 0.7043306327160493,
910
+ "pass@2": 0.7786207713619127,
911
+ "pass@4": 0.8288944483791375,
912
+ "pass@8": 0.86676143621755,
913
+ "pass@16": 0.8943791026870926,
914
+ "pass@32": 0.9153670080570203,
915
+ "pass@64": 0.9331493513324162,
916
+ "pass@128": 0.9475308641975309
917
+ },
918
+ "14": {
919
+ "pass@1": 0.6179293397626113,
920
+ "pass@2": 0.703675538563985,
921
+ "pass@4": 0.7640525688183634,
922
+ "pass@8": 0.8070382454357383,
923
+ "pass@16": 0.8387920432900056,
924
+ "pass@32": 0.8638308703360127,
925
+ "pass@64": 0.8855514513931839,
926
+ "pass@128": 0.9050445103857567
927
+ },
928
+ "15": {
929
+ "pass@1": 0.46923225308641975,
930
+ "pass@2": 0.5619528136239913,
931
+ "pass@4": 0.6342046055007012,
932
+ "pass@8": 0.6883368569053809,
933
+ "pass@16": 0.7319691872044063,
934
+ "pass@32": 0.7717383879708698,
935
+ "pass@64": 0.8095051745441575,
936
+ "pass@128": 0.8425925925925926
937
+ },
938
+ "16": {
939
+ "pass@1": 0.31752396755162243,
940
+ "pass@2": 0.3971402254198313,
941
+ "pass@4": 0.46753584065266157,
942
+ "pass@8": 0.5274963190047928,
943
+ "pass@16": 0.578633799879253,
944
+ "pass@32": 0.6236236229841198,
945
+ "pass@64": 0.6655990132285015,
946
+ "pass@128": 0.7079646017699115
947
+ },
948
+ "17": {
949
+ "pass@1": 0.2409403237951807,
950
+ "pass@2": 0.3118437084953988,
951
+ "pass@4": 0.38310006194632296,
952
+ "pass@8": 0.4517112008785566,
953
+ "pass@16": 0.5143629770086949,
954
+ "pass@32": 0.5692526943598468,
955
+ "pass@64": 0.6171244785989948,
956
+ "pass@128": 0.6566265060240963
957
+ },
958
+ "18": {
959
+ "pass@1": 0.21854838709677418,
960
+ "pass@2": 0.2914830454660909,
961
+ "pass@4": 0.36253027202244875,
962
+ "pass@8": 0.4232167175836581,
963
+ "pass@16": 0.4722504664411351,
964
+ "pass@32": 0.5141885209927322,
965
+ "pass@64": 0.5525587893884821,
966
+ "pass@128": 0.5870967741935483
967
+ },
968
+ "19": {
969
+ "pass@1": 0.1644873489425982,
970
+ "pass@2": 0.22967898815329338,
971
+ "pass@4": 0.296141610225459,
972
+ "pass@8": 0.35542356027993677,
973
+ "pass@16": 0.40438333117475245,
974
+ "pass@32": 0.44624875815883597,
975
+ "pass@64": 0.4841766947910102,
976
+ "pass@128": 0.5196374622356495
977
+ },
978
+ "20": {
979
+ "pass@1": 0.16366366366366367,
980
+ "pass@2": 0.22465292556335845,
981
+ "pass@4": 0.28830474653506144,
982
+ "pass@8": 0.3515397113540908,
983
+ "pass@16": 0.4098242301095972,
984
+ "pass@32": 0.4585545918829372,
985
+ "pass@64": 0.49711705635141545,
986
+ "pass@128": 0.5285285285285285
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.7163149671052632,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.7163149671052632,
1002
+ "pass@2": 0.7523812033775175,
1003
+ "pass@4": 0.7827041436464997,
1004
+ "pass@8": 0.8079569912329014,
1005
+ "pass@16": 0.8286190437385256,
1006
+ "pass@32": 0.8458090712996409,
1007
+ "pass@64": 0.8608293054420325,
1008
+ "pass@128": 0.874421052631579
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.954203125,
1013
+ "pass@2": 0.9668100393700787,
1014
+ "pass@4": 0.9751169003562058,
1015
+ "pass@8": 0.9805623138162619,
1016
+ "pass@16": 0.9841740075635769,
1017
+ "pass@32": 0.9868633714733398,
1018
+ "pass@64": 0.9886806402008643,
1019
+ "pass@128": 0.99
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 1.0,
1023
+ "pass@2": 1.0,
1024
+ "pass@4": 1.0,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.9999921875,
1033
+ "pass@2": 1.0,
1034
+ "pass@4": 1.0,
1035
+ "pass@8": 1.0,
1036
+ "pass@16": 1.0,
1037
+ "pass@32": 1.0,
1038
+ "pass@64": 1.0,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9990234375,
1043
+ "pass@2": 0.999410063976378,
1044
+ "pass@4": 0.9996751137982751,
1045
+ "pass@8": 0.9998989158969318,
1046
+ "pass@16": 0.9999915037321406,
1047
+ "pass@32": 0.9999999700068081,
1048
+ "pass@64": 0.9999999999999963,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.9949921875,
1053
+ "pass@2": 0.9967944143700785,
1054
+ "pass@4": 0.9977303270528682,
1055
+ "pass@8": 0.9982286507773027,
1056
+ "pass@16": 0.9986409291833077,
1057
+ "pass@32": 0.999127515746347,
1058
+ "pass@64": 0.9996924094487892,
1059
+ "pass@128": 1.0
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9986875,
1063
+ "pass@2": 0.9990652066929134,
1064
+ "pass@4": 0.9992026385451819,
1065
+ "pass@8": 0.99937067318584,
1066
+ "pass@16": 0.9996167406956445,
1067
+ "pass@32": 0.9998739053984705,
1068
+ "pass@64": 0.9999934280775878,
1069
+ "pass@128": 1.0
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9977421875,
1073
+ "pass@2": 0.9986300442913385,
1074
+ "pass@4": 0.9992237436257967,
1075
+ "pass@8": 0.999713898666317,
1076
+ "pass@16": 0.9999638315713223,
1077
+ "pass@32": 0.9999996347541911,
1078
+ "pass@64": 0.9999999999983727,
1079
+ "pass@128": 1.0
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.9968125,
1083
+ "pass@2": 0.998474655511811,
1084
+ "pass@4": 0.9989533392388451,
1085
+ "pass@8": 0.9989996318701388,
1086
+ "pass@16": 0.9989999999891657,
1087
+ "pass@32": 0.999,
1088
+ "pass@64": 0.999,
1089
+ "pass@128": 0.999
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.994109375,
1093
+ "pass@2": 0.9955601624015749,
1094
+ "pass@4": 0.9961808598612675,
1095
+ "pass@8": 0.9967566254314778,
1096
+ "pass@16": 0.9973485536370097,
1097
+ "pass@32": 0.9979841908426146,
1098
+ "pass@64": 0.9986296156860006,
1099
+ "pass@128": 0.999
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.9424921875,
1103
+ "pass@2": 0.9608925935039377,
1104
+ "pass@4": 0.9707150841769778,
1105
+ "pass@8": 0.9776590164210477,
1106
+ "pass@16": 0.983031112952659,
1107
+ "pass@32": 0.9868108166417683,
1108
+ "pass@64": 0.9895059512740915,
1109
+ "pass@128": 0.992
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.850125,
1113
+ "pass@2": 0.8868829970472448,
1114
+ "pass@4": 0.9069662009748772,
1115
+ "pass@8": 0.9210538138715969,
1116
+ "pass@16": 0.932209399638482,
1117
+ "pass@32": 0.941670117173944,
1118
+ "pass@64": 0.9509294871368753,
1119
+ "pass@128": 0.961
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.723671875,
1123
+ "pass@2": 0.7991273375984257,
1124
+ "pass@4": 0.848705464191976,
1125
+ "pass@8": 0.8832906602998529,
1126
+ "pass@16": 0.907676298226515,
1127
+ "pass@32": 0.9258390025587503,
1128
+ "pass@64": 0.9408623838730613,
1129
+ "pass@128": 0.954
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.621359375,
1133
+ "pass@2": 0.7092471702755899,
1134
+ "pass@4": 0.7710694283839518,
1135
+ "pass@8": 0.8143357774146883,
1136
+ "pass@16": 0.8459435216064888,
1137
+ "pass@32": 0.8706776031668482,
1138
+ "pass@64": 0.8916396601771912,
1139
+ "pass@128": 0.911
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.4664140625,
1143
+ "pass@2": 0.5631503444881897,
1144
+ "pass@4": 0.6409323717660281,
1145
+ "pass@8": 0.700385082200824,
1146
+ "pass@16": 0.7464661637312656,
1147
+ "pass@32": 0.7838249784646978,
1148
+ "pass@64": 0.8150213816108757,
1149
+ "pass@128": 0.842
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.300421875,
1153
+ "pass@2": 0.38325036909448806,
1154
+ "pass@4": 0.45934014332583345,
1155
+ "pass@8": 0.5256147490839751,
1156
+ "pass@16": 0.5812667481374817,
1157
+ "pass@32": 0.6290118259853492,
1158
+ "pass@64": 0.6714562961595262,
1159
+ "pass@128": 0.71
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.2392421875,
1163
+ "pass@2": 0.3153343996062992,
1164
+ "pass@4": 0.390559457255343,
1165
+ "pass@8": 0.45835220794408055,
1166
+ "pass@16": 0.515216747490821,
1167
+ "pass@32": 0.5633337200265438,
1168
+ "pass@64": 0.6057651819056846,
1169
+ "pass@128": 0.644
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.20425,
1173
+ "pass@2": 0.27352423720472446,
1174
+ "pass@4": 0.3419236596362953,
1175
+ "pass@8": 0.4029165957544186,
1176
+ "pass@16": 0.45499798232417954,
1177
+ "pass@32": 0.5004840348220753,
1178
+ "pass@64": 0.5415533976453777,
1179
+ "pass@128": 0.578
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.169625,
1183
+ "pass@2": 0.23175504429133828,
1184
+ "pass@4": 0.29506946662917083,
1185
+ "pass@8": 0.353428445257125,
1186
+ "pass@16": 0.40308684493744273,
1187
+ "pass@32": 0.4453773587288245,
1188
+ "pass@64": 0.4839624351859198,
1189
+ "pass@128": 0.52
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.1568203125,
1193
+ "pass@2": 0.217333784448819,
1194
+ "pass@4": 0.2800145304649416,
1195
+ "pass@8": 0.3406157755330686,
1196
+ "pass@16": 0.39513144561425667,
1197
+ "pass@32": 0.4404943089023678,
1198
+ "pass@64": 0.47906453501838553,
1199
+ "pass@128": 0.514
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.954203125,
1204
+ "2": 1.0,
1205
+ "3": 0.9999921875,
1206
+ "4": 0.9990234375,
1207
+ "5": 0.9949921875,
1208
+ "6": 0.9986875,
1209
+ "7": 0.9977421875,
1210
+ "8": 0.9968125,
1211
+ "9": 0.994109375,
1212
+ "11": 0.9424921875,
1213
+ "12": 0.850125,
1214
+ "13": 0.723671875,
1215
+ "14": 0.621359375,
1216
+ "15": 0.4664140625,
1217
+ "16": 0.300421875,
1218
+ "17": 0.2392421875,
1219
+ "18": 0.20425,
1220
+ "19": 0.169625,
1221
+ "20": 0.1568203125
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 280.9476015625,
1225
+ "2": 115.7252734375,
1226
+ "3": 144.8482265625,
1227
+ "4": 160.08678125,
1228
+ "5": 173.194703125,
1229
+ "6": 199.14746875,
1230
+ "7": 224.20671875,
1231
+ "8": 243.602875,
1232
+ "9": 259.4781875,
1233
+ "11": 298.1274375,
1234
+ "12": 307.074171875,
1235
+ "13": 322.5293828125,
1236
+ "14": 340.953328125,
1237
+ "15": 345.885015625,
1238
+ "16": 332.6650234375,
1239
+ "17": 347.350203125,
1240
+ "18": 347.9971171875,
1241
+ "19": 336.1864375,
1242
+ "20": 347.200953125
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 280.9476015625,
1247
+ "2": 115.7252734375,
1248
+ "3": 144.8482265625,
1249
+ "4": 160.08678125,
1250
+ "5": 173.194703125,
1251
+ "6": 199.14746875,
1252
+ "7": 224.20671875,
1253
+ "8": 243.602875,
1254
+ "9": 259.4781875,
1255
+ "11": 298.1274375,
1256
+ "12": 307.074171875,
1257
+ "13": 322.5293828125,
1258
+ "14": 340.953328125,
1259
+ "15": 345.885015625,
1260
+ "16": 332.6650234375,
1261
+ "17": 347.350203125,
1262
+ "18": 347.9971171875,
1263
+ "19": 336.1864375,
1264
+ "20": 347.200953125
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 576926,
1271
+ "answer_accuracy": 0.7124935780904205,
1272
+ "avg_response_len": 257.13824642349033,
1273
+ "resp_tokens_sum": 208212038,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.7124935780904205,
1276
+ "pass@2": 0.7482208665462601,
1277
+ "pass@4": 0.7782977803705171,
1278
+ "pass@8": 0.8033142972254913,
1279
+ "pass@16": 0.8239431045015444,
1280
+ "pass@32": 0.841313857869306,
1281
+ "pass@64": 0.8566171919665101,
1282
+ "pass@128": 0.8706923806512804
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9639564896755162,
1287
+ "pass@2": 0.9751933663159359,
1288
+ "pass@4": 0.9805886808396739,
1289
+ "pass@8": 0.9832729426842688,
1290
+ "pass@16": 0.9853094435850058,
1291
+ "pass@32": 0.9875159779599025,
1292
+ "pass@64": 0.9903780082743677,
1293
+ "pass@128": 0.9941002949852508
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 1.0,
1297
+ "pass@2": 1.0,
1298
+ "pass@4": 1.0,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9999761085626911,
1307
+ "pass@2": 1.0,
1308
+ "pass@4": 1.0,
1309
+ "pass@8": 1.0,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.997314453125,
1317
+ "pass@2": 0.9981799028051181,
1318
+ "pass@4": 0.9989848275801462,
1319
+ "pass@8": 0.9996841121782726,
1320
+ "pass@16": 0.9999734491629393,
1321
+ "pass@32": 0.9999999062712751,
1322
+ "pass@64": 0.9999999999999887,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9964285714285714,
1327
+ "pass@2": 0.9987411729783777,
1328
+ "pass@4": 0.9997791192767571,
1329
+ "pass@8": 0.9999910052051663,
1330
+ "pass@16": 0.999999987524699,
1331
+ "pass@32": 0.999999999999998,
1332
+ "pass@64": 1.0,
1333
+ "pass@128": 1.0
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9970761601796407,
1337
+ "pass@2": 0.9973235124239709,
1338
+ "pass@4": 0.9976157328200742,
1339
+ "pass@8": 0.9981157895218378,
1340
+ "pass@16": 0.998852517052828,
1341
+ "pass@32": 0.9996224712529058,
1342
+ "pass@64": 0.9999803235855924,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9972725591715976,
1347
+ "pass@2": 0.9980547919675722,
1348
+ "pass@4": 0.9987735604794961,
1349
+ "pass@8": 0.9995079830474547,
1350
+ "pass@16": 0.999928825622786,
1351
+ "pass@32": 0.9999991140832827,
1352
+ "pass@64": 0.9999999999952738,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9991486378205128,
1357
+ "pass@2": 0.9999641158136482,
1358
+ "pass@4": 0.999999888836011,
1359
+ "pass@8": 0.9999999999996302,
1360
+ "pass@16": 1.0,
1361
+ "pass@32": 1.0,
1362
+ "pass@64": 1.0,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9955325704225352,
1367
+ "pass@2": 0.9969474326272597,
1368
+ "pass@4": 0.9973978846834287,
1369
+ "pass@8": 0.9976821039095084,
1370
+ "pass@16": 0.998119293157022,
1371
+ "pass@32": 0.9988210100498001,
1372
+ "pass@64": 0.9996562049462128,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9345122167138811,
1377
+ "pass@2": 0.9542368143695208,
1378
+ "pass@4": 0.9657462063877991,
1379
+ "pass@8": 0.9749512748470323,
1380
+ "pass@16": 0.9827357531203197,
1381
+ "pass@32": 0.9884344587547554,
1382
+ "pass@64": 0.9925691216469926,
1383
+ "pass@128": 0.9971671388101983
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8454493290960452,
1387
+ "pass@2": 0.8801169285777846,
1388
+ "pass@4": 0.8989873583069065,
1389
+ "pass@8": 0.9130351649779792,
1390
+ "pass@16": 0.9250958046726186,
1391
+ "pass@32": 0.9364960935539219,
1392
+ "pass@64": 0.9474650567648585,
1393
+ "pass@128": 0.9576271186440678
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.74047315830721,
1397
+ "pass@2": 0.8119554217164869,
1398
+ "pass@4": 0.8599147251969675,
1399
+ "pass@8": 0.8945134207641785,
1400
+ "pass@16": 0.9205661933194598,
1401
+ "pass@32": 0.9404403427759488,
1402
+ "pass@64": 0.9559823521026871,
1403
+ "pass@128": 0.9686520376175548
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.6125632788161994,
1407
+ "pass@2": 0.7013274370201388,
1408
+ "pass@4": 0.7658476230190846,
1409
+ "pass@8": 0.8097702298192307,
1410
+ "pass@16": 0.8407102562368179,
1411
+ "pass@32": 0.8644964958281782,
1412
+ "pass@64": 0.8838021812976606,
1413
+ "pass@128": 0.9003115264797508
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.48457821531791906,
1417
+ "pass@2": 0.5827305027081152,
1418
+ "pass@4": 0.6590647869702699,
1419
+ "pass@8": 0.7158310076759379,
1420
+ "pass@16": 0.7589664128271321,
1421
+ "pass@32": 0.7930825505983045,
1422
+ "pass@64": 0.8204516163076292,
1423
+ "pass@128": 0.8410404624277457
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.2755040322580645,
1427
+ "pass@2": 0.3586213328676657,
1428
+ "pass@4": 0.43517604109970115,
1429
+ "pass@8": 0.5017510566263869,
1430
+ "pass@16": 0.5583572473734497,
1431
+ "pass@32": 0.6078831915021241,
1432
+ "pass@64": 0.6517453480905581,
1433
+ "pass@128": 0.6903225806451613
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.21780166184971098,
1437
+ "pass@2": 0.29274348857584986,
1438
+ "pass@4": 0.36758594012453627,
1439
+ "pass@8": 0.4327260544001389,
1440
+ "pass@16": 0.4852625887317101,
1441
+ "pass@32": 0.5308733536032256,
1442
+ "pass@64": 0.5731955827266514,
1443
+ "pass@128": 0.615606936416185
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.18082589285714284,
1447
+ "pass@2": 0.24475007030371193,
1448
+ "pass@4": 0.30927755396646855,
1449
+ "pass@8": 0.3685482161042458,
1450
+ "pass@16": 0.4204588630253465,
1451
+ "pass@32": 0.46585847061159913,
1452
+ "pass@64": 0.5068792519157992,
1453
+ "pass@128": 0.5457142857142857
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.1753472222222222,
1457
+ "pass@2": 0.23986560379276908,
1458
+ "pass@4": 0.3042127763308865,
1459
+ "pass@8": 0.3648899684541569,
1460
+ "pass@16": 0.41793983675736257,
1461
+ "pass@32": 0.46237397780199607,
1462
+ "pass@64": 0.5029501083188042,
1463
+ "pass@128": 0.5405405405405406
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.15636730480480482,
1467
+ "pass@2": 0.2165424528859568,
1468
+ "pass@4": 0.27821547475484476,
1469
+ "pass@8": 0.3365076147164066,
1470
+ "pass@16": 0.3888313889531125,
1471
+ "pass@32": 0.4337362063462613,
1472
+ "pass@64": 0.4738117632886543,
1473
+ "pass@128": 0.5135135135135135
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 589293,
1480
+ "answer_accuracy": 0.71766976812159,
1481
+ "avg_response_len": 266.5054985872954,
1482
+ "resp_tokens_sum": 218832995,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.71766976812159,
1485
+ "pass@2": 0.7541205482352454,
1486
+ "pass@4": 0.7849935205585589,
1487
+ "pass@8": 0.8106964627292033,
1488
+ "pass@16": 0.831317315567593,
1489
+ "pass@32": 0.8479933921096046,
1490
+ "pass@64": 0.8623722337450352,
1491
+ "pass@128": 0.875448168355417
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9412384969325154,
1496
+ "pass@2": 0.9570337030819772,
1497
+ "pass@4": 0.9684386097635029,
1498
+ "pass@8": 0.9755421990555402,
1499
+ "pass@16": 0.9790090135040668,
1500
+ "pass@32": 0.9808182150392867,
1501
+ "pass@64": 0.9815547734367644,
1502
+ "pass@128": 0.9815950920245399
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 1.0,
1506
+ "pass@2": 1.0,
1507
+ "pass@4": 1.0,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 1.0,
1516
+ "pass@2": 1.0,
1517
+ "pass@4": 1.0,
1518
+ "pass@8": 1.0,
1519
+ "pass@16": 1.0,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9999075443786982,
1526
+ "pass@2": 0.9999978160089457,
1527
+ "pass@4": 0.9999999997226678,
1528
+ "pass@8": 1.0,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9963768115942029,
1536
+ "pass@2": 0.9971339010612804,
1537
+ "pass@4": 0.9974446123582379,
1538
+ "pass@8": 0.9977680501821345,
1539
+ "pass@16": 0.9983124881128967,
1540
+ "pass@32": 0.999097406302473,
1541
+ "pass@64": 0.9998273650576287,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9997313753581661,
1546
+ "pass@2": 0.9999841363400491,
1547
+ "pass@4": 0.9999999435958757,
1548
+ "pass@8": 0.9999999999999097,
1549
+ "pass@16": 1.0,
1550
+ "pass@32": 1.0,
1551
+ "pass@64": 1.0,
1552
+ "pass@128": 1.0
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.997674313583815,
1556
+ "pass@2": 0.998275069978608,
1557
+ "pass@4": 0.9989801556597332,
1558
+ "pass@8": 0.9996539139236084,
1559
+ "pass@16": 0.9999649956979594,
1560
+ "pass@32": 0.9999998098093686,
1561
+ "pass@64": 0.9999999999999141,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9957306338028169,
1566
+ "pass@2": 0.9968081124542529,
1567
+ "pass@4": 0.9971466248937193,
1568
+ "pass@8": 0.9971827493473799,
1569
+ "pass@16": 0.9971830985789736,
1570
+ "pass@32": 0.9971830985915493,
1571
+ "pass@64": 0.9971830985915493,
1572
+ "pass@128": 0.9971830985915493
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.993310254491018,
1576
+ "pass@2": 0.9940558106228489,
1577
+ "pass@4": 0.994196379457059,
1578
+ "pass@8": 0.9943759135003243,
1579
+ "pass@16": 0.9947162761091989,
1580
+ "pass@32": 0.995326276580697,
1581
+ "pass@64": 0.9962633787543024,
1582
+ "pass@128": 0.9970059880239521
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9478068524096386,
1586
+ "pass@2": 0.9611561403092685,
1587
+ "pass@4": 0.9686419691702693,
1588
+ "pass@8": 0.9750669382908692,
1589
+ "pass@16": 0.9806081356024587,
1590
+ "pass@32": 0.9842428892059527,
1591
+ "pass@64": 0.9862549466314855,
1592
+ "pass@128": 0.9879518072289156
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8410315438871473,
1596
+ "pass@2": 0.8755083244884359,
1597
+ "pass@4": 0.8937380000305601,
1598
+ "pass@8": 0.905965360565369,
1599
+ "pass@16": 0.9151980908995769,
1600
+ "pass@32": 0.9235809759059825,
1601
+ "pass@64": 0.9339121914379174,
1602
+ "pass@128": 0.9467084639498433
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.7262123599439776,
1606
+ "pass@2": 0.8062757091025381,
1607
+ "pass@4": 0.8566690912585506,
1608
+ "pass@8": 0.8882637919932588,
1609
+ "pass@16": 0.9082264797954909,
1610
+ "pass@32": 0.9222958616323473,
1611
+ "pass@64": 0.9343518871725501,
1612
+ "pass@128": 0.9467787114845938
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.6329952485380117,
1616
+ "pass@2": 0.7221707792282543,
1617
+ "pass@4": 0.7828848704767167,
1618
+ "pass@8": 0.8258118272832482,
1619
+ "pass@16": 0.8579023706600551,
1620
+ "pass@32": 0.8832258032244631,
1621
+ "pass@64": 0.9049950902372479,
1622
+ "pass@128": 0.9269005847953217
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.4446022727272727,
1626
+ "pass@2": 0.5437966028394173,
1627
+ "pass@4": 0.6285262524002679,
1628
+ "pass@8": 0.6960194300230498,
1629
+ "pass@16": 0.7475932370874276,
1630
+ "pass@32": 0.785985388651888,
1631
+ "pass@64": 0.8147437752003887,
1632
+ "pass@128": 0.8424242424242424
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.3059116809116809,
1636
+ "pass@2": 0.39158751990936996,
1637
+ "pass@4": 0.4727661555660242,
1638
+ "pass@8": 0.5448737019577484,
1639
+ "pass@16": 0.6040431432838912,
1640
+ "pass@32": 0.6528764342680178,
1641
+ "pass@64": 0.694521859735016,
1642
+ "pass@128": 0.7293447293447294
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.26052989130434784,
1646
+ "pass@2": 0.34320820291485293,
1647
+ "pass@4": 0.42293633977041684,
1648
+ "pass@8": 0.4927356348756277,
1649
+ "pass@16": 0.5482837994495754,
1650
+ "pass@32": 0.5921106992309274,
1651
+ "pass@64": 0.6290502589670679,
1652
+ "pass@128": 0.6614906832298136
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.2153262867647059,
1656
+ "pass@2": 0.28677049559981455,
1657
+ "pass@4": 0.35674156300315396,
1658
+ "pass@8": 0.4197868754911729,
1659
+ "pass@16": 0.47482275196634255,
1660
+ "pass@32": 0.5236327311772606,
1661
+ "pass@64": 0.5672130434247589,
1662
+ "pass@128": 0.6029411764705882
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.16901506696428573,
1666
+ "pass@2": 0.22576206294525675,
1667
+ "pass@4": 0.28495160454273577,
1668
+ "pass@8": 0.34010383127747634,
1669
+ "pass@16": 0.38708927582856617,
1670
+ "pass@32": 0.4276740630065036,
1671
+ "pass@64": 0.4649332236307543,
1672
+ "pass@128": 0.5
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.1504491017964072,
1676
+ "pass@2": 0.2108255192371163,
1677
+ "pass@4": 0.2735428047167756,
1678
+ "pass@8": 0.3338204070406974,
1679
+ "pass@16": 0.38676384570770866,
1680
+ "pass@32": 0.4292259673414519,
1681
+ "pass@64": 0.4663031080186876,
1682
+ "pass@128": 0.5
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 575859,
1689
+ "answer_accuracy": 0.7187886942802365,
1690
+ "avg_response_len": 286.13477966727913,
1691
+ "resp_tokens_sum": 229237451,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.7187886942802365,
1694
+ "pass@2": 0.7548033784106732,
1695
+ "pass@4": 0.7848112376220922,
1696
+ "pass@8": 0.8098416329714997,
1697
+ "pass@16": 0.8305795131153559,
1698
+ "pass@32": 0.8481136410654446,
1699
+ "pass@64": 0.8635051234293231,
1700
+ "pass@128": 0.8771369228311232
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9569496268656716,
1705
+ "pass@2": 0.9678403014455282,
1706
+ "pass@4": 0.976078659010907,
1707
+ "pass@8": 0.9827045652363846,
1708
+ "pass@16": 0.9880512471221918,
1709
+ "pass@32": 0.9920857219167919,
1710
+ "pass@64": 0.9938974306133376,
1711
+ "pass@128": 0.9940298507462687
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 1.0,
1715
+ "pass@2": 1.0,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 1.0,
1725
+ "pass@2": 1.0,
1726
+ "pass@4": 1.0,
1727
+ "pass@8": 1.0,
1728
+ "pass@16": 1.0,
1729
+ "pass@32": 1.0,
1730
+ "pass@64": 1.0,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9997487207602339,
1735
+ "pass@2": 0.9999802142330892,
1736
+ "pass@4": 0.9999999095507799,
1737
+ "pass@8": 0.9999999999996626,
1738
+ "pass@16": 1.0,
1739
+ "pass@32": 1.0,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9922564338235295,
1745
+ "pass@2": 0.994646320634553,
1746
+ "pass@4": 0.9961220976973466,
1747
+ "pass@8": 0.9970632553671733,
1748
+ "pass@16": 0.997715072688465,
1749
+ "pass@32": 0.9983497369764539,
1750
+ "pass@64": 0.9992704955997272,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9992360015772871,
1755
+ "pass@2": 0.9998886118631859,
1756
+ "pass@4": 0.9999968563669289,
1757
+ "pass@8": 0.9999999983772165,
1758
+ "pass@16": 1.0,
1759
+ "pass@32": 1.0,
1760
+ "pass@64": 1.0,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.998318829113924,
1765
+ "pass@2": 0.9996340202332306,
1766
+ "pass@4": 0.9999719820425929,
1767
+ "pass@8": 0.9999998296794577,
1768
+ "pass@16": 0.999999999997869,
1769
+ "pass@32": 1.0,
1770
+ "pass@64": 1.0,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.995777027027027,
1775
+ "pass@2": 0.9988557701402191,
1776
+ "pass@4": 0.9998988651193375,
1777
+ "pass@8": 0.9999992668226257,
1778
+ "pass@16": 0.9999999999808709,
1779
+ "pass@32": 1.0,
1780
+ "pass@64": 1.0,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9933430466237942,
1785
+ "pass@2": 0.9955922285236853,
1786
+ "pass@4": 0.996922894083577,
1787
+ "pass@8": 0.99825698853535,
1788
+ "pass@16": 0.9992957181536803,
1789
+ "pass@32": 0.9998834594758289,
1790
+ "pass@64": 0.9999990174474539,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.9458333333333333,
1795
+ "pass@2": 0.9680735220597424,
1796
+ "pass@4": 0.9784683795477946,
1797
+ "pass@8": 0.9834253742459571,
1798
+ "pass@16": 0.9859158447021265,
1799
+ "pass@32": 0.9876978205871854,
1800
+ "pass@64": 0.9894997112731746,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8640577217125383,
1805
+ "pass@2": 0.9053041248284324,
1806
+ "pass@4": 0.9285084223990344,
1807
+ "pass@8": 0.9444538698747363,
1808
+ "pass@16": 0.9565054855884125,
1809
+ "pass@32": 0.9649179472227764,
1810
+ "pass@64": 0.971280941814739,
1811
+ "pass@128": 0.9785932721712538
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.7043306327160493,
1815
+ "pass@2": 0.7786207713619127,
1816
+ "pass@4": 0.8288944483791375,
1817
+ "pass@8": 0.86676143621755,
1818
+ "pass@16": 0.8943791026870926,
1819
+ "pass@32": 0.9153670080570203,
1820
+ "pass@64": 0.9331493513324162,
1821
+ "pass@128": 0.9475308641975309
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.6179293397626113,
1825
+ "pass@2": 0.703675538563985,
1826
+ "pass@4": 0.7640525688183634,
1827
+ "pass@8": 0.8070382454357383,
1828
+ "pass@16": 0.8387920432900056,
1829
+ "pass@32": 0.8638308703360127,
1830
+ "pass@64": 0.8855514513931839,
1831
+ "pass@128": 0.9050445103857567
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.46923225308641975,
1835
+ "pass@2": 0.5619528136239913,
1836
+ "pass@4": 0.6342046055007012,
1837
+ "pass@8": 0.6883368569053809,
1838
+ "pass@16": 0.7319691872044063,
1839
+ "pass@32": 0.7717383879708698,
1840
+ "pass@64": 0.8095051745441575,
1841
+ "pass@128": 0.8425925925925926
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.31752396755162243,
1845
+ "pass@2": 0.3971402254198313,
1846
+ "pass@4": 0.46753584065266157,
1847
+ "pass@8": 0.5274963190047928,
1848
+ "pass@16": 0.578633799879253,
1849
+ "pass@32": 0.6236236229841198,
1850
+ "pass@64": 0.6655990132285015,
1851
+ "pass@128": 0.7079646017699115
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.2409403237951807,
1855
+ "pass@2": 0.3118437084953988,
1856
+ "pass@4": 0.38310006194632296,
1857
+ "pass@8": 0.4517112008785566,
1858
+ "pass@16": 0.5143629770086949,
1859
+ "pass@32": 0.5692526943598468,
1860
+ "pass@64": 0.6171244785989948,
1861
+ "pass@128": 0.6566265060240963
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.21854838709677418,
1865
+ "pass@2": 0.2914830454660909,
1866
+ "pass@4": 0.36253027202244875,
1867
+ "pass@8": 0.4232167175836581,
1868
+ "pass@16": 0.4722504664411351,
1869
+ "pass@32": 0.5141885209927322,
1870
+ "pass@64": 0.5525587893884821,
1871
+ "pass@128": 0.5870967741935483
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.1644873489425982,
1875
+ "pass@2": 0.22967898815329338,
1876
+ "pass@4": 0.296141610225459,
1877
+ "pass@8": 0.35542356027993677,
1878
+ "pass@16": 0.40438333117475245,
1879
+ "pass@32": 0.44624875815883597,
1880
+ "pass@64": 0.4841766947910102,
1881
+ "pass@128": 0.5196374622356495
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.16366366366366367,
1885
+ "pass@2": 0.22465292556335845,
1886
+ "pass@4": 0.28830474653506144,
1887
+ "pass@8": 0.3515397113540908,
1888
+ "pass@16": 0.4098242301095972,
1889
+ "pass@32": 0.4585545918829372,
1890
+ "pass@64": 0.49711705635141545,
1891
+ "pass@128": 0.5285285285285285
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch4-op7-10/SDAR-100M-block16-epoch4-op7-10_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72b45f86223b6ed3a620ca6ab351bf510533278b22dd9c0708a9628915097190
3
+ size 2587735856
results/composition/SDAR-100M-block16-epoch4-op7-10/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch4-op7-10,0.9928402777777777,0.4674421875,0.7163149671052632,0.0,0.0,0.0,200.1375373263889,332.59690703125,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch4-op7-10/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch4-op7-10",
4
+ "id_acc": 0.9928402777777777,
5
+ "ood_acc": 0.4674421875,
6
+ "total_acc": 0.7163149671052632,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 200.1375373263889,
11
+ "ood_avg_resp_len": 332.59690703125,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 576926,
19
+ "answer_accuracy": 0.7124935780904205,
20
+ "avg_response_len": 257.13824642349033,
21
+ "resp_tokens_sum": 208212038,
22
+ "pass_at_k": {
23
+ "pass@1": 0.7124935780904205,
24
+ "pass@2": 0.7482208665462601,
25
+ "pass@4": 0.7782977803705171,
26
+ "pass@8": 0.8033142972254913,
27
+ "pass@16": 0.8239431045015444,
28
+ "pass@32": 0.841313857869306,
29
+ "pass@64": 0.8566171919665101,
30
+ "pass@128": 0.8706923806512804
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9639564896755162,
35
+ "pass@2": 0.9751933663159359,
36
+ "pass@4": 0.9805886808396739,
37
+ "pass@8": 0.9832729426842688,
38
+ "pass@16": 0.9853094435850058,
39
+ "pass@32": 0.9875159779599025,
40
+ "pass@64": 0.9903780082743677,
41
+ "pass@128": 0.9941002949852508
42
+ },
43
+ "2": {
44
+ "pass@1": 1.0,
45
+ "pass@2": 1.0,
46
+ "pass@4": 1.0,
47
+ "pass@8": 1.0,
48
+ "pass@16": 1.0,
49
+ "pass@32": 1.0,
50
+ "pass@64": 1.0,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 0.9999761085626911,
55
+ "pass@2": 1.0,
56
+ "pass@4": 1.0,
57
+ "pass@8": 1.0,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.997314453125,
65
+ "pass@2": 0.9981799028051181,
66
+ "pass@4": 0.9989848275801462,
67
+ "pass@8": 0.9996841121782726,
68
+ "pass@16": 0.9999734491629393,
69
+ "pass@32": 0.9999999062712751,
70
+ "pass@64": 0.9999999999999887,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9964285714285714,
75
+ "pass@2": 0.9987411729783777,
76
+ "pass@4": 0.9997791192767571,
77
+ "pass@8": 0.9999910052051663,
78
+ "pass@16": 0.999999987524699,
79
+ "pass@32": 0.999999999999998,
80
+ "pass@64": 1.0,
81
+ "pass@128": 1.0
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9970761601796407,
85
+ "pass@2": 0.9973235124239709,
86
+ "pass@4": 0.9976157328200742,
87
+ "pass@8": 0.9981157895218378,
88
+ "pass@16": 0.998852517052828,
89
+ "pass@32": 0.9996224712529058,
90
+ "pass@64": 0.9999803235855924,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9972725591715976,
95
+ "pass@2": 0.9980547919675722,
96
+ "pass@4": 0.9987735604794961,
97
+ "pass@8": 0.9995079830474547,
98
+ "pass@16": 0.999928825622786,
99
+ "pass@32": 0.9999991140832827,
100
+ "pass@64": 0.9999999999952738,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9991486378205128,
105
+ "pass@2": 0.9999641158136482,
106
+ "pass@4": 0.999999888836011,
107
+ "pass@8": 0.9999999999996302,
108
+ "pass@16": 1.0,
109
+ "pass@32": 1.0,
110
+ "pass@64": 1.0,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9955325704225352,
115
+ "pass@2": 0.9969474326272597,
116
+ "pass@4": 0.9973978846834287,
117
+ "pass@8": 0.9976821039095084,
118
+ "pass@16": 0.998119293157022,
119
+ "pass@32": 0.9988210100498001,
120
+ "pass@64": 0.9996562049462128,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.9345122167138811,
125
+ "pass@2": 0.9542368143695208,
126
+ "pass@4": 0.9657462063877991,
127
+ "pass@8": 0.9749512748470323,
128
+ "pass@16": 0.9827357531203197,
129
+ "pass@32": 0.9884344587547554,
130
+ "pass@64": 0.9925691216469926,
131
+ "pass@128": 0.9971671388101983
132
+ },
133
+ "12": {
134
+ "pass@1": 0.8454493290960452,
135
+ "pass@2": 0.8801169285777846,
136
+ "pass@4": 0.8989873583069065,
137
+ "pass@8": 0.9130351649779792,
138
+ "pass@16": 0.9250958046726186,
139
+ "pass@32": 0.9364960935539219,
140
+ "pass@64": 0.9474650567648585,
141
+ "pass@128": 0.9576271186440678
142
+ },
143
+ "13": {
144
+ "pass@1": 0.74047315830721,
145
+ "pass@2": 0.8119554217164869,
146
+ "pass@4": 0.8599147251969675,
147
+ "pass@8": 0.8945134207641785,
148
+ "pass@16": 0.9205661933194598,
149
+ "pass@32": 0.9404403427759488,
150
+ "pass@64": 0.9559823521026871,
151
+ "pass@128": 0.9686520376175548
152
+ },
153
+ "14": {
154
+ "pass@1": 0.6125632788161994,
155
+ "pass@2": 0.7013274370201388,
156
+ "pass@4": 0.7658476230190846,
157
+ "pass@8": 0.8097702298192307,
158
+ "pass@16": 0.8407102562368179,
159
+ "pass@32": 0.8644964958281782,
160
+ "pass@64": 0.8838021812976606,
161
+ "pass@128": 0.9003115264797508
162
+ },
163
+ "15": {
164
+ "pass@1": 0.48457821531791906,
165
+ "pass@2": 0.5827305027081152,
166
+ "pass@4": 0.6590647869702699,
167
+ "pass@8": 0.7158310076759379,
168
+ "pass@16": 0.7589664128271321,
169
+ "pass@32": 0.7930825505983045,
170
+ "pass@64": 0.8204516163076292,
171
+ "pass@128": 0.8410404624277457
172
+ },
173
+ "16": {
174
+ "pass@1": 0.2755040322580645,
175
+ "pass@2": 0.3586213328676657,
176
+ "pass@4": 0.43517604109970115,
177
+ "pass@8": 0.5017510566263869,
178
+ "pass@16": 0.5583572473734497,
179
+ "pass@32": 0.6078831915021241,
180
+ "pass@64": 0.6517453480905581,
181
+ "pass@128": 0.6903225806451613
182
+ },
183
+ "17": {
184
+ "pass@1": 0.21780166184971098,
185
+ "pass@2": 0.29274348857584986,
186
+ "pass@4": 0.36758594012453627,
187
+ "pass@8": 0.4327260544001389,
188
+ "pass@16": 0.4852625887317101,
189
+ "pass@32": 0.5308733536032256,
190
+ "pass@64": 0.5731955827266514,
191
+ "pass@128": 0.615606936416185
192
+ },
193
+ "18": {
194
+ "pass@1": 0.18082589285714284,
195
+ "pass@2": 0.24475007030371193,
196
+ "pass@4": 0.30927755396646855,
197
+ "pass@8": 0.3685482161042458,
198
+ "pass@16": 0.4204588630253465,
199
+ "pass@32": 0.46585847061159913,
200
+ "pass@64": 0.5068792519157992,
201
+ "pass@128": 0.5457142857142857
202
+ },
203
+ "19": {
204
+ "pass@1": 0.1753472222222222,
205
+ "pass@2": 0.23986560379276908,
206
+ "pass@4": 0.3042127763308865,
207
+ "pass@8": 0.3648899684541569,
208
+ "pass@16": 0.41793983675736257,
209
+ "pass@32": 0.46237397780199607,
210
+ "pass@64": 0.5029501083188042,
211
+ "pass@128": 0.5405405405405406
212
+ },
213
+ "20": {
214
+ "pass@1": 0.15636730480480482,
215
+ "pass@2": 0.2165424528859568,
216
+ "pass@4": 0.27821547475484476,
217
+ "pass@8": 0.3365076147164066,
218
+ "pass@16": 0.3888313889531125,
219
+ "pass@32": 0.4337362063462613,
220
+ "pass@64": 0.4738117632886543,
221
+ "pass@128": 0.5135135135135135
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 589293,
228
+ "answer_accuracy": 0.71766976812159,
229
+ "avg_response_len": 266.5054985872954,
230
+ "resp_tokens_sum": 218832995,
231
+ "pass_at_k": {
232
+ "pass@1": 0.71766976812159,
233
+ "pass@2": 0.7541205482352454,
234
+ "pass@4": 0.7849935205585589,
235
+ "pass@8": 0.8106964627292033,
236
+ "pass@16": 0.831317315567593,
237
+ "pass@32": 0.8479933921096046,
238
+ "pass@64": 0.8623722337450352,
239
+ "pass@128": 0.875448168355417
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9412384969325154,
244
+ "pass@2": 0.9570337030819772,
245
+ "pass@4": 0.9684386097635029,
246
+ "pass@8": 0.9755421990555402,
247
+ "pass@16": 0.9790090135040668,
248
+ "pass@32": 0.9808182150392867,
249
+ "pass@64": 0.9815547734367644,
250
+ "pass@128": 0.9815950920245399
251
+ },
252
+ "2": {
253
+ "pass@1": 1.0,
254
+ "pass@2": 1.0,
255
+ "pass@4": 1.0,
256
+ "pass@8": 1.0,
257
+ "pass@16": 1.0,
258
+ "pass@32": 1.0,
259
+ "pass@64": 1.0,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 1.0,
264
+ "pass@2": 1.0,
265
+ "pass@4": 1.0,
266
+ "pass@8": 1.0,
267
+ "pass@16": 1.0,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9999075443786982,
274
+ "pass@2": 0.9999978160089457,
275
+ "pass@4": 0.9999999997226678,
276
+ "pass@8": 1.0,
277
+ "pass@16": 1.0,
278
+ "pass@32": 1.0,
279
+ "pass@64": 1.0,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9963768115942029,
284
+ "pass@2": 0.9971339010612804,
285
+ "pass@4": 0.9974446123582379,
286
+ "pass@8": 0.9977680501821345,
287
+ "pass@16": 0.9983124881128967,
288
+ "pass@32": 0.999097406302473,
289
+ "pass@64": 0.9998273650576287,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9997313753581661,
294
+ "pass@2": 0.9999841363400491,
295
+ "pass@4": 0.9999999435958757,
296
+ "pass@8": 0.9999999999999097,
297
+ "pass@16": 1.0,
298
+ "pass@32": 1.0,
299
+ "pass@64": 1.0,
300
+ "pass@128": 1.0
301
+ },
302
+ "7": {
303
+ "pass@1": 0.997674313583815,
304
+ "pass@2": 0.998275069978608,
305
+ "pass@4": 0.9989801556597332,
306
+ "pass@8": 0.9996539139236084,
307
+ "pass@16": 0.9999649956979594,
308
+ "pass@32": 0.9999998098093686,
309
+ "pass@64": 0.9999999999999141,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9957306338028169,
314
+ "pass@2": 0.9968081124542529,
315
+ "pass@4": 0.9971466248937193,
316
+ "pass@8": 0.9971827493473799,
317
+ "pass@16": 0.9971830985789736,
318
+ "pass@32": 0.9971830985915493,
319
+ "pass@64": 0.9971830985915493,
320
+ "pass@128": 0.9971830985915493
321
+ },
322
+ "9": {
323
+ "pass@1": 0.993310254491018,
324
+ "pass@2": 0.9940558106228489,
325
+ "pass@4": 0.994196379457059,
326
+ "pass@8": 0.9943759135003243,
327
+ "pass@16": 0.9947162761091989,
328
+ "pass@32": 0.995326276580697,
329
+ "pass@64": 0.9962633787543024,
330
+ "pass@128": 0.9970059880239521
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9478068524096386,
334
+ "pass@2": 0.9611561403092685,
335
+ "pass@4": 0.9686419691702693,
336
+ "pass@8": 0.9750669382908692,
337
+ "pass@16": 0.9806081356024587,
338
+ "pass@32": 0.9842428892059527,
339
+ "pass@64": 0.9862549466314855,
340
+ "pass@128": 0.9879518072289156
341
+ },
342
+ "12": {
343
+ "pass@1": 0.8410315438871473,
344
+ "pass@2": 0.8755083244884359,
345
+ "pass@4": 0.8937380000305601,
346
+ "pass@8": 0.905965360565369,
347
+ "pass@16": 0.9151980908995769,
348
+ "pass@32": 0.9235809759059825,
349
+ "pass@64": 0.9339121914379174,
350
+ "pass@128": 0.9467084639498433
351
+ },
352
+ "13": {
353
+ "pass@1": 0.7262123599439776,
354
+ "pass@2": 0.8062757091025381,
355
+ "pass@4": 0.8566690912585506,
356
+ "pass@8": 0.8882637919932588,
357
+ "pass@16": 0.9082264797954909,
358
+ "pass@32": 0.9222958616323473,
359
+ "pass@64": 0.9343518871725501,
360
+ "pass@128": 0.9467787114845938
361
+ },
362
+ "14": {
363
+ "pass@1": 0.6329952485380117,
364
+ "pass@2": 0.7221707792282543,
365
+ "pass@4": 0.7828848704767167,
366
+ "pass@8": 0.8258118272832482,
367
+ "pass@16": 0.8579023706600551,
368
+ "pass@32": 0.8832258032244631,
369
+ "pass@64": 0.9049950902372479,
370
+ "pass@128": 0.9269005847953217
371
+ },
372
+ "15": {
373
+ "pass@1": 0.4446022727272727,
374
+ "pass@2": 0.5437966028394173,
375
+ "pass@4": 0.6285262524002679,
376
+ "pass@8": 0.6960194300230498,
377
+ "pass@16": 0.7475932370874276,
378
+ "pass@32": 0.785985388651888,
379
+ "pass@64": 0.8147437752003887,
380
+ "pass@128": 0.8424242424242424
381
+ },
382
+ "16": {
383
+ "pass@1": 0.3059116809116809,
384
+ "pass@2": 0.39158751990936996,
385
+ "pass@4": 0.4727661555660242,
386
+ "pass@8": 0.5448737019577484,
387
+ "pass@16": 0.6040431432838912,
388
+ "pass@32": 0.6528764342680178,
389
+ "pass@64": 0.694521859735016,
390
+ "pass@128": 0.7293447293447294
391
+ },
392
+ "17": {
393
+ "pass@1": 0.26052989130434784,
394
+ "pass@2": 0.34320820291485293,
395
+ "pass@4": 0.42293633977041684,
396
+ "pass@8": 0.4927356348756277,
397
+ "pass@16": 0.5482837994495754,
398
+ "pass@32": 0.5921106992309274,
399
+ "pass@64": 0.6290502589670679,
400
+ "pass@128": 0.6614906832298136
401
+ },
402
+ "18": {
403
+ "pass@1": 0.2153262867647059,
404
+ "pass@2": 0.28677049559981455,
405
+ "pass@4": 0.35674156300315396,
406
+ "pass@8": 0.4197868754911729,
407
+ "pass@16": 0.47482275196634255,
408
+ "pass@32": 0.5236327311772606,
409
+ "pass@64": 0.5672130434247589,
410
+ "pass@128": 0.6029411764705882
411
+ },
412
+ "19": {
413
+ "pass@1": 0.16901506696428573,
414
+ "pass@2": 0.22576206294525675,
415
+ "pass@4": 0.28495160454273577,
416
+ "pass@8": 0.34010383127747634,
417
+ "pass@16": 0.38708927582856617,
418
+ "pass@32": 0.4276740630065036,
419
+ "pass@64": 0.4649332236307543,
420
+ "pass@128": 0.5
421
+ },
422
+ "20": {
423
+ "pass@1": 0.1504491017964072,
424
+ "pass@2": 0.2108255192371163,
425
+ "pass@4": 0.2735428047167756,
426
+ "pass@8": 0.3338204070406974,
427
+ "pass@16": 0.38676384570770866,
428
+ "pass@32": 0.4292259673414519,
429
+ "pass@64": 0.4663031080186876,
430
+ "pass@128": 0.5
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 575859,
437
+ "answer_accuracy": 0.7187886942802365,
438
+ "avg_response_len": 286.13477966727913,
439
+ "resp_tokens_sum": 229237451,
440
+ "pass_at_k": {
441
+ "pass@1": 0.7187886942802365,
442
+ "pass@2": 0.7548033784106732,
443
+ "pass@4": 0.7848112376220922,
444
+ "pass@8": 0.8098416329714997,
445
+ "pass@16": 0.8305795131153559,
446
+ "pass@32": 0.8481136410654446,
447
+ "pass@64": 0.8635051234293231,
448
+ "pass@128": 0.8771369228311232
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9569496268656716,
453
+ "pass@2": 0.9678403014455282,
454
+ "pass@4": 0.976078659010907,
455
+ "pass@8": 0.9827045652363846,
456
+ "pass@16": 0.9880512471221918,
457
+ "pass@32": 0.9920857219167919,
458
+ "pass@64": 0.9938974306133376,
459
+ "pass@128": 0.9940298507462687
460
+ },
461
+ "2": {
462
+ "pass@1": 1.0,
463
+ "pass@2": 1.0,
464
+ "pass@4": 1.0,
465
+ "pass@8": 1.0,
466
+ "pass@16": 1.0,
467
+ "pass@32": 1.0,
468
+ "pass@64": 1.0,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 1.0,
473
+ "pass@2": 1.0,
474
+ "pass@4": 1.0,
475
+ "pass@8": 1.0,
476
+ "pass@16": 1.0,
477
+ "pass@32": 1.0,
478
+ "pass@64": 1.0,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9997487207602339,
483
+ "pass@2": 0.9999802142330892,
484
+ "pass@4": 0.9999999095507799,
485
+ "pass@8": 0.9999999999996626,
486
+ "pass@16": 1.0,
487
+ "pass@32": 1.0,
488
+ "pass@64": 1.0,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9922564338235295,
493
+ "pass@2": 0.994646320634553,
494
+ "pass@4": 0.9961220976973466,
495
+ "pass@8": 0.9970632553671733,
496
+ "pass@16": 0.997715072688465,
497
+ "pass@32": 0.9983497369764539,
498
+ "pass@64": 0.9992704955997272,
499
+ "pass@128": 1.0
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9992360015772871,
503
+ "pass@2": 0.9998886118631859,
504
+ "pass@4": 0.9999968563669289,
505
+ "pass@8": 0.9999999983772165,
506
+ "pass@16": 1.0,
507
+ "pass@32": 1.0,
508
+ "pass@64": 1.0,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.998318829113924,
513
+ "pass@2": 0.9996340202332306,
514
+ "pass@4": 0.9999719820425929,
515
+ "pass@8": 0.9999998296794577,
516
+ "pass@16": 0.999999999997869,
517
+ "pass@32": 1.0,
518
+ "pass@64": 1.0,
519
+ "pass@128": 1.0
520
+ },
521
+ "8": {
522
+ "pass@1": 0.995777027027027,
523
+ "pass@2": 0.9988557701402191,
524
+ "pass@4": 0.9998988651193375,
525
+ "pass@8": 0.9999992668226257,
526
+ "pass@16": 0.9999999999808709,
527
+ "pass@32": 1.0,
528
+ "pass@64": 1.0,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9933430466237942,
533
+ "pass@2": 0.9955922285236853,
534
+ "pass@4": 0.996922894083577,
535
+ "pass@8": 0.99825698853535,
536
+ "pass@16": 0.9992957181536803,
537
+ "pass@32": 0.9998834594758289,
538
+ "pass@64": 0.9999990174474539,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.9458333333333333,
543
+ "pass@2": 0.9680735220597424,
544
+ "pass@4": 0.9784683795477946,
545
+ "pass@8": 0.9834253742459571,
546
+ "pass@16": 0.9859158447021265,
547
+ "pass@32": 0.9876978205871854,
548
+ "pass@64": 0.9894997112731746,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.8640577217125383,
553
+ "pass@2": 0.9053041248284324,
554
+ "pass@4": 0.9285084223990344,
555
+ "pass@8": 0.9444538698747363,
556
+ "pass@16": 0.9565054855884125,
557
+ "pass@32": 0.9649179472227764,
558
+ "pass@64": 0.971280941814739,
559
+ "pass@128": 0.9785932721712538
560
+ },
561
+ "13": {
562
+ "pass@1": 0.7043306327160493,
563
+ "pass@2": 0.7786207713619127,
564
+ "pass@4": 0.8288944483791375,
565
+ "pass@8": 0.86676143621755,
566
+ "pass@16": 0.8943791026870926,
567
+ "pass@32": 0.9153670080570203,
568
+ "pass@64": 0.9331493513324162,
569
+ "pass@128": 0.9475308641975309
570
+ },
571
+ "14": {
572
+ "pass@1": 0.6179293397626113,
573
+ "pass@2": 0.703675538563985,
574
+ "pass@4": 0.7640525688183634,
575
+ "pass@8": 0.8070382454357383,
576
+ "pass@16": 0.8387920432900056,
577
+ "pass@32": 0.8638308703360127,
578
+ "pass@64": 0.8855514513931839,
579
+ "pass@128": 0.9050445103857567
580
+ },
581
+ "15": {
582
+ "pass@1": 0.46923225308641975,
583
+ "pass@2": 0.5619528136239913,
584
+ "pass@4": 0.6342046055007012,
585
+ "pass@8": 0.6883368569053809,
586
+ "pass@16": 0.7319691872044063,
587
+ "pass@32": 0.7717383879708698,
588
+ "pass@64": 0.8095051745441575,
589
+ "pass@128": 0.8425925925925926
590
+ },
591
+ "16": {
592
+ "pass@1": 0.31752396755162243,
593
+ "pass@2": 0.3971402254198313,
594
+ "pass@4": 0.46753584065266157,
595
+ "pass@8": 0.5274963190047928,
596
+ "pass@16": 0.578633799879253,
597
+ "pass@32": 0.6236236229841198,
598
+ "pass@64": 0.6655990132285015,
599
+ "pass@128": 0.7079646017699115
600
+ },
601
+ "17": {
602
+ "pass@1": 0.2409403237951807,
603
+ "pass@2": 0.3118437084953988,
604
+ "pass@4": 0.38310006194632296,
605
+ "pass@8": 0.4517112008785566,
606
+ "pass@16": 0.5143629770086949,
607
+ "pass@32": 0.5692526943598468,
608
+ "pass@64": 0.6171244785989948,
609
+ "pass@128": 0.6566265060240963
610
+ },
611
+ "18": {
612
+ "pass@1": 0.21854838709677418,
613
+ "pass@2": 0.2914830454660909,
614
+ "pass@4": 0.36253027202244875,
615
+ "pass@8": 0.4232167175836581,
616
+ "pass@16": 0.4722504664411351,
617
+ "pass@32": 0.5141885209927322,
618
+ "pass@64": 0.5525587893884821,
619
+ "pass@128": 0.5870967741935483
620
+ },
621
+ "19": {
622
+ "pass@1": 0.1644873489425982,
623
+ "pass@2": 0.22967898815329338,
624
+ "pass@4": 0.296141610225459,
625
+ "pass@8": 0.35542356027993677,
626
+ "pass@16": 0.40438333117475245,
627
+ "pass@32": 0.44624875815883597,
628
+ "pass@64": 0.4841766947910102,
629
+ "pass@128": 0.5196374622356495
630
+ },
631
+ "20": {
632
+ "pass@1": 0.16366366366366367,
633
+ "pass@2": 0.22465292556335845,
634
+ "pass@4": 0.28830474653506144,
635
+ "pass@8": 0.3515397113540908,
636
+ "pass@16": 0.4098242301095972,
637
+ "pass@32": 0.4585545918829372,
638
+ "pass@64": 0.49711705635141545,
639
+ "pass@128": 0.5285285285285285
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fd488450c44bfa545a35cc390d60b8190a3a29317b9041b92c1dd21c8dbe957
3
+ size 1789580934
results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block16-epoch4-op9-12",
3
+ "id": {
4
+ "answer_accuracy": 0.9921467013888889,
5
+ "avg_response_len": 199.46203125,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9921467013888889,
11
+ "pass@2": 0.9944876284995613,
12
+ "pass@4": 0.9959433585697617,
13
+ "pass@8": 0.9968784496129127,
14
+ "pass@16": 0.9976004460052793,
15
+ "pass@32": 0.9981871074093316,
16
+ "pass@64": 0.9986349505024621,
17
+ "pass@128": 0.999
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9524140625,
22
+ "pass@2": 0.9645937500000002,
23
+ "pass@4": 0.9730021943194608,
24
+ "pass@8": 0.9786367820253195,
25
+ "pass@16": 0.9831046505061921,
26
+ "pass@32": 0.9868706993361868,
27
+ "pass@64": 0.9899770280239669,
28
+ "pass@128": 0.993
29
+ },
30
+ "2": {
31
+ "pass@1": 1.0,
32
+ "pass@2": 1.0,
33
+ "pass@4": 1.0,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 1.0,
42
+ "pass@2": 1.0,
43
+ "pass@4": 1.0,
44
+ "pass@8": 1.0,
45
+ "pass@16": 1.0,
46
+ "pass@32": 1.0,
47
+ "pass@64": 1.0,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9987109375,
52
+ "pass@2": 0.999171998031496,
53
+ "pass@4": 0.9994346713535808,
54
+ "pass@8": 0.9996911656406087,
55
+ "pass@16": 0.9999126144227602,
56
+ "pass@32": 0.9999949322702752,
57
+ "pass@64": 0.9999999977569969,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.993890625,
62
+ "pass@2": 0.996357775590551,
63
+ "pass@4": 0.9976947060367454,
64
+ "pass@8": 0.9983995073818661,
65
+ "pass@16": 0.9989712933683291,
66
+ "pass@32": 0.9994882787743641,
67
+ "pass@64": 0.9998748568874752,
68
+ "pass@128": 1.0
69
+ },
70
+ "6": {
71
+ "pass@1": 0.997625,
72
+ "pass@2": 0.9985620078740157,
73
+ "pass@4": 0.9990585259655044,
74
+ "pass@8": 0.9993178674757099,
75
+ "pass@16": 0.9995588461117559,
76
+ "pass@32": 0.9998290717621782,
77
+ "pass@64": 0.9999861763011332,
78
+ "pass@128": 1.0
79
+ },
80
+ "7": {
81
+ "pass@1": 0.997484375,
82
+ "pass@2": 0.9984462352362204,
83
+ "pass@4": 0.9990977215973003,
84
+ "pass@8": 0.9995759792654791,
85
+ "pass@16": 0.9998778554116803,
86
+ "pass@32": 0.9999900984417422,
87
+ "pass@64": 0.999999987930507,
88
+ "pass@128": 1.0
89
+ },
90
+ "8": {
91
+ "pass@1": 0.995140625,
92
+ "pass@2": 0.9973559301181102,
93
+ "pass@4": 0.9985708786089239,
94
+ "pass@8": 0.9991065163275745,
95
+ "pass@16": 0.9993309358545857,
96
+ "pass@32": 0.9995814582857677,
97
+ "pass@64": 0.9998779527559055,
98
+ "pass@128": 1.0
99
+ },
100
+ "9": {
101
+ "pass@1": 0.9940546875,
102
+ "pass@2": 0.9959009596456694,
103
+ "pass@4": 0.9966315292463443,
104
+ "pass@8": 0.99717822839966,
105
+ "pass@16": 0.9976478183722098,
106
+ "pass@32": 0.9979294278134688,
107
+ "pass@64": 0.9979985548661734,
108
+ "pass@128": 0.998
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9524140625,
113
+ "2": 1.0,
114
+ "3": 1.0,
115
+ "4": 0.9987109375,
116
+ "5": 0.993890625,
117
+ "6": 0.997625,
118
+ "7": 0.997484375,
119
+ "8": 0.995140625,
120
+ "9": 0.9940546875
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 279.8089140625,
124
+ "2": 115.4520703125,
125
+ "3": 144.40334375,
126
+ "4": 159.6451328125,
127
+ "5": 172.749,
128
+ "6": 198.4754375,
129
+ "7": 223.35846875,
130
+ "8": 242.756265625,
131
+ "9": 258.5096484375
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 279.8089140625,
136
+ "2": 115.4520703125,
137
+ "3": 144.40334375,
138
+ "4": 159.6451328125,
139
+ "5": 172.749,
140
+ "6": 198.4754375,
141
+ "7": 223.35846875,
142
+ "8": 242.756265625,
143
+ "9": 258.5096484375
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 376348,
150
+ "answer_accuracy": 0.9929816784869976,
151
+ "avg_response_len": 189.93894587977036,
152
+ "resp_tokens_sum": 71988380,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9929816784869976,
155
+ "pass@2": 0.9952404553154262,
156
+ "pass@4": 0.996518110166574,
157
+ "pass@8": 0.9972523121377939,
158
+ "pass@16": 0.9978434752201003,
159
+ "pass@32": 0.9984079378454725,
160
+ "pass@64": 0.9988835021935919,
161
+ "pass@128": 0.9993245525160419
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9574575958702065,
166
+ "pass@2": 0.9697600196269713,
167
+ "pass@4": 0.9770701256369501,
168
+ "pass@8": 0.9807352442070735,
169
+ "pass@16": 0.9835831385400656,
170
+ "pass@32": 0.9868139186902647,
171
+ "pass@64": 0.9902929813385202,
172
+ "pass@128": 0.9941002949852508
173
+ },
174
+ "2": {
175
+ "pass@1": 1.0,
176
+ "pass@2": 1.0,
177
+ "pass@4": 1.0,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 1.0,
186
+ "pass@2": 1.0,
187
+ "pass@4": 1.0,
188
+ "pass@8": 1.0,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.996923828125,
196
+ "pass@2": 0.9976178026574803,
197
+ "pass@4": 0.9982453347628422,
198
+ "pass@8": 0.9990349229744625,
199
+ "pass@16": 0.9997269200711651,
200
+ "pass@32": 0.9999841633446103,
201
+ "pass@64": 0.9999999929906153,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9954861111111111,
206
+ "pass@2": 0.9981666354205724,
207
+ "pass@4": 0.9995852298224628,
208
+ "pass@8": 0.9999710725449563,
209
+ "pass@16": 0.9999998592866416,
210
+ "pass@32": 0.9999999999993872,
211
+ "pass@64": 1.0,
212
+ "pass@128": 1.0
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9968890344311377,
216
+ "pass@2": 0.997264575180348,
217
+ "pass@4": 0.997534851556729,
218
+ "pass@8": 0.9979835642250304,
219
+ "pass@16": 0.9986793120796699,
220
+ "pass@32": 0.9994882388094946,
221
+ "pass@64": 0.9999586116800393,
222
+ "pass@128": 1.0
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9973187869822485,
226
+ "pass@2": 0.9984067785258351,
227
+ "pass@4": 0.9992237471795316,
228
+ "pass@8": 0.9998073811974663,
229
+ "pass@16": 0.9999900064095892,
230
+ "pass@32": 0.9999999886267649,
231
+ "pass@64": 1.0,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9983974358974359,
236
+ "pass@2": 0.999743290051484,
237
+ "pass@4": 0.9999842372467864,
238
+ "pass@8": 0.9999999472370541,
239
+ "pass@16": 0.9999999999998607,
240
+ "pass@32": 1.0,
241
+ "pass@64": 1.0,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9957526408450704,
246
+ "pass@2": 0.9972687007874016,
247
+ "pass@4": 0.997876473539399,
248
+ "pass@8": 0.9984546377928358,
249
+ "pass@16": 0.999188066655105,
250
+ "pass@32": 0.9998084093975909,
251
+ "pass@64": 0.9999959303192606,
252
+ "pass@128": 1.0
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 387303,
259
+ "answer_accuracy": 0.991417001146789,
260
+ "avg_response_len": 197.48486648099606,
261
+ "resp_tokens_sum": 77148648,
262
+ "pass_at_k": {
263
+ "pass@1": 0.991417001146789,
264
+ "pass@2": 0.993621147485578,
265
+ "pass@4": 0.9950336770751789,
266
+ "pass@8": 0.9959590200409408,
267
+ "pass@16": 0.9966420865580624,
268
+ "pass@32": 0.9971659138581003,
269
+ "pass@64": 0.9976167577265961,
270
+ "pass@128": 0.9980340760157274
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9421731211656442,
275
+ "pass@2": 0.9570635174629243,
276
+ "pass@4": 0.9675180118397778,
277
+ "pass@8": 0.9744761195433342,
278
+ "pass@16": 0.9791587744555608,
279
+ "pass@32": 0.9822004591222739,
280
+ "pass@64": 0.984571936009979,
281
+ "pass@128": 0.9877300613496932
282
+ },
283
+ "2": {
284
+ "pass@1": 1.0,
285
+ "pass@2": 1.0,
286
+ "pass@4": 1.0,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 1.0,
295
+ "pass@2": 1.0,
296
+ "pass@4": 1.0,
297
+ "pass@8": 1.0,
298
+ "pass@16": 1.0,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9999306582840237,
305
+ "pass@2": 0.9999989080044729,
306
+ "pass@4": 1.0,
307
+ "pass@8": 1.0,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9956295289855073,
315
+ "pass@2": 0.9968182985279016,
316
+ "pass@4": 0.9973139007080636,
317
+ "pass@8": 0.9976146824957167,
318
+ "pass@16": 0.998064779875966,
319
+ "pass@32": 0.9987868364280551,
320
+ "pass@64": 0.9996462398721899,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9972242120343839,
325
+ "pass@2": 0.9987002374613632,
326
+ "pass@4": 0.9996714137452016,
327
+ "pass@8": 0.9999752561101265,
328
+ "pass@16": 0.9999998735735035,
329
+ "pass@32": 0.999999999999447,
330
+ "pass@64": 1.0,
331
+ "pass@128": 1.0
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9973807803468208,
335
+ "pass@2": 0.9977498748350099,
336
+ "pass@4": 0.9982556172530457,
337
+ "pass@8": 0.9989655495774096,
338
+ "pass@16": 0.9996567447729243,
339
+ "pass@32": 0.999971393889872,
340
+ "pass@64": 0.9999999651170723,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9954665492957746,
345
+ "pass@2": 0.9968725740268383,
346
+ "pass@4": 0.9974011850631346,
347
+ "pass@8": 0.9976822657434412,
348
+ "pass@16": 0.9981192931743675,
349
+ "pass@32": 0.9988210100498002,
350
+ "pass@64": 0.9996562049462128,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9930295658682635,
355
+ "pass@2": 0.9939206233202886,
356
+ "pass@4": 0.9940102609741148,
357
+ "pass@8": 0.9940119756217468,
358
+ "pass@16": 0.9940119760479041,
359
+ "pass@32": 0.9940119760479041,
360
+ "pass@64": 0.9940119760479041,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 379302,
368
+ "answer_accuracy": 0.9920645714763977,
369
+ "avg_response_len": 210.92241379310346,
370
+ "resp_tokens_sum": 80643232,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9920645714763977,
373
+ "pass@2": 0.9946266910549394,
374
+ "pass@4": 0.9963030868735149,
375
+ "pass@8": 0.9974472785775886,
376
+ "pass@16": 0.9983387464832911,
377
+ "pass@32": 0.9990116148739929,
378
+ "pass@64": 0.999428911933499,
379
+ "pass@128": 0.9996652159357214
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9572761194029851,
384
+ "pass@2": 0.9666937213538606,
385
+ "pass@4": 0.9742225369216908,
386
+ "pass@8": 0.9805621470686411,
387
+ "pass@16": 0.9864603166226772,
388
+ "pass@32": 0.991472929033808,
389
+ "pass@64": 0.9949171827192708,
390
+ "pass@128": 0.9970149253731343
391
+ },
392
+ "2": {
393
+ "pass@1": 1.0,
394
+ "pass@2": 1.0,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 1.0,
404
+ "pass@2": 1.0,
405
+ "pass@4": 1.0,
406
+ "pass@8": 1.0,
407
+ "pass@16": 1.0,
408
+ "pass@32": 1.0,
409
+ "pass@64": 1.0,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9991776315789473,
414
+ "pass@2": 0.9998089774140073,
415
+ "pass@4": 0.9999887842966998,
416
+ "pass@8": 0.9999999716046218,
417
+ "pass@16": 0.9999999999999635,
418
+ "pass@32": 1.0,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9906479779411764,
424
+ "pass@2": 0.9942146248263086,
425
+ "pass@4": 0.996329596759964,
426
+ "pass@8": 0.9977398649093597,
427
+ "pass@16": 0.9989382018700852,
428
+ "pass@32": 0.9997259417849353,
429
+ "pass@64": 0.9999908945046169,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9988416798107256,
434
+ "pass@2": 0.9997768356143968,
435
+ "pass@4": 0.9999891550425282,
436
+ "pass@8": 0.9999999767890079,
437
+ "pass@16": 0.9999999999999797,
438
+ "pass@32": 1.0,
439
+ "pass@64": 1.0,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.997774920886076,
444
+ "pass@2": 0.9992509094986542,
445
+ "pass@4": 0.9998849730413445,
446
+ "pass@8": 0.9999968489460502,
447
+ "pass@16": 0.9999999985879974,
448
+ "pass@32": 1.0,
449
+ "pass@64": 1.0,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9917417417417418,
454
+ "pass@2": 0.9956344139415007,
455
+ "pass@4": 0.998493621292834,
456
+ "pass@8": 0.999787773125201,
457
+ "pass@16": 0.9999957560892757,
458
+ "pass@32": 0.9999999991534195,
459
+ "pass@64": 1.0,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9932174437299035,
464
+ "pass@2": 0.9964665037850975,
465
+ "pass@4": 0.9980255754807175,
466
+ "pass@8": 0.9991216467059164,
467
+ "pass@16": 0.9997943881339146,
468
+ "pass@32": 0.9999917764544182,
469
+ "pass@64": 0.9999999987229452,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.5692328125,
478
+ "avg_response_len": 350.66635078125,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.5692328125,
484
+ "pass@2": 0.638573991141724,
485
+ "pass@4": 0.6956425176509109,
486
+ "pass@8": 0.7415618754091828,
487
+ "pass@16": 0.7774673995948668,
488
+ "pass@32": 0.805792751873983,
489
+ "pass@64": 0.8294758529319611,
490
+ "pass@128": 0.8504
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.9613984375,
495
+ "pass@2": 0.9735031988188977,
496
+ "pass@4": 0.9804231168916386,
497
+ "pass@8": 0.9847109635922205,
498
+ "pass@16": 0.9870854668143153,
499
+ "pass@32": 0.9886758259692309,
500
+ "pass@64": 0.9904900450431707,
501
+ "pass@128": 0.993
502
+ },
503
+ "12": {
504
+ "pass@1": 0.899875,
505
+ "pass@2": 0.9206909448818899,
506
+ "pass@4": 0.9333334387889014,
507
+ "pass@8": 0.9429585488390639,
508
+ "pass@16": 0.9508381683007954,
509
+ "pass@32": 0.957124237853498,
510
+ "pass@64": 0.9620219505951612,
511
+ "pass@128": 0.966
512
+ },
513
+ "13": {
514
+ "pass@1": 0.8238984375,
515
+ "pass@2": 0.8724411909448823,
516
+ "pass@4": 0.9021637046306712,
517
+ "pass@8": 0.9231090688142323,
518
+ "pass@16": 0.9379672291934675,
519
+ "pass@32": 0.9484869565082493,
520
+ "pass@64": 0.9568897147822485,
521
+ "pass@128": 0.963
522
+ },
523
+ "14": {
524
+ "pass@1": 0.7794453125,
525
+ "pass@2": 0.8445839074803148,
526
+ "pass@4": 0.8855294000749906,
527
+ "pass@8": 0.9114116915660494,
528
+ "pass@16": 0.9273511399284904,
529
+ "pass@32": 0.9386215902003806,
530
+ "pass@64": 0.9495939611176091,
531
+ "pass@128": 0.962
532
+ },
533
+ "15": {
534
+ "pass@1": 0.6651171875,
535
+ "pass@2": 0.7486331200787407,
536
+ "pass@4": 0.806329887701538,
537
+ "pass@8": 0.8473808946123879,
538
+ "pass@16": 0.8765889720497365,
539
+ "pass@32": 0.8979515241106913,
540
+ "pass@64": 0.9145665425705841,
541
+ "pass@128": 0.928
542
+ },
543
+ "16": {
544
+ "pass@1": 0.4879609375,
545
+ "pass@2": 0.5945366633858277,
546
+ "pass@4": 0.6776242561867271,
547
+ "pass@8": 0.739991947562845,
548
+ "pass@16": 0.7879800006920792,
549
+ "pass@32": 0.8266620166339039,
550
+ "pass@64": 0.8592523951014696,
551
+ "pass@128": 0.89
552
+ },
553
+ "17": {
554
+ "pass@1": 0.369609375,
555
+ "pass@2": 0.47383956692913415,
556
+ "pass@4": 0.566708111173604,
557
+ "pass@8": 0.6421709788558302,
558
+ "pass@16": 0.699630230918739,
559
+ "pass@32": 0.7440501332532892,
560
+ "pass@64": 0.7798027890890089,
561
+ "pass@128": 0.811
562
+ },
563
+ "18": {
564
+ "pass@1": 0.3044921875,
565
+ "pass@2": 0.402151697834646,
566
+ "pass@4": 0.49101950562429714,
567
+ "pass@8": 0.5642945200089501,
568
+ "pass@16": 0.6220010627856662,
569
+ "pass@32": 0.6671294229443815,
570
+ "pass@64": 0.7039513297789204,
571
+ "pass@128": 0.734
572
+ },
573
+ "19": {
574
+ "pass@1": 0.20428125,
575
+ "pass@2": 0.28356422244094465,
576
+ "pass@4": 0.363903641076115,
577
+ "pass@8": 0.4365091889268906,
578
+ "pass@16": 0.4967211171125234,
579
+ "pass@32": 0.5470671625654399,
580
+ "pass@64": 0.5925308099061153,
581
+ "pass@128": 0.635
582
+ },
583
+ "20": {
584
+ "pass@1": 0.19625,
585
+ "pass@2": 0.27179539862204705,
586
+ "pass@4": 0.3493901143607049,
587
+ "pass@8": 0.4230809513134006,
588
+ "pass@16": 0.4885106081527199,
589
+ "pass@32": 0.5421586487007338,
590
+ "pass@64": 0.5856589913352467,
591
+ "pass@128": 0.622
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.9613984375,
596
+ "12": 0.899875,
597
+ "13": 0.8238984375,
598
+ "14": 0.7794453125,
599
+ "15": 0.6651171875,
600
+ "16": 0.4879609375,
601
+ "17": 0.369609375,
602
+ "18": 0.3044921875,
603
+ "19": 0.20428125,
604
+ "20": 0.19625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 297.492234375,
608
+ "12": 307.809671875,
609
+ "13": 326.856828125,
610
+ "14": 351.034875,
611
+ "15": 362.367296875,
612
+ "16": 361.9160234375,
613
+ "17": 375.5798671875,
614
+ "18": 376.86175,
615
+ "19": 367.4975859375,
616
+ "20": 379.247375
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 297.492234375,
621
+ "12": 307.809671875,
622
+ "13": 326.856828125,
623
+ "14": 351.034875,
624
+ "15": 362.367296875,
625
+ "16": 361.9160234375,
626
+ "17": 375.5798671875,
627
+ "18": 376.86175,
628
+ "19": 367.4975859375,
629
+ "20": 379.247375
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 245625,
636
+ "answer_accuracy": 0.5702660661218425,
637
+ "avg_response_len": 332.6203403603269,
638
+ "resp_tokens_sum": 143266233,
639
+ "pass_at_k": {
640
+ "pass@1": 0.5702660661218425,
641
+ "pass@2": 0.6388949541364937,
642
+ "pass@4": 0.6949541250137222,
643
+ "pass@8": 0.7397971193299135,
644
+ "pass@16": 0.7748348215257485,
645
+ "pass@32": 0.8027802603018277,
646
+ "pass@64": 0.8269381943859584,
647
+ "pass@128": 0.8484398216939079
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9572193696883853,
652
+ "pass@2": 0.9719146628449066,
653
+ "pass@4": 0.981724504514839,
654
+ "pass@8": 0.9890359200866053,
655
+ "pass@16": 0.9932697779011666,
656
+ "pass@32": 0.9957361168730702,
657
+ "pass@64": 0.9978415915007174,
658
+ "pass@128": 1.0
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8983271539548022,
662
+ "pass@2": 0.920239098158281,
663
+ "pass@4": 0.9334048053315367,
664
+ "pass@8": 0.9434168617466765,
665
+ "pass@16": 0.9517176538857137,
666
+ "pass@32": 0.9580437536392892,
667
+ "pass@64": 0.961533417356335,
668
+ "pass@128": 0.963276836158192
669
+ },
670
+ "13": {
671
+ "pass@1": 0.8390722962382445,
672
+ "pass@2": 0.8873212765038385,
673
+ "pass@4": 0.9195430402704361,
674
+ "pass@8": 0.9430086769866958,
675
+ "pass@16": 0.9574399410309512,
676
+ "pass@32": 0.9647881464197946,
677
+ "pass@64": 0.9690691600729979,
678
+ "pass@128": 0.9717868338557993
679
+ },
680
+ "14": {
681
+ "pass@1": 0.7840975467289719,
682
+ "pass@2": 0.8466142192214293,
683
+ "pass@4": 0.8872996447873914,
684
+ "pass@8": 0.9131255212050312,
685
+ "pass@16": 0.9296828208789675,
686
+ "pass@32": 0.9427319318749825,
687
+ "pass@64": 0.9554954405969336,
688
+ "pass@128": 0.9688473520249221
689
+ },
690
+ "15": {
691
+ "pass@1": 0.693099710982659,
692
+ "pass@2": 0.7778968583587461,
693
+ "pass@4": 0.833728548292012,
694
+ "pass@8": 0.8711126510602295,
695
+ "pass@16": 0.8958576584940476,
696
+ "pass@32": 0.9133306283181201,
697
+ "pass@64": 0.9284685661529859,
698
+ "pass@128": 0.9421965317919075
699
+ },
700
+ "16": {
701
+ "pass@1": 0.4913054435483871,
702
+ "pass@2": 0.5962447612395223,
703
+ "pass@4": 0.6747601975156323,
704
+ "pass@8": 0.7348407821560055,
705
+ "pass@16": 0.7834148610294204,
706
+ "pass@32": 0.8244895329338998,
707
+ "pass@64": 0.8601964216978978,
708
+ "pass@128": 0.8903225806451613
709
+ },
710
+ "17": {
711
+ "pass@1": 0.35248825867052025,
712
+ "pass@2": 0.4572849580128351,
713
+ "pass@4": 0.5492719623919837,
714
+ "pass@8": 0.6229445136286828,
715
+ "pass@16": 0.6790291809157839,
716
+ "pass@32": 0.7227658365318381,
717
+ "pass@64": 0.7602209527875393,
718
+ "pass@128": 0.7976878612716763
719
+ },
720
+ "18": {
721
+ "pass@1": 0.27520089285714283,
722
+ "pass@2": 0.3652481721034869,
723
+ "pass@4": 0.4477995007766887,
724
+ "pass@8": 0.5185924991538704,
725
+ "pass@16": 0.5777052275710203,
726
+ "pass@32": 0.6248875096332369,
727
+ "pass@64": 0.6638893307765378,
728
+ "pass@128": 0.6971428571428572
729
+ },
730
+ "19": {
731
+ "pass@1": 0.2045795795795796,
732
+ "pass@2": 0.28665747440353734,
733
+ "pass@4": 0.368495601338121,
734
+ "pass@8": 0.44158292317157055,
735
+ "pass@16": 0.5021246498473418,
736
+ "pass@32": 0.5528371750040144,
737
+ "pass@64": 0.5987933372999518,
738
+ "pass@128": 0.6396396396396397
739
+ },
740
+ "20": {
741
+ "pass@1": 0.19566441441441443,
742
+ "pass@2": 0.2724017078101725,
743
+ "pass@4": 0.34911851418347484,
744
+ "pass@8": 0.41767593951939913,
745
+ "pass@16": 0.47676148728079454,
746
+ "pass@32": 0.5280550430504534,
747
+ "pass@64": 0.5749912993919669,
748
+ "pass@128": 0.6156156156156156
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 248292,
755
+ "answer_accuracy": 0.5768008474576272,
756
+ "avg_response_len": 346.25764523862625,
757
+ "resp_tokens_sum": 149051451,
758
+ "pass_at_k": {
759
+ "pass@1": 0.5768008474576272,
760
+ "pass@2": 0.6462570475133562,
761
+ "pass@4": 0.7029366738948009,
762
+ "pass@8": 0.7483487965631102,
763
+ "pass@16": 0.7839769463433715,
764
+ "pass@32": 0.8124999007618324,
765
+ "pass@64": 0.8363338575179757,
766
+ "pass@128": 0.8581623550401427
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9631024096385542,
771
+ "pass@2": 0.972274185323973,
772
+ "pass@4": 0.9774970881840069,
773
+ "pass@8": 0.9805179334532464,
774
+ "pass@16": 0.9822529509986989,
775
+ "pass@32": 0.9834096807350399,
776
+ "pass@64": 0.9849397431912068,
777
+ "pass@128": 0.9879518072289156
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8928781347962382,
781
+ "pass@2": 0.9105734579024021,
782
+ "pass@4": 0.920715678859108,
783
+ "pass@8": 0.928907449166582,
784
+ "pass@16": 0.9358474938634546,
785
+ "pass@32": 0.9417129174503823,
786
+ "pass@64": 0.9473011819327243,
787
+ "pass@128": 0.9529780564263323
788
+ },
789
+ "13": {
790
+ "pass@1": 0.8309042366946778,
791
+ "pass@2": 0.8788773875692008,
792
+ "pass@4": 0.9060005905249233,
793
+ "pass@8": 0.9230680510501225,
794
+ "pass@16": 0.9352975407775181,
795
+ "pass@32": 0.9462738870529341,
796
+ "pass@64": 0.956931111957274,
797
+ "pass@128": 0.9663865546218487
798
+ },
799
+ "14": {
800
+ "pass@1": 0.7888797514619883,
801
+ "pass@2": 0.8581284966846247,
802
+ "pass@4": 0.9010667306937505,
803
+ "pass@8": 0.9274503748185816,
804
+ "pass@16": 0.9419725164541798,
805
+ "pass@32": 0.9505580533617753,
806
+ "pass@64": 0.9588922804756361,
807
+ "pass@128": 0.9707602339181286
808
+ },
809
+ "15": {
810
+ "pass@1": 0.6662642045454545,
811
+ "pass@2": 0.7540548198520639,
812
+ "pass@4": 0.8157365976980145,
813
+ "pass@8": 0.8565477326282641,
814
+ "pass@16": 0.8831288202183204,
815
+ "pass@32": 0.9034765538582886,
816
+ "pass@64": 0.9217986009661605,
817
+ "pass@128": 0.9393939393939394
818
+ },
819
+ "16": {
820
+ "pass@1": 0.5013799857549858,
821
+ "pass@2": 0.6062000162639922,
822
+ "pass@4": 0.688734711846916,
823
+ "pass@8": 0.7507240841046224,
824
+ "pass@16": 0.7988058883708389,
825
+ "pass@32": 0.8382289811645698,
826
+ "pass@64": 0.8684671985838386,
827
+ "pass@128": 0.8945868945868946
828
+ },
829
+ "17": {
830
+ "pass@1": 0.39540469720496896,
831
+ "pass@2": 0.5015440070670513,
832
+ "pass@4": 0.5931639159429605,
833
+ "pass@8": 0.6665612968773015,
834
+ "pass@16": 0.7219907477028704,
835
+ "pass@32": 0.7646644496183821,
836
+ "pass@64": 0.7996400240328937,
837
+ "pass@128": 0.8322981366459627
838
+ },
839
+ "18": {
840
+ "pass@1": 0.32417279411764705,
841
+ "pass@2": 0.42494535954145407,
842
+ "pass@4": 0.517787185425351,
843
+ "pass@8": 0.5978804895419353,
844
+ "pass@16": 0.6618267341101194,
845
+ "pass@32": 0.7122069429681127,
846
+ "pass@64": 0.7544866903270767,
847
+ "pass@128": 0.788235294117647
848
+ },
849
+ "19": {
850
+ "pass@1": 0.20305524553571427,
851
+ "pass@2": 0.2782470794666291,
852
+ "pass@4": 0.35577190797578867,
853
+ "pass@8": 0.42750031369822505,
854
+ "pass@16": 0.4901520073095929,
855
+ "pass@32": 0.5430748566756388,
856
+ "pass@64": 0.5864859314590565,
857
+ "pass@128": 0.625
858
+ },
859
+ "20": {
860
+ "pass@1": 0.20106661676646706,
861
+ "pass@2": 0.2747646930548353,
862
+ "pass@4": 0.3491225167774687,
863
+ "pass@8": 0.42036866918817706,
864
+ "pass@16": 0.4844573511671379,
865
+ "pass@32": 0.5371766445282311,
866
+ "pass@64": 0.5800651608494923,
867
+ "pass@128": 0.6197604790419161
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 234701,
874
+ "answer_accuracy": 0.5603916755806846,
875
+ "avg_response_len": 373.75660194452934,
876
+ "resp_tokens_sum": 156535245,
877
+ "pass_at_k": {
878
+ "pass@1": 0.5603916755806846,
879
+ "pass@2": 0.6303471699146198,
880
+ "pass@4": 0.6888534570690135,
881
+ "pass@8": 0.7364011139074038,
882
+ "pass@16": 0.7734842148415902,
883
+ "pass@32": 0.8019971811008832,
884
+ "pass@64": 0.825036901704703,
885
+ "pass@128": 0.844437652811736
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.9642857142857143,
890
+ "pass@2": 0.9765787089113863,
891
+ "pass@4": 0.9820486778438408,
892
+ "pass@8": 0.9842835869687955,
893
+ "pass@16": 0.9852484301067794,
894
+ "pass@32": 0.9863141673301701,
895
+ "pass@64": 0.9881014873140843,
896
+ "pass@128": 0.9904761904761905
897
+ },
898
+ "12": {
899
+ "pass@1": 0.9083763379204893,
900
+ "pass@2": 0.9310500644128201,
901
+ "pass@4": 0.9455652481513205,
902
+ "pass@8": 0.956169735463612,
903
+ "pass@16": 0.9645099947486562,
904
+ "pass@32": 0.9711630837875163,
905
+ "pass@64": 0.9769114489739431,
906
+ "pass@128": 0.981651376146789
907
+ },
908
+ "13": {
909
+ "pass@1": 0.8012393904320988,
910
+ "pass@2": 0.8506990011665209,
911
+ "pass@4": 0.8808248887870499,
912
+ "pass@8": 0.9035617488598224,
913
+ "pass@16": 0.9217366232932731,
914
+ "pass@32": 0.9348758028470283,
915
+ "pass@64": 0.9448526103401694,
916
+ "pass@128": 0.9506172839506173
917
+ },
918
+ "14": {
919
+ "pass@1": 0.7654395400593472,
920
+ "pass@2": 0.8289044428608145,
921
+ "pass@4": 0.8680753477773732,
922
+ "pass@8": 0.8935025847812458,
923
+ "pass@16": 0.9102918510356471,
924
+ "pass@32": 0.9225928362575211,
925
+ "pass@64": 0.934536393956455,
926
+ "pass@128": 0.9465875370919882
927
+ },
928
+ "15": {
929
+ "pass@1": 0.6340663580246914,
930
+ "pass@2": 0.7118602362204722,
931
+ "pass@4": 0.7674899159017153,
932
+ "pass@8": 0.8127011283278409,
933
+ "pass@16": 0.849350961539353,
934
+ "pass@32": 0.8759008146277355,
935
+ "pass@64": 0.892354569021042,
936
+ "pass@128": 0.9012345679012346
937
+ },
938
+ "16": {
939
+ "pass@1": 0.4710084808259587,
940
+ "pass@2": 0.580898471070541,
941
+ "pass@4": 0.668739560762868,
942
+ "pass@8": 0.7335904176217118,
943
+ "pass@16": 0.7809455072412861,
944
+ "pass@32": 0.8166722390431593,
945
+ "pass@64": 0.8488481347262333,
946
+ "pass@128": 0.8849557522123894
947
+ },
948
+ "17": {
949
+ "pass@1": 0.36243411144578314,
950
+ "pass@2": 0.4642222927141631,
951
+ "pass@4": 0.5592205760612748,
952
+ "pass@8": 0.6385525287524542,
953
+ "pass@16": 0.6994129926552873,
954
+ "pass@32": 0.7462385573378134,
955
+ "pass@64": 0.7809706376082151,
956
+ "pass@128": 0.8042168674698795
957
+ },
958
+ "18": {
959
+ "pass@1": 0.31597782258064516,
960
+ "pass@2": 0.4188174688849377,
961
+ "pass@4": 0.5104581848639886,
962
+ "pass@8": 0.5790573511639918,
963
+ "pass@16": 0.6283327210915123,
964
+ "pass@32": 0.6653820450438427,
965
+ "pass@64": 0.6937567396642746,
966
+ "pass@128": 0.7161290322580646
967
+ },
968
+ "19": {
969
+ "pass@1": 0.2052256419939577,
970
+ "pass@2": 0.28584974248876,
971
+ "pass@4": 0.36753850377841757,
972
+ "pass@8": 0.44054975863490586,
973
+ "pass@16": 0.49795327570192016,
974
+ "pass@32": 0.5453148986286661,
975
+ "pass@64": 0.5923666332779112,
976
+ "pass@128": 0.6404833836858006
977
+ },
978
+ "20": {
979
+ "pass@1": 0.19200450450450451,
980
+ "pass@2": 0.26821087820103573,
981
+ "pass@4": 0.3499301157175173,
982
+ "pass@8": 0.4312063902239929,
983
+ "pass@16": 0.5043251579531292,
984
+ "pass@32": 0.5612592194970087,
985
+ "pass@64": 0.6019373120540266,
986
+ "pass@128": 0.6306306306306306
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.7695604440789474,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.7695604440789474,
1002
+ "pass@2": 0.8071646614691095,
1003
+ "pass@4": 0.8378902844019319,
1004
+ "pass@8": 0.8625013052951703,
1005
+ "pass@16": 0.8817409478945274,
1006
+ "pass@32": 0.8969269202854718,
1007
+ "pass@64": 0.9096038465179878,
1008
+ "pass@128": 0.9207894736842105
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9524140625,
1013
+ "pass@2": 0.9645937500000002,
1014
+ "pass@4": 0.9730021943194608,
1015
+ "pass@8": 0.9786367820253195,
1016
+ "pass@16": 0.9831046505061921,
1017
+ "pass@32": 0.9868706993361868,
1018
+ "pass@64": 0.9899770280239669,
1019
+ "pass@128": 0.993
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 1.0,
1023
+ "pass@2": 1.0,
1024
+ "pass@4": 1.0,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 1.0,
1033
+ "pass@2": 1.0,
1034
+ "pass@4": 1.0,
1035
+ "pass@8": 1.0,
1036
+ "pass@16": 1.0,
1037
+ "pass@32": 1.0,
1038
+ "pass@64": 1.0,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9987109375,
1043
+ "pass@2": 0.999171998031496,
1044
+ "pass@4": 0.9994346713535808,
1045
+ "pass@8": 0.9996911656406087,
1046
+ "pass@16": 0.9999126144227602,
1047
+ "pass@32": 0.9999949322702752,
1048
+ "pass@64": 0.9999999977569969,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.993890625,
1053
+ "pass@2": 0.996357775590551,
1054
+ "pass@4": 0.9976947060367454,
1055
+ "pass@8": 0.9983995073818661,
1056
+ "pass@16": 0.9989712933683291,
1057
+ "pass@32": 0.9994882787743641,
1058
+ "pass@64": 0.9998748568874752,
1059
+ "pass@128": 1.0
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.997625,
1063
+ "pass@2": 0.9985620078740157,
1064
+ "pass@4": 0.9990585259655044,
1065
+ "pass@8": 0.9993178674757099,
1066
+ "pass@16": 0.9995588461117559,
1067
+ "pass@32": 0.9998290717621782,
1068
+ "pass@64": 0.9999861763011332,
1069
+ "pass@128": 1.0
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.997484375,
1073
+ "pass@2": 0.9984462352362204,
1074
+ "pass@4": 0.9990977215973003,
1075
+ "pass@8": 0.9995759792654791,
1076
+ "pass@16": 0.9998778554116803,
1077
+ "pass@32": 0.9999900984417422,
1078
+ "pass@64": 0.999999987930507,
1079
+ "pass@128": 1.0
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.995140625,
1083
+ "pass@2": 0.9973559301181102,
1084
+ "pass@4": 0.9985708786089239,
1085
+ "pass@8": 0.9991065163275745,
1086
+ "pass@16": 0.9993309358545857,
1087
+ "pass@32": 0.9995814582857677,
1088
+ "pass@64": 0.9998779527559055,
1089
+ "pass@128": 1.0
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.9940546875,
1093
+ "pass@2": 0.9959009596456694,
1094
+ "pass@4": 0.9966315292463443,
1095
+ "pass@8": 0.99717822839966,
1096
+ "pass@16": 0.9976478183722098,
1097
+ "pass@32": 0.9979294278134688,
1098
+ "pass@64": 0.9979985548661734,
1099
+ "pass@128": 0.998
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.9613984375,
1103
+ "pass@2": 0.9735031988188977,
1104
+ "pass@4": 0.9804231168916386,
1105
+ "pass@8": 0.9847109635922205,
1106
+ "pass@16": 0.9870854668143153,
1107
+ "pass@32": 0.9886758259692309,
1108
+ "pass@64": 0.9904900450431707,
1109
+ "pass@128": 0.993
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.899875,
1113
+ "pass@2": 0.9206909448818899,
1114
+ "pass@4": 0.9333334387889014,
1115
+ "pass@8": 0.9429585488390639,
1116
+ "pass@16": 0.9508381683007954,
1117
+ "pass@32": 0.957124237853498,
1118
+ "pass@64": 0.9620219505951612,
1119
+ "pass@128": 0.966
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.8238984375,
1123
+ "pass@2": 0.8724411909448823,
1124
+ "pass@4": 0.9021637046306712,
1125
+ "pass@8": 0.9231090688142323,
1126
+ "pass@16": 0.9379672291934675,
1127
+ "pass@32": 0.9484869565082493,
1128
+ "pass@64": 0.9568897147822485,
1129
+ "pass@128": 0.963
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.7794453125,
1133
+ "pass@2": 0.8445839074803148,
1134
+ "pass@4": 0.8855294000749906,
1135
+ "pass@8": 0.9114116915660494,
1136
+ "pass@16": 0.9273511399284904,
1137
+ "pass@32": 0.9386215902003806,
1138
+ "pass@64": 0.9495939611176091,
1139
+ "pass@128": 0.962
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.6651171875,
1143
+ "pass@2": 0.7486331200787407,
1144
+ "pass@4": 0.806329887701538,
1145
+ "pass@8": 0.8473808946123879,
1146
+ "pass@16": 0.8765889720497365,
1147
+ "pass@32": 0.8979515241106913,
1148
+ "pass@64": 0.9145665425705841,
1149
+ "pass@128": 0.928
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.4879609375,
1153
+ "pass@2": 0.5945366633858277,
1154
+ "pass@4": 0.6776242561867271,
1155
+ "pass@8": 0.739991947562845,
1156
+ "pass@16": 0.7879800006920792,
1157
+ "pass@32": 0.8266620166339039,
1158
+ "pass@64": 0.8592523951014696,
1159
+ "pass@128": 0.89
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.369609375,
1163
+ "pass@2": 0.47383956692913415,
1164
+ "pass@4": 0.566708111173604,
1165
+ "pass@8": 0.6421709788558302,
1166
+ "pass@16": 0.699630230918739,
1167
+ "pass@32": 0.7440501332532892,
1168
+ "pass@64": 0.7798027890890089,
1169
+ "pass@128": 0.811
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.3044921875,
1173
+ "pass@2": 0.402151697834646,
1174
+ "pass@4": 0.49101950562429714,
1175
+ "pass@8": 0.5642945200089501,
1176
+ "pass@16": 0.6220010627856662,
1177
+ "pass@32": 0.6671294229443815,
1178
+ "pass@64": 0.7039513297789204,
1179
+ "pass@128": 0.734
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.20428125,
1183
+ "pass@2": 0.28356422244094465,
1184
+ "pass@4": 0.363903641076115,
1185
+ "pass@8": 0.4365091889268906,
1186
+ "pass@16": 0.4967211171125234,
1187
+ "pass@32": 0.5470671625654399,
1188
+ "pass@64": 0.5925308099061153,
1189
+ "pass@128": 0.635
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.19625,
1193
+ "pass@2": 0.27179539862204705,
1194
+ "pass@4": 0.3493901143607049,
1195
+ "pass@8": 0.4230809513134006,
1196
+ "pass@16": 0.4885106081527199,
1197
+ "pass@32": 0.5421586487007338,
1198
+ "pass@64": 0.5856589913352467,
1199
+ "pass@128": 0.622
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9524140625,
1204
+ "2": 1.0,
1205
+ "3": 1.0,
1206
+ "4": 0.9987109375,
1207
+ "5": 0.993890625,
1208
+ "6": 0.997625,
1209
+ "7": 0.997484375,
1210
+ "8": 0.995140625,
1211
+ "9": 0.9940546875,
1212
+ "11": 0.9613984375,
1213
+ "12": 0.899875,
1214
+ "13": 0.8238984375,
1215
+ "14": 0.7794453125,
1216
+ "15": 0.6651171875,
1217
+ "16": 0.4879609375,
1218
+ "17": 0.369609375,
1219
+ "18": 0.3044921875,
1220
+ "19": 0.20428125,
1221
+ "20": 0.19625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 279.8089140625,
1225
+ "2": 115.4520703125,
1226
+ "3": 144.40334375,
1227
+ "4": 159.6451328125,
1228
+ "5": 172.749,
1229
+ "6": 198.4754375,
1230
+ "7": 223.35846875,
1231
+ "8": 242.756265625,
1232
+ "9": 258.5096484375,
1233
+ "11": 297.492234375,
1234
+ "12": 307.809671875,
1235
+ "13": 326.856828125,
1236
+ "14": 351.034875,
1237
+ "15": 362.367296875,
1238
+ "16": 361.9160234375,
1239
+ "17": 375.5798671875,
1240
+ "18": 376.86175,
1241
+ "19": 367.4975859375,
1242
+ "20": 379.247375
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 279.8089140625,
1247
+ "2": 115.4520703125,
1248
+ "3": 144.40334375,
1249
+ "4": 159.6451328125,
1250
+ "5": 172.749,
1251
+ "6": 198.4754375,
1252
+ "7": 223.35846875,
1253
+ "8": 242.756265625,
1254
+ "9": 258.5096484375,
1255
+ "11": 297.492234375,
1256
+ "12": 307.809671875,
1257
+ "13": 326.856828125,
1258
+ "14": 351.034875,
1259
+ "15": 362.367296875,
1260
+ "16": 361.9160234375,
1261
+ "17": 375.5798671875,
1262
+ "18": 376.86175,
1263
+ "19": 367.4975859375,
1264
+ "20": 379.247375
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 621973,
1271
+ "answer_accuracy": 0.7681258397881758,
1272
+ "avg_response_len": 265.83570408828643,
1273
+ "resp_tokens_sum": 215254613,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.7681258397881758,
1276
+ "pass@2": 0.8056889833794223,
1277
+ "pass@4": 0.8361066637487128,
1278
+ "pass@8": 0.8603037310757391,
1279
+ "pass@16": 0.8792181006261285,
1280
+ "pass@32": 0.8943473727278064,
1281
+ "pass@64": 0.9074203405159689,
1282
+ "pass@128": 0.9190641795763516
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9574575958702065,
1287
+ "pass@2": 0.9697600196269713,
1288
+ "pass@4": 0.9770701256369501,
1289
+ "pass@8": 0.9807352442070735,
1290
+ "pass@16": 0.9835831385400656,
1291
+ "pass@32": 0.9868139186902647,
1292
+ "pass@64": 0.9902929813385202,
1293
+ "pass@128": 0.9941002949852508
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 1.0,
1297
+ "pass@2": 1.0,
1298
+ "pass@4": 1.0,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 1.0,
1307
+ "pass@2": 1.0,
1308
+ "pass@4": 1.0,
1309
+ "pass@8": 1.0,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.996923828125,
1317
+ "pass@2": 0.9976178026574803,
1318
+ "pass@4": 0.9982453347628422,
1319
+ "pass@8": 0.9990349229744625,
1320
+ "pass@16": 0.9997269200711651,
1321
+ "pass@32": 0.9999841633446103,
1322
+ "pass@64": 0.9999999929906153,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9954861111111111,
1327
+ "pass@2": 0.9981666354205724,
1328
+ "pass@4": 0.9995852298224628,
1329
+ "pass@8": 0.9999710725449563,
1330
+ "pass@16": 0.9999998592866416,
1331
+ "pass@32": 0.9999999999993872,
1332
+ "pass@64": 1.0,
1333
+ "pass@128": 1.0
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9968890344311377,
1337
+ "pass@2": 0.997264575180348,
1338
+ "pass@4": 0.997534851556729,
1339
+ "pass@8": 0.9979835642250304,
1340
+ "pass@16": 0.9986793120796699,
1341
+ "pass@32": 0.9994882388094946,
1342
+ "pass@64": 0.9999586116800393,
1343
+ "pass@128": 1.0
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9973187869822485,
1347
+ "pass@2": 0.9984067785258351,
1348
+ "pass@4": 0.9992237471795316,
1349
+ "pass@8": 0.9998073811974663,
1350
+ "pass@16": 0.9999900064095892,
1351
+ "pass@32": 0.9999999886267649,
1352
+ "pass@64": 1.0,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9983974358974359,
1357
+ "pass@2": 0.999743290051484,
1358
+ "pass@4": 0.9999842372467864,
1359
+ "pass@8": 0.9999999472370541,
1360
+ "pass@16": 0.9999999999998607,
1361
+ "pass@32": 1.0,
1362
+ "pass@64": 1.0,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9957526408450704,
1367
+ "pass@2": 0.9972687007874016,
1368
+ "pass@4": 0.997876473539399,
1369
+ "pass@8": 0.9984546377928358,
1370
+ "pass@16": 0.999188066655105,
1371
+ "pass@32": 0.9998084093975909,
1372
+ "pass@64": 0.9999959303192606,
1373
+ "pass@128": 1.0
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9572193696883853,
1377
+ "pass@2": 0.9719146628449066,
1378
+ "pass@4": 0.981724504514839,
1379
+ "pass@8": 0.9890359200866053,
1380
+ "pass@16": 0.9932697779011666,
1381
+ "pass@32": 0.9957361168730702,
1382
+ "pass@64": 0.9978415915007174,
1383
+ "pass@128": 1.0
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8983271539548022,
1387
+ "pass@2": 0.920239098158281,
1388
+ "pass@4": 0.9334048053315367,
1389
+ "pass@8": 0.9434168617466765,
1390
+ "pass@16": 0.9517176538857137,
1391
+ "pass@32": 0.9580437536392892,
1392
+ "pass@64": 0.961533417356335,
1393
+ "pass@128": 0.963276836158192
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.8390722962382445,
1397
+ "pass@2": 0.8873212765038385,
1398
+ "pass@4": 0.9195430402704361,
1399
+ "pass@8": 0.9430086769866958,
1400
+ "pass@16": 0.9574399410309512,
1401
+ "pass@32": 0.9647881464197946,
1402
+ "pass@64": 0.9690691600729979,
1403
+ "pass@128": 0.9717868338557993
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.7840975467289719,
1407
+ "pass@2": 0.8466142192214293,
1408
+ "pass@4": 0.8872996447873914,
1409
+ "pass@8": 0.9131255212050312,
1410
+ "pass@16": 0.9296828208789675,
1411
+ "pass@32": 0.9427319318749825,
1412
+ "pass@64": 0.9554954405969336,
1413
+ "pass@128": 0.9688473520249221
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.693099710982659,
1417
+ "pass@2": 0.7778968583587461,
1418
+ "pass@4": 0.833728548292012,
1419
+ "pass@8": 0.8711126510602295,
1420
+ "pass@16": 0.8958576584940476,
1421
+ "pass@32": 0.9133306283181201,
1422
+ "pass@64": 0.9284685661529859,
1423
+ "pass@128": 0.9421965317919075
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.4913054435483871,
1427
+ "pass@2": 0.5962447612395223,
1428
+ "pass@4": 0.6747601975156323,
1429
+ "pass@8": 0.7348407821560055,
1430
+ "pass@16": 0.7834148610294204,
1431
+ "pass@32": 0.8244895329338998,
1432
+ "pass@64": 0.8601964216978978,
1433
+ "pass@128": 0.8903225806451613
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.35248825867052025,
1437
+ "pass@2": 0.4572849580128351,
1438
+ "pass@4": 0.5492719623919837,
1439
+ "pass@8": 0.6229445136286828,
1440
+ "pass@16": 0.6790291809157839,
1441
+ "pass@32": 0.7227658365318381,
1442
+ "pass@64": 0.7602209527875393,
1443
+ "pass@128": 0.7976878612716763
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.27520089285714283,
1447
+ "pass@2": 0.3652481721034869,
1448
+ "pass@4": 0.4477995007766887,
1449
+ "pass@8": 0.5185924991538704,
1450
+ "pass@16": 0.5777052275710203,
1451
+ "pass@32": 0.6248875096332369,
1452
+ "pass@64": 0.6638893307765378,
1453
+ "pass@128": 0.6971428571428572
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.2045795795795796,
1457
+ "pass@2": 0.28665747440353734,
1458
+ "pass@4": 0.368495601338121,
1459
+ "pass@8": 0.44158292317157055,
1460
+ "pass@16": 0.5021246498473418,
1461
+ "pass@32": 0.5528371750040144,
1462
+ "pass@64": 0.5987933372999518,
1463
+ "pass@128": 0.6396396396396397
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.19566441441441443,
1467
+ "pass@2": 0.2724017078101725,
1468
+ "pass@4": 0.34911851418347484,
1469
+ "pass@8": 0.41767593951939913,
1470
+ "pass@16": 0.47676148728079454,
1471
+ "pass@32": 0.5280550430504534,
1472
+ "pass@64": 0.5749912993919669,
1473
+ "pass@128": 0.6156156156156156
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 635595,
1480
+ "answer_accuracy": 0.774058602883866,
1481
+ "avg_response_len": 275.4775172934528,
1482
+ "resp_tokens_sum": 226200099,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.774058602883866,
1485
+ "pass@2": 0.8115189700566369,
1486
+ "pass@4": 0.8419047259145173,
1487
+ "pass@8": 0.8661518210454625,
1488
+ "pass@16": 0.8851545001914215,
1489
+ "pass@32": 0.9003565916378781,
1490
+ "pass@64": 0.9130658000646252,
1491
+ "pass@128": 0.9247077162899454
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9421731211656442,
1496
+ "pass@2": 0.9570635174629243,
1497
+ "pass@4": 0.9675180118397778,
1498
+ "pass@8": 0.9744761195433342,
1499
+ "pass@16": 0.9791587744555608,
1500
+ "pass@32": 0.9822004591222739,
1501
+ "pass@64": 0.984571936009979,
1502
+ "pass@128": 0.9877300613496932
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 1.0,
1506
+ "pass@2": 1.0,
1507
+ "pass@4": 1.0,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 1.0,
1516
+ "pass@2": 1.0,
1517
+ "pass@4": 1.0,
1518
+ "pass@8": 1.0,
1519
+ "pass@16": 1.0,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9999306582840237,
1526
+ "pass@2": 0.9999989080044729,
1527
+ "pass@4": 1.0,
1528
+ "pass@8": 1.0,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9956295289855073,
1536
+ "pass@2": 0.9968182985279016,
1537
+ "pass@4": 0.9973139007080636,
1538
+ "pass@8": 0.9976146824957167,
1539
+ "pass@16": 0.998064779875966,
1540
+ "pass@32": 0.9987868364280551,
1541
+ "pass@64": 0.9996462398721899,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9972242120343839,
1546
+ "pass@2": 0.9987002374613632,
1547
+ "pass@4": 0.9996714137452016,
1548
+ "pass@8": 0.9999752561101265,
1549
+ "pass@16": 0.9999998735735035,
1550
+ "pass@32": 0.999999999999447,
1551
+ "pass@64": 1.0,
1552
+ "pass@128": 1.0
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9973807803468208,
1556
+ "pass@2": 0.9977498748350099,
1557
+ "pass@4": 0.9982556172530457,
1558
+ "pass@8": 0.9989655495774096,
1559
+ "pass@16": 0.9996567447729243,
1560
+ "pass@32": 0.999971393889872,
1561
+ "pass@64": 0.9999999651170723,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9954665492957746,
1566
+ "pass@2": 0.9968725740268383,
1567
+ "pass@4": 0.9974011850631346,
1568
+ "pass@8": 0.9976822657434412,
1569
+ "pass@16": 0.9981192931743675,
1570
+ "pass@32": 0.9988210100498002,
1571
+ "pass@64": 0.9996562049462128,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9930295658682635,
1576
+ "pass@2": 0.9939206233202886,
1577
+ "pass@4": 0.9940102609741148,
1578
+ "pass@8": 0.9940119756217468,
1579
+ "pass@16": 0.9940119760479041,
1580
+ "pass@32": 0.9940119760479041,
1581
+ "pass@64": 0.9940119760479041,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9631024096385542,
1586
+ "pass@2": 0.972274185323973,
1587
+ "pass@4": 0.9774970881840069,
1588
+ "pass@8": 0.9805179334532464,
1589
+ "pass@16": 0.9822529509986989,
1590
+ "pass@32": 0.9834096807350399,
1591
+ "pass@64": 0.9849397431912068,
1592
+ "pass@128": 0.9879518072289156
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8928781347962382,
1596
+ "pass@2": 0.9105734579024021,
1597
+ "pass@4": 0.920715678859108,
1598
+ "pass@8": 0.928907449166582,
1599
+ "pass@16": 0.9358474938634546,
1600
+ "pass@32": 0.9417129174503823,
1601
+ "pass@64": 0.9473011819327243,
1602
+ "pass@128": 0.9529780564263323
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.8309042366946778,
1606
+ "pass@2": 0.8788773875692008,
1607
+ "pass@4": 0.9060005905249233,
1608
+ "pass@8": 0.9230680510501225,
1609
+ "pass@16": 0.9352975407775181,
1610
+ "pass@32": 0.9462738870529341,
1611
+ "pass@64": 0.956931111957274,
1612
+ "pass@128": 0.9663865546218487
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.7888797514619883,
1616
+ "pass@2": 0.8581284966846247,
1617
+ "pass@4": 0.9010667306937505,
1618
+ "pass@8": 0.9274503748185816,
1619
+ "pass@16": 0.9419725164541798,
1620
+ "pass@32": 0.9505580533617753,
1621
+ "pass@64": 0.9588922804756361,
1622
+ "pass@128": 0.9707602339181286
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.6662642045454545,
1626
+ "pass@2": 0.7540548198520639,
1627
+ "pass@4": 0.8157365976980145,
1628
+ "pass@8": 0.8565477326282641,
1629
+ "pass@16": 0.8831288202183204,
1630
+ "pass@32": 0.9034765538582886,
1631
+ "pass@64": 0.9217986009661605,
1632
+ "pass@128": 0.9393939393939394
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.5013799857549858,
1636
+ "pass@2": 0.6062000162639922,
1637
+ "pass@4": 0.688734711846916,
1638
+ "pass@8": 0.7507240841046224,
1639
+ "pass@16": 0.7988058883708389,
1640
+ "pass@32": 0.8382289811645698,
1641
+ "pass@64": 0.8684671985838386,
1642
+ "pass@128": 0.8945868945868946
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.39540469720496896,
1646
+ "pass@2": 0.5015440070670513,
1647
+ "pass@4": 0.5931639159429605,
1648
+ "pass@8": 0.6665612968773015,
1649
+ "pass@16": 0.7219907477028704,
1650
+ "pass@32": 0.7646644496183821,
1651
+ "pass@64": 0.7996400240328937,
1652
+ "pass@128": 0.8322981366459627
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.32417279411764705,
1656
+ "pass@2": 0.42494535954145407,
1657
+ "pass@4": 0.517787185425351,
1658
+ "pass@8": 0.5978804895419353,
1659
+ "pass@16": 0.6618267341101194,
1660
+ "pass@32": 0.7122069429681127,
1661
+ "pass@64": 0.7544866903270767,
1662
+ "pass@128": 0.788235294117647
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.20305524553571427,
1666
+ "pass@2": 0.2782470794666291,
1667
+ "pass@4": 0.35577190797578867,
1668
+ "pass@8": 0.42750031369822505,
1669
+ "pass@16": 0.4901520073095929,
1670
+ "pass@32": 0.5430748566756388,
1671
+ "pass@64": 0.5864859314590565,
1672
+ "pass@128": 0.625
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.20106661676646706,
1676
+ "pass@2": 0.2747646930548353,
1677
+ "pass@4": 0.3491225167774687,
1678
+ "pass@8": 0.42036866918817706,
1679
+ "pass@16": 0.4844573511671379,
1680
+ "pass@32": 0.5371766445282311,
1681
+ "pass@64": 0.5800651608494923,
1682
+ "pass@128": 0.6197604790419161
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 614003,
1689
+ "answer_accuracy": 0.7664001338073174,
1690
+ "avg_response_len": 296.0467888740214,
1691
+ "resp_tokens_sum": 237178477,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.7664001338073174,
1694
+ "pass@2": 0.8041933002303396,
1695
+ "pass@4": 0.8355785000832348,
1696
+ "pass@8": 0.8609809020316768,
1697
+ "pass@16": 0.8807921691495971,
1698
+ "pass@32": 0.8960189279742334,
1699
+ "pass@64": 0.9082624863913064,
1700
+ "pass@128": 0.918517335037546
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9572761194029851,
1705
+ "pass@2": 0.9666937213538606,
1706
+ "pass@4": 0.9742225369216908,
1707
+ "pass@8": 0.9805621470686411,
1708
+ "pass@16": 0.9864603166226772,
1709
+ "pass@32": 0.991472929033808,
1710
+ "pass@64": 0.9949171827192708,
1711
+ "pass@128": 0.9970149253731343
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 1.0,
1715
+ "pass@2": 1.0,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 1.0,
1725
+ "pass@2": 1.0,
1726
+ "pass@4": 1.0,
1727
+ "pass@8": 1.0,
1728
+ "pass@16": 1.0,
1729
+ "pass@32": 1.0,
1730
+ "pass@64": 1.0,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9991776315789473,
1735
+ "pass@2": 0.9998089774140073,
1736
+ "pass@4": 0.9999887842966998,
1737
+ "pass@8": 0.9999999716046218,
1738
+ "pass@16": 0.9999999999999635,
1739
+ "pass@32": 1.0,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9906479779411764,
1745
+ "pass@2": 0.9942146248263086,
1746
+ "pass@4": 0.996329596759964,
1747
+ "pass@8": 0.9977398649093597,
1748
+ "pass@16": 0.9989382018700852,
1749
+ "pass@32": 0.9997259417849353,
1750
+ "pass@64": 0.9999908945046169,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9988416798107256,
1755
+ "pass@2": 0.9997768356143968,
1756
+ "pass@4": 0.9999891550425282,
1757
+ "pass@8": 0.9999999767890079,
1758
+ "pass@16": 0.9999999999999797,
1759
+ "pass@32": 1.0,
1760
+ "pass@64": 1.0,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.997774920886076,
1765
+ "pass@2": 0.9992509094986542,
1766
+ "pass@4": 0.9998849730413445,
1767
+ "pass@8": 0.9999968489460502,
1768
+ "pass@16": 0.9999999985879974,
1769
+ "pass@32": 1.0,
1770
+ "pass@64": 1.0,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9917417417417418,
1775
+ "pass@2": 0.9956344139415007,
1776
+ "pass@4": 0.998493621292834,
1777
+ "pass@8": 0.999787773125201,
1778
+ "pass@16": 0.9999957560892757,
1779
+ "pass@32": 0.9999999991534195,
1780
+ "pass@64": 1.0,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9932174437299035,
1785
+ "pass@2": 0.9964665037850975,
1786
+ "pass@4": 0.9980255754807175,
1787
+ "pass@8": 0.9991216467059164,
1788
+ "pass@16": 0.9997943881339146,
1789
+ "pass@32": 0.9999917764544182,
1790
+ "pass@64": 0.9999999987229452,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.9642857142857143,
1795
+ "pass@2": 0.9765787089113863,
1796
+ "pass@4": 0.9820486778438408,
1797
+ "pass@8": 0.9842835869687955,
1798
+ "pass@16": 0.9852484301067794,
1799
+ "pass@32": 0.9863141673301701,
1800
+ "pass@64": 0.9881014873140843,
1801
+ "pass@128": 0.9904761904761905
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.9083763379204893,
1805
+ "pass@2": 0.9310500644128201,
1806
+ "pass@4": 0.9455652481513205,
1807
+ "pass@8": 0.956169735463612,
1808
+ "pass@16": 0.9645099947486562,
1809
+ "pass@32": 0.9711630837875163,
1810
+ "pass@64": 0.9769114489739431,
1811
+ "pass@128": 0.981651376146789
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.8012393904320988,
1815
+ "pass@2": 0.8506990011665209,
1816
+ "pass@4": 0.8808248887870499,
1817
+ "pass@8": 0.9035617488598224,
1818
+ "pass@16": 0.9217366232932731,
1819
+ "pass@32": 0.9348758028470283,
1820
+ "pass@64": 0.9448526103401694,
1821
+ "pass@128": 0.9506172839506173
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.7654395400593472,
1825
+ "pass@2": 0.8289044428608145,
1826
+ "pass@4": 0.8680753477773732,
1827
+ "pass@8": 0.8935025847812458,
1828
+ "pass@16": 0.9102918510356471,
1829
+ "pass@32": 0.9225928362575211,
1830
+ "pass@64": 0.934536393956455,
1831
+ "pass@128": 0.9465875370919882
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.6340663580246914,
1835
+ "pass@2": 0.7118602362204722,
1836
+ "pass@4": 0.7674899159017153,
1837
+ "pass@8": 0.8127011283278409,
1838
+ "pass@16": 0.849350961539353,
1839
+ "pass@32": 0.8759008146277355,
1840
+ "pass@64": 0.892354569021042,
1841
+ "pass@128": 0.9012345679012346
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.4710084808259587,
1845
+ "pass@2": 0.580898471070541,
1846
+ "pass@4": 0.668739560762868,
1847
+ "pass@8": 0.7335904176217118,
1848
+ "pass@16": 0.7809455072412861,
1849
+ "pass@32": 0.8166722390431593,
1850
+ "pass@64": 0.8488481347262333,
1851
+ "pass@128": 0.8849557522123894
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.36243411144578314,
1855
+ "pass@2": 0.4642222927141631,
1856
+ "pass@4": 0.5592205760612748,
1857
+ "pass@8": 0.6385525287524542,
1858
+ "pass@16": 0.6994129926552873,
1859
+ "pass@32": 0.7462385573378134,
1860
+ "pass@64": 0.7809706376082151,
1861
+ "pass@128": 0.8042168674698795
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.31597782258064516,
1865
+ "pass@2": 0.4188174688849377,
1866
+ "pass@4": 0.5104581848639886,
1867
+ "pass@8": 0.5790573511639918,
1868
+ "pass@16": 0.6283327210915123,
1869
+ "pass@32": 0.6653820450438427,
1870
+ "pass@64": 0.6937567396642746,
1871
+ "pass@128": 0.7161290322580646
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.2052256419939577,
1875
+ "pass@2": 0.28584974248876,
1876
+ "pass@4": 0.36753850377841757,
1877
+ "pass@8": 0.44054975863490586,
1878
+ "pass@16": 0.49795327570192016,
1879
+ "pass@32": 0.5453148986286661,
1880
+ "pass@64": 0.5923666332779112,
1881
+ "pass@128": 0.6404833836858006
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.19200450450450451,
1885
+ "pass@2": 0.26821087820103573,
1886
+ "pass@4": 0.3499301157175173,
1887
+ "pass@8": 0.4312063902239929,
1888
+ "pass@16": 0.5043251579531292,
1889
+ "pass@32": 0.5612592194970087,
1890
+ "pass@64": 0.6019373120540266,
1891
+ "pass@128": 0.6306306306306306
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block16-epoch4-op9-12/SDAR-100M-block16-epoch4-op9-12_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1290247a50d2ec6142ba64ff7c2f58e783799cc1cf88c8770d0d1e4bdf9569bd
3
+ size 2641428571
results/composition/SDAR-100M-block16-epoch4-op9-12/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block16-epoch4-op9-12,0.9921467013888889,0.5692328125,0.7695604440789474,0.0,0.0,0.0,199.46203125,350.66635078125,1152000,1280000,2432000
results/composition/SDAR-100M-block16-epoch4-op9-12/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block16-epoch4-op9-12",
4
+ "id_acc": 0.9921467013888889,
5
+ "ood_acc": 0.5692328125,
6
+ "total_acc": 0.7695604440789474,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.46203125,
11
+ "ood_avg_resp_len": 350.66635078125,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 621973,
19
+ "answer_accuracy": 0.7681258397881758,
20
+ "avg_response_len": 265.83570408828643,
21
+ "resp_tokens_sum": 215254613,
22
+ "pass_at_k": {
23
+ "pass@1": 0.7681258397881758,
24
+ "pass@2": 0.8056889833794223,
25
+ "pass@4": 0.8361066637487128,
26
+ "pass@8": 0.8603037310757391,
27
+ "pass@16": 0.8792181006261285,
28
+ "pass@32": 0.8943473727278064,
29
+ "pass@64": 0.9074203405159689,
30
+ "pass@128": 0.9190641795763516
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9574575958702065,
35
+ "pass@2": 0.9697600196269713,
36
+ "pass@4": 0.9770701256369501,
37
+ "pass@8": 0.9807352442070735,
38
+ "pass@16": 0.9835831385400656,
39
+ "pass@32": 0.9868139186902647,
40
+ "pass@64": 0.9902929813385202,
41
+ "pass@128": 0.9941002949852508
42
+ },
43
+ "2": {
44
+ "pass@1": 1.0,
45
+ "pass@2": 1.0,
46
+ "pass@4": 1.0,
47
+ "pass@8": 1.0,
48
+ "pass@16": 1.0,
49
+ "pass@32": 1.0,
50
+ "pass@64": 1.0,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 1.0,
55
+ "pass@2": 1.0,
56
+ "pass@4": 1.0,
57
+ "pass@8": 1.0,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.996923828125,
65
+ "pass@2": 0.9976178026574803,
66
+ "pass@4": 0.9982453347628422,
67
+ "pass@8": 0.9990349229744625,
68
+ "pass@16": 0.9997269200711651,
69
+ "pass@32": 0.9999841633446103,
70
+ "pass@64": 0.9999999929906153,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9954861111111111,
75
+ "pass@2": 0.9981666354205724,
76
+ "pass@4": 0.9995852298224628,
77
+ "pass@8": 0.9999710725449563,
78
+ "pass@16": 0.9999998592866416,
79
+ "pass@32": 0.9999999999993872,
80
+ "pass@64": 1.0,
81
+ "pass@128": 1.0
82
+ },
83
+ "6": {
84
+ "pass@1": 0.9968890344311377,
85
+ "pass@2": 0.997264575180348,
86
+ "pass@4": 0.997534851556729,
87
+ "pass@8": 0.9979835642250304,
88
+ "pass@16": 0.9986793120796699,
89
+ "pass@32": 0.9994882388094946,
90
+ "pass@64": 0.9999586116800393,
91
+ "pass@128": 1.0
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9973187869822485,
95
+ "pass@2": 0.9984067785258351,
96
+ "pass@4": 0.9992237471795316,
97
+ "pass@8": 0.9998073811974663,
98
+ "pass@16": 0.9999900064095892,
99
+ "pass@32": 0.9999999886267649,
100
+ "pass@64": 1.0,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9983974358974359,
105
+ "pass@2": 0.999743290051484,
106
+ "pass@4": 0.9999842372467864,
107
+ "pass@8": 0.9999999472370541,
108
+ "pass@16": 0.9999999999998607,
109
+ "pass@32": 1.0,
110
+ "pass@64": 1.0,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9957526408450704,
115
+ "pass@2": 0.9972687007874016,
116
+ "pass@4": 0.997876473539399,
117
+ "pass@8": 0.9984546377928358,
118
+ "pass@16": 0.999188066655105,
119
+ "pass@32": 0.9998084093975909,
120
+ "pass@64": 0.9999959303192606,
121
+ "pass@128": 1.0
122
+ },
123
+ "11": {
124
+ "pass@1": 0.9572193696883853,
125
+ "pass@2": 0.9719146628449066,
126
+ "pass@4": 0.981724504514839,
127
+ "pass@8": 0.9890359200866053,
128
+ "pass@16": 0.9932697779011666,
129
+ "pass@32": 0.9957361168730702,
130
+ "pass@64": 0.9978415915007174,
131
+ "pass@128": 1.0
132
+ },
133
+ "12": {
134
+ "pass@1": 0.8983271539548022,
135
+ "pass@2": 0.920239098158281,
136
+ "pass@4": 0.9334048053315367,
137
+ "pass@8": 0.9434168617466765,
138
+ "pass@16": 0.9517176538857137,
139
+ "pass@32": 0.9580437536392892,
140
+ "pass@64": 0.961533417356335,
141
+ "pass@128": 0.963276836158192
142
+ },
143
+ "13": {
144
+ "pass@1": 0.8390722962382445,
145
+ "pass@2": 0.8873212765038385,
146
+ "pass@4": 0.9195430402704361,
147
+ "pass@8": 0.9430086769866958,
148
+ "pass@16": 0.9574399410309512,
149
+ "pass@32": 0.9647881464197946,
150
+ "pass@64": 0.9690691600729979,
151
+ "pass@128": 0.9717868338557993
152
+ },
153
+ "14": {
154
+ "pass@1": 0.7840975467289719,
155
+ "pass@2": 0.8466142192214293,
156
+ "pass@4": 0.8872996447873914,
157
+ "pass@8": 0.9131255212050312,
158
+ "pass@16": 0.9296828208789675,
159
+ "pass@32": 0.9427319318749825,
160
+ "pass@64": 0.9554954405969336,
161
+ "pass@128": 0.9688473520249221
162
+ },
163
+ "15": {
164
+ "pass@1": 0.693099710982659,
165
+ "pass@2": 0.7778968583587461,
166
+ "pass@4": 0.833728548292012,
167
+ "pass@8": 0.8711126510602295,
168
+ "pass@16": 0.8958576584940476,
169
+ "pass@32": 0.9133306283181201,
170
+ "pass@64": 0.9284685661529859,
171
+ "pass@128": 0.9421965317919075
172
+ },
173
+ "16": {
174
+ "pass@1": 0.4913054435483871,
175
+ "pass@2": 0.5962447612395223,
176
+ "pass@4": 0.6747601975156323,
177
+ "pass@8": 0.7348407821560055,
178
+ "pass@16": 0.7834148610294204,
179
+ "pass@32": 0.8244895329338998,
180
+ "pass@64": 0.8601964216978978,
181
+ "pass@128": 0.8903225806451613
182
+ },
183
+ "17": {
184
+ "pass@1": 0.35248825867052025,
185
+ "pass@2": 0.4572849580128351,
186
+ "pass@4": 0.5492719623919837,
187
+ "pass@8": 0.6229445136286828,
188
+ "pass@16": 0.6790291809157839,
189
+ "pass@32": 0.7227658365318381,
190
+ "pass@64": 0.7602209527875393,
191
+ "pass@128": 0.7976878612716763
192
+ },
193
+ "18": {
194
+ "pass@1": 0.27520089285714283,
195
+ "pass@2": 0.3652481721034869,
196
+ "pass@4": 0.4477995007766887,
197
+ "pass@8": 0.5185924991538704,
198
+ "pass@16": 0.5777052275710203,
199
+ "pass@32": 0.6248875096332369,
200
+ "pass@64": 0.6638893307765378,
201
+ "pass@128": 0.6971428571428572
202
+ },
203
+ "19": {
204
+ "pass@1": 0.2045795795795796,
205
+ "pass@2": 0.28665747440353734,
206
+ "pass@4": 0.368495601338121,
207
+ "pass@8": 0.44158292317157055,
208
+ "pass@16": 0.5021246498473418,
209
+ "pass@32": 0.5528371750040144,
210
+ "pass@64": 0.5987933372999518,
211
+ "pass@128": 0.6396396396396397
212
+ },
213
+ "20": {
214
+ "pass@1": 0.19566441441441443,
215
+ "pass@2": 0.2724017078101725,
216
+ "pass@4": 0.34911851418347484,
217
+ "pass@8": 0.41767593951939913,
218
+ "pass@16": 0.47676148728079454,
219
+ "pass@32": 0.5280550430504534,
220
+ "pass@64": 0.5749912993919669,
221
+ "pass@128": 0.6156156156156156
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 635595,
228
+ "answer_accuracy": 0.774058602883866,
229
+ "avg_response_len": 275.4775172934528,
230
+ "resp_tokens_sum": 226200099,
231
+ "pass_at_k": {
232
+ "pass@1": 0.774058602883866,
233
+ "pass@2": 0.8115189700566369,
234
+ "pass@4": 0.8419047259145173,
235
+ "pass@8": 0.8661518210454625,
236
+ "pass@16": 0.8851545001914215,
237
+ "pass@32": 0.9003565916378781,
238
+ "pass@64": 0.9130658000646252,
239
+ "pass@128": 0.9247077162899454
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9421731211656442,
244
+ "pass@2": 0.9570635174629243,
245
+ "pass@4": 0.9675180118397778,
246
+ "pass@8": 0.9744761195433342,
247
+ "pass@16": 0.9791587744555608,
248
+ "pass@32": 0.9822004591222739,
249
+ "pass@64": 0.984571936009979,
250
+ "pass@128": 0.9877300613496932
251
+ },
252
+ "2": {
253
+ "pass@1": 1.0,
254
+ "pass@2": 1.0,
255
+ "pass@4": 1.0,
256
+ "pass@8": 1.0,
257
+ "pass@16": 1.0,
258
+ "pass@32": 1.0,
259
+ "pass@64": 1.0,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 1.0,
264
+ "pass@2": 1.0,
265
+ "pass@4": 1.0,
266
+ "pass@8": 1.0,
267
+ "pass@16": 1.0,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9999306582840237,
274
+ "pass@2": 0.9999989080044729,
275
+ "pass@4": 1.0,
276
+ "pass@8": 1.0,
277
+ "pass@16": 1.0,
278
+ "pass@32": 1.0,
279
+ "pass@64": 1.0,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9956295289855073,
284
+ "pass@2": 0.9968182985279016,
285
+ "pass@4": 0.9973139007080636,
286
+ "pass@8": 0.9976146824957167,
287
+ "pass@16": 0.998064779875966,
288
+ "pass@32": 0.9987868364280551,
289
+ "pass@64": 0.9996462398721899,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9972242120343839,
294
+ "pass@2": 0.9987002374613632,
295
+ "pass@4": 0.9996714137452016,
296
+ "pass@8": 0.9999752561101265,
297
+ "pass@16": 0.9999998735735035,
298
+ "pass@32": 0.999999999999447,
299
+ "pass@64": 1.0,
300
+ "pass@128": 1.0
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9973807803468208,
304
+ "pass@2": 0.9977498748350099,
305
+ "pass@4": 0.9982556172530457,
306
+ "pass@8": 0.9989655495774096,
307
+ "pass@16": 0.9996567447729243,
308
+ "pass@32": 0.999971393889872,
309
+ "pass@64": 0.9999999651170723,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9954665492957746,
314
+ "pass@2": 0.9968725740268383,
315
+ "pass@4": 0.9974011850631346,
316
+ "pass@8": 0.9976822657434412,
317
+ "pass@16": 0.9981192931743675,
318
+ "pass@32": 0.9988210100498002,
319
+ "pass@64": 0.9996562049462128,
320
+ "pass@128": 1.0
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9930295658682635,
324
+ "pass@2": 0.9939206233202886,
325
+ "pass@4": 0.9940102609741148,
326
+ "pass@8": 0.9940119756217468,
327
+ "pass@16": 0.9940119760479041,
328
+ "pass@32": 0.9940119760479041,
329
+ "pass@64": 0.9940119760479041,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.9631024096385542,
334
+ "pass@2": 0.972274185323973,
335
+ "pass@4": 0.9774970881840069,
336
+ "pass@8": 0.9805179334532464,
337
+ "pass@16": 0.9822529509986989,
338
+ "pass@32": 0.9834096807350399,
339
+ "pass@64": 0.9849397431912068,
340
+ "pass@128": 0.9879518072289156
341
+ },
342
+ "12": {
343
+ "pass@1": 0.8928781347962382,
344
+ "pass@2": 0.9105734579024021,
345
+ "pass@4": 0.920715678859108,
346
+ "pass@8": 0.928907449166582,
347
+ "pass@16": 0.9358474938634546,
348
+ "pass@32": 0.9417129174503823,
349
+ "pass@64": 0.9473011819327243,
350
+ "pass@128": 0.9529780564263323
351
+ },
352
+ "13": {
353
+ "pass@1": 0.8309042366946778,
354
+ "pass@2": 0.8788773875692008,
355
+ "pass@4": 0.9060005905249233,
356
+ "pass@8": 0.9230680510501225,
357
+ "pass@16": 0.9352975407775181,
358
+ "pass@32": 0.9462738870529341,
359
+ "pass@64": 0.956931111957274,
360
+ "pass@128": 0.9663865546218487
361
+ },
362
+ "14": {
363
+ "pass@1": 0.7888797514619883,
364
+ "pass@2": 0.8581284966846247,
365
+ "pass@4": 0.9010667306937505,
366
+ "pass@8": 0.9274503748185816,
367
+ "pass@16": 0.9419725164541798,
368
+ "pass@32": 0.9505580533617753,
369
+ "pass@64": 0.9588922804756361,
370
+ "pass@128": 0.9707602339181286
371
+ },
372
+ "15": {
373
+ "pass@1": 0.6662642045454545,
374
+ "pass@2": 0.7540548198520639,
375
+ "pass@4": 0.8157365976980145,
376
+ "pass@8": 0.8565477326282641,
377
+ "pass@16": 0.8831288202183204,
378
+ "pass@32": 0.9034765538582886,
379
+ "pass@64": 0.9217986009661605,
380
+ "pass@128": 0.9393939393939394
381
+ },
382
+ "16": {
383
+ "pass@1": 0.5013799857549858,
384
+ "pass@2": 0.6062000162639922,
385
+ "pass@4": 0.688734711846916,
386
+ "pass@8": 0.7507240841046224,
387
+ "pass@16": 0.7988058883708389,
388
+ "pass@32": 0.8382289811645698,
389
+ "pass@64": 0.8684671985838386,
390
+ "pass@128": 0.8945868945868946
391
+ },
392
+ "17": {
393
+ "pass@1": 0.39540469720496896,
394
+ "pass@2": 0.5015440070670513,
395
+ "pass@4": 0.5931639159429605,
396
+ "pass@8": 0.6665612968773015,
397
+ "pass@16": 0.7219907477028704,
398
+ "pass@32": 0.7646644496183821,
399
+ "pass@64": 0.7996400240328937,
400
+ "pass@128": 0.8322981366459627
401
+ },
402
+ "18": {
403
+ "pass@1": 0.32417279411764705,
404
+ "pass@2": 0.42494535954145407,
405
+ "pass@4": 0.517787185425351,
406
+ "pass@8": 0.5978804895419353,
407
+ "pass@16": 0.6618267341101194,
408
+ "pass@32": 0.7122069429681127,
409
+ "pass@64": 0.7544866903270767,
410
+ "pass@128": 0.788235294117647
411
+ },
412
+ "19": {
413
+ "pass@1": 0.20305524553571427,
414
+ "pass@2": 0.2782470794666291,
415
+ "pass@4": 0.35577190797578867,
416
+ "pass@8": 0.42750031369822505,
417
+ "pass@16": 0.4901520073095929,
418
+ "pass@32": 0.5430748566756388,
419
+ "pass@64": 0.5864859314590565,
420
+ "pass@128": 0.625
421
+ },
422
+ "20": {
423
+ "pass@1": 0.20106661676646706,
424
+ "pass@2": 0.2747646930548353,
425
+ "pass@4": 0.3491225167774687,
426
+ "pass@8": 0.42036866918817706,
427
+ "pass@16": 0.4844573511671379,
428
+ "pass@32": 0.5371766445282311,
429
+ "pass@64": 0.5800651608494923,
430
+ "pass@128": 0.6197604790419161
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 614003,
437
+ "answer_accuracy": 0.7664001338073174,
438
+ "avg_response_len": 296.0467888740214,
439
+ "resp_tokens_sum": 237178477,
440
+ "pass_at_k": {
441
+ "pass@1": 0.7664001338073174,
442
+ "pass@2": 0.8041933002303396,
443
+ "pass@4": 0.8355785000832348,
444
+ "pass@8": 0.8609809020316768,
445
+ "pass@16": 0.8807921691495971,
446
+ "pass@32": 0.8960189279742334,
447
+ "pass@64": 0.9082624863913064,
448
+ "pass@128": 0.918517335037546
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9572761194029851,
453
+ "pass@2": 0.9666937213538606,
454
+ "pass@4": 0.9742225369216908,
455
+ "pass@8": 0.9805621470686411,
456
+ "pass@16": 0.9864603166226772,
457
+ "pass@32": 0.991472929033808,
458
+ "pass@64": 0.9949171827192708,
459
+ "pass@128": 0.9970149253731343
460
+ },
461
+ "2": {
462
+ "pass@1": 1.0,
463
+ "pass@2": 1.0,
464
+ "pass@4": 1.0,
465
+ "pass@8": 1.0,
466
+ "pass@16": 1.0,
467
+ "pass@32": 1.0,
468
+ "pass@64": 1.0,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 1.0,
473
+ "pass@2": 1.0,
474
+ "pass@4": 1.0,
475
+ "pass@8": 1.0,
476
+ "pass@16": 1.0,
477
+ "pass@32": 1.0,
478
+ "pass@64": 1.0,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9991776315789473,
483
+ "pass@2": 0.9998089774140073,
484
+ "pass@4": 0.9999887842966998,
485
+ "pass@8": 0.9999999716046218,
486
+ "pass@16": 0.9999999999999635,
487
+ "pass@32": 1.0,
488
+ "pass@64": 1.0,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.9906479779411764,
493
+ "pass@2": 0.9942146248263086,
494
+ "pass@4": 0.996329596759964,
495
+ "pass@8": 0.9977398649093597,
496
+ "pass@16": 0.9989382018700852,
497
+ "pass@32": 0.9997259417849353,
498
+ "pass@64": 0.9999908945046169,
499
+ "pass@128": 1.0
500
+ },
501
+ "6": {
502
+ "pass@1": 0.9988416798107256,
503
+ "pass@2": 0.9997768356143968,
504
+ "pass@4": 0.9999891550425282,
505
+ "pass@8": 0.9999999767890079,
506
+ "pass@16": 0.9999999999999797,
507
+ "pass@32": 1.0,
508
+ "pass@64": 1.0,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.997774920886076,
513
+ "pass@2": 0.9992509094986542,
514
+ "pass@4": 0.9998849730413445,
515
+ "pass@8": 0.9999968489460502,
516
+ "pass@16": 0.9999999985879974,
517
+ "pass@32": 1.0,
518
+ "pass@64": 1.0,
519
+ "pass@128": 1.0
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9917417417417418,
523
+ "pass@2": 0.9956344139415007,
524
+ "pass@4": 0.998493621292834,
525
+ "pass@8": 0.999787773125201,
526
+ "pass@16": 0.9999957560892757,
527
+ "pass@32": 0.9999999991534195,
528
+ "pass@64": 1.0,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9932174437299035,
533
+ "pass@2": 0.9964665037850975,
534
+ "pass@4": 0.9980255754807175,
535
+ "pass@8": 0.9991216467059164,
536
+ "pass@16": 0.9997943881339146,
537
+ "pass@32": 0.9999917764544182,
538
+ "pass@64": 0.9999999987229452,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.9642857142857143,
543
+ "pass@2": 0.9765787089113863,
544
+ "pass@4": 0.9820486778438408,
545
+ "pass@8": 0.9842835869687955,
546
+ "pass@16": 0.9852484301067794,
547
+ "pass@32": 0.9863141673301701,
548
+ "pass@64": 0.9881014873140843,
549
+ "pass@128": 0.9904761904761905
550
+ },
551
+ "12": {
552
+ "pass@1": 0.9083763379204893,
553
+ "pass@2": 0.9310500644128201,
554
+ "pass@4": 0.9455652481513205,
555
+ "pass@8": 0.956169735463612,
556
+ "pass@16": 0.9645099947486562,
557
+ "pass@32": 0.9711630837875163,
558
+ "pass@64": 0.9769114489739431,
559
+ "pass@128": 0.981651376146789
560
+ },
561
+ "13": {
562
+ "pass@1": 0.8012393904320988,
563
+ "pass@2": 0.8506990011665209,
564
+ "pass@4": 0.8808248887870499,
565
+ "pass@8": 0.9035617488598224,
566
+ "pass@16": 0.9217366232932731,
567
+ "pass@32": 0.9348758028470283,
568
+ "pass@64": 0.9448526103401694,
569
+ "pass@128": 0.9506172839506173
570
+ },
571
+ "14": {
572
+ "pass@1": 0.7654395400593472,
573
+ "pass@2": 0.8289044428608145,
574
+ "pass@4": 0.8680753477773732,
575
+ "pass@8": 0.8935025847812458,
576
+ "pass@16": 0.9102918510356471,
577
+ "pass@32": 0.9225928362575211,
578
+ "pass@64": 0.934536393956455,
579
+ "pass@128": 0.9465875370919882
580
+ },
581
+ "15": {
582
+ "pass@1": 0.6340663580246914,
583
+ "pass@2": 0.7118602362204722,
584
+ "pass@4": 0.7674899159017153,
585
+ "pass@8": 0.8127011283278409,
586
+ "pass@16": 0.849350961539353,
587
+ "pass@32": 0.8759008146277355,
588
+ "pass@64": 0.892354569021042,
589
+ "pass@128": 0.9012345679012346
590
+ },
591
+ "16": {
592
+ "pass@1": 0.4710084808259587,
593
+ "pass@2": 0.580898471070541,
594
+ "pass@4": 0.668739560762868,
595
+ "pass@8": 0.7335904176217118,
596
+ "pass@16": 0.7809455072412861,
597
+ "pass@32": 0.8166722390431593,
598
+ "pass@64": 0.8488481347262333,
599
+ "pass@128": 0.8849557522123894
600
+ },
601
+ "17": {
602
+ "pass@1": 0.36243411144578314,
603
+ "pass@2": 0.4642222927141631,
604
+ "pass@4": 0.5592205760612748,
605
+ "pass@8": 0.6385525287524542,
606
+ "pass@16": 0.6994129926552873,
607
+ "pass@32": 0.7462385573378134,
608
+ "pass@64": 0.7809706376082151,
609
+ "pass@128": 0.8042168674698795
610
+ },
611
+ "18": {
612
+ "pass@1": 0.31597782258064516,
613
+ "pass@2": 0.4188174688849377,
614
+ "pass@4": 0.5104581848639886,
615
+ "pass@8": 0.5790573511639918,
616
+ "pass@16": 0.6283327210915123,
617
+ "pass@32": 0.6653820450438427,
618
+ "pass@64": 0.6937567396642746,
619
+ "pass@128": 0.7161290322580646
620
+ },
621
+ "19": {
622
+ "pass@1": 0.2052256419939577,
623
+ "pass@2": 0.28584974248876,
624
+ "pass@4": 0.36753850377841757,
625
+ "pass@8": 0.44054975863490586,
626
+ "pass@16": 0.49795327570192016,
627
+ "pass@32": 0.5453148986286661,
628
+ "pass@64": 0.5923666332779112,
629
+ "pass@128": 0.6404833836858006
630
+ },
631
+ "20": {
632
+ "pass@1": 0.19200450450450451,
633
+ "pass@2": 0.26821087820103573,
634
+ "pass@4": 0.3499301157175173,
635
+ "pass@8": 0.4312063902239929,
636
+ "pass@16": 0.5043251579531292,
637
+ "pass@32": 0.5612592194970087,
638
+ "pass@64": 0.6019373120540266,
639
+ "pass@128": 0.6306306306306306
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9db9f6d58176cd3599c8421eff106c2759c02b9f77c7871f8743386df77e636e
3
+ size 1791373215
results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block4-epoch1-op11-14",
3
+ "id": {
4
+ "answer_accuracy": 0.9827213541666666,
5
+ "avg_response_len": 199.76456336805555,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9827213541666666,
11
+ "pass@2": 0.9873874945319294,
12
+ "pass@4": 0.9899934680664909,
13
+ "pass@8": 0.9919332484253088,
14
+ "pass@16": 0.993608891907506,
15
+ "pass@32": 0.9950211980869957,
16
+ "pass@64": 0.9962416535993714,
17
+ "pass@128": 0.9974444444444445
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.92821875,
22
+ "pass@2": 0.9428015501968511,
23
+ "pass@4": 0.95275325712411,
24
+ "pass@8": 0.9602836900431464,
25
+ "pass@16": 0.9667643456431163,
26
+ "pass@32": 0.9724262544552759,
27
+ "pass@64": 0.9773916206622802,
28
+ "pass@128": 0.983
29
+ },
30
+ "2": {
31
+ "pass@1": 0.999953125,
32
+ "pass@2": 0.9999998769685039,
33
+ "pass@4": 1.0,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 0.99825,
42
+ "pass@2": 0.9988725393700788,
43
+ "pass@4": 0.9990494026996625,
44
+ "pass@8": 0.9991214535331311,
45
+ "pass@16": 0.9992352362176316,
46
+ "pass@32": 0.9994389763779528,
47
+ "pass@64": 0.999751968503937,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.99721875,
52
+ "pass@2": 0.9981986958661418,
53
+ "pass@4": 0.99854321287964,
54
+ "pass@8": 0.9989217378407179,
55
+ "pass@16": 0.9993741635872916,
56
+ "pass@32": 0.9997590674152546,
57
+ "pass@64": 0.9999711690328693,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.980671875,
62
+ "pass@2": 0.9855286663385826,
63
+ "pass@4": 0.9891177830896137,
64
+ "pass@8": 0.9923601897966485,
65
+ "pass@16": 0.9951406369282718,
66
+ "pass@32": 0.9969241199883416,
67
+ "pass@64": 0.9980015759808923,
68
+ "pass@128": 0.999
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9930234375,
72
+ "pass@2": 0.994091658464567,
73
+ "pass@4": 0.9947228139295088,
74
+ "pass@8": 0.9952883871401448,
75
+ "pass@16": 0.9959038226355749,
76
+ "pass@32": 0.9965991523224128,
77
+ "pass@64": 0.9973491337914664,
78
+ "pass@128": 0.998
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9889375,
82
+ "pass@2": 0.9931058070866141,
83
+ "pass@4": 0.9949000196850394,
84
+ "pass@8": 0.9961006060233324,
85
+ "pass@16": 0.9972014482074887,
86
+ "pass@32": 0.998169276945154,
87
+ "pass@64": 0.99912682009508,
88
+ "pass@128": 1.0
89
+ },
90
+ "8": {
91
+ "pass@1": 0.9813515625,
92
+ "pass@2": 0.9889160925196855,
93
+ "pass@4": 0.9925639879077615,
94
+ "pass@8": 0.9947117324737054,
95
+ "pass@16": 0.9964938152859911,
96
+ "pass@32": 0.9980639650979307,
97
+ "pass@64": 0.9992073394908063,
98
+ "pass@128": 1.0
99
+ },
100
+ "9": {
101
+ "pass@1": 0.9768671875,
102
+ "pass@2": 0.9849725639763786,
103
+ "pass@4": 0.9882907352830895,
104
+ "pass@8": 0.9906114389769584,
105
+ "pass@16": 0.9923665586621832,
106
+ "pass@32": 0.9938099701806373,
107
+ "pass@64": 0.9953752548370095,
108
+ "pass@128": 0.997
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.92821875,
113
+ "2": 0.999953125,
114
+ "3": 0.99825,
115
+ "4": 0.99721875,
116
+ "5": 0.980671875,
117
+ "6": 0.9930234375,
118
+ "7": 0.9889375,
119
+ "8": 0.9813515625,
120
+ "9": 0.9768671875
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 279.5917890625,
124
+ "2": 115.366984375,
125
+ "3": 144.562125,
126
+ "4": 159.6835546875,
127
+ "5": 174.0367578125,
128
+ "6": 198.8411171875,
129
+ "7": 223.6661796875,
130
+ "8": 243.4614921875,
131
+ "9": 258.6710703125
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 279.5917890625,
136
+ "2": 115.366984375,
137
+ "3": 144.562125,
138
+ "4": 159.6835546875,
139
+ "5": 174.0367578125,
140
+ "6": 198.8411171875,
141
+ "7": 223.6661796875,
142
+ "8": 243.4614921875,
143
+ "9": 258.6710703125
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 372242,
150
+ "answer_accuracy": 0.9821481340763256,
151
+ "avg_response_len": 190.2342826536643,
152
+ "resp_tokens_sum": 72100315,
153
+ "pass_at_k": {
154
+ "pass@1": 0.9821481340763256,
155
+ "pass@2": 0.9872708730557621,
156
+ "pass@4": 0.9901135191497727,
157
+ "pass@8": 0.9920274304013039,
158
+ "pass@16": 0.9934861887733804,
159
+ "pass@32": 0.9946622220879497,
160
+ "pass@64": 0.9956885677389574,
161
+ "pass@128": 0.9966227625802094
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9259080014749262,
166
+ "pass@2": 0.9433224020393468,
167
+ "pass@4": 0.9555353711317055,
168
+ "pass@8": 0.9626092113862927,
169
+ "pass@16": 0.9672899140408969,
170
+ "pass@32": 0.971873515401767,
171
+ "pass@64": 0.9768429344531748,
172
+ "pass@128": 0.9823008849557522
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9999513239875389,
176
+ "pass@2": 1.0,
177
+ "pass@4": 1.0,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9999044342507645,
186
+ "pass@2": 0.9999977425413568,
187
+ "pass@4": 0.9999999997133384,
188
+ "pass@8": 1.0,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.9938720703125,
196
+ "pass@2": 0.9946638933316929,
197
+ "pass@4": 0.995454402184102,
198
+ "pass@8": 0.9966304348689696,
199
+ "pass@16": 0.9980442612102862,
200
+ "pass@32": 0.9992470856726705,
201
+ "pass@64": 0.9999099032277166,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9814980158730159,
206
+ "pass@2": 0.9859626921634798,
207
+ "pass@4": 0.9891130867570126,
208
+ "pass@8": 0.9915862161820935,
209
+ "pass@16": 0.9933890901170934,
210
+ "pass@32": 0.9943969337435874,
211
+ "pass@64": 0.9952380498015997,
212
+ "pass@128": 0.9968253968253968
213
+ },
214
+ "6": {
215
+ "pass@1": 0.992374625748503,
216
+ "pass@2": 0.9937386545806025,
217
+ "pass@4": 0.9939952238380383,
218
+ "pass@8": 0.9940119126755125,
219
+ "pass@16": 0.9940119760476699,
220
+ "pass@32": 0.9940119760479041,
221
+ "pass@64": 0.9940119760479041,
222
+ "pass@128": 0.9940119760479041
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9904539571005917,
226
+ "pass@2": 0.9942641114941992,
227
+ "pass@4": 0.995715497378656,
228
+ "pass@8": 0.9967252188259338,
229
+ "pass@16": 0.9975999557269617,
230
+ "pass@32": 0.9983364053352927,
231
+ "pass@64": 0.9992661788080938,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9777393830128205,
236
+ "pass@2": 0.9862255987532803,
237
+ "pass@4": 0.990327881490775,
238
+ "pass@8": 0.9937928780958446,
239
+ "pass@16": 0.996912117506564,
240
+ "pass@32": 0.9990931142481682,
241
+ "pass@64": 0.9999455717474616,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9797535211267606,
246
+ "pass@2": 0.9886772901186649,
247
+ "pass@4": 0.9919861029695232,
248
+ "pass@8": 0.9939178049019348,
249
+ "pass@16": 0.9951668490880456,
250
+ "pass@32": 0.996002421656276,
251
+ "pass@64": 0.9968393035287623,
252
+ "pass@128": 0.9971830985915493
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 383958,
259
+ "answer_accuracy": 0.9828544806684142,
260
+ "avg_response_len": 197.82942793659896,
261
+ "resp_tokens_sum": 77283253,
262
+ "pass_at_k": {
263
+ "pass@1": 0.9828544806684142,
264
+ "pass@2": 0.9864802011202161,
265
+ "pass@4": 0.9885723154608951,
266
+ "pass@8": 0.9902874693103434,
267
+ "pass@16": 0.9919324015926441,
268
+ "pass@32": 0.9934557354166121,
269
+ "pass@64": 0.9949606718992721,
270
+ "pass@128": 0.9967234600262124
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9163391487730062,
275
+ "pass@2": 0.9258629564996861,
276
+ "pass@4": 0.9336692571212339,
277
+ "pass@8": 0.9416640558086454,
278
+ "pass@16": 0.9500473404479759,
279
+ "pass@32": 0.9578994674031669,
280
+ "pass@64": 0.9655855093279391,
281
+ "pass@128": 0.9754601226993865
282
+ },
283
+ "2": {
284
+ "pass@1": 0.9999762537993921,
285
+ "pass@2": 1.0,
286
+ "pass@4": 1.0,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9982717803030303,
295
+ "pass@2": 0.9995746092817943,
296
+ "pass@4": 0.9999632247673588,
297
+ "pass@8": 0.9999996922182559,
298
+ "pass@16": 0.9999999999913911,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9986131656804734,
305
+ "pass@2": 0.999854036597866,
306
+ "pass@4": 0.9999976565429319,
307
+ "pass@8": 0.9999999993354288,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.9863224637681159,
315
+ "pass@2": 0.9906003937007875,
316
+ "pass@4": 0.9930274229308294,
317
+ "pass@8": 0.9950581299215392,
318
+ "pass@16": 0.9969160592890997,
319
+ "pass@32": 0.998278697645461,
320
+ "pass@64": 0.9992808085046374,
321
+ "pass@128": 1.0
322
+ },
323
+ "6": {
324
+ "pass@1": 0.9967765042979942,
325
+ "pass@2": 0.9973253869322924,
326
+ "pass@4": 0.9975613690301607,
327
+ "pass@8": 0.9979359856831456,
328
+ "pass@16": 0.9985470770655008,
329
+ "pass@32": 0.9993380089666443,
330
+ "pass@64": 0.9999174243620693,
331
+ "pass@128": 1.0
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9916230130057804,
335
+ "pass@2": 0.9936738342587959,
336
+ "pass@4": 0.9944413127911034,
337
+ "pass@8": 0.9949076292657711,
338
+ "pass@16": 0.9955414667011108,
339
+ "pass@32": 0.9966227126233496,
340
+ "pass@64": 0.9982021755951027,
341
+ "pass@128": 1.0
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9815140845070423,
345
+ "pass@2": 0.9889029056227125,
346
+ "pass@4": 0.9929117410499744,
347
+ "pass@8": 0.9949431586627508,
348
+ "pass@16": 0.9962341274090126,
349
+ "pass@32": 0.9974052544005088,
350
+ "pass@64": 0.9985526092972625,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9739427395209581,
355
+ "pass@2": 0.9802412890753925,
356
+ "pass@4": 0.9834003469439074,
357
+ "pass@8": 0.9861000031633753,
358
+ "pass@16": 0.9883617473597814,
359
+ "pass@32": 0.9900186779076805,
360
+ "pass@64": 0.9917721721639942,
361
+ "pass@128": 0.9940119760479041
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 375895,
368
+ "answer_accuracy": 0.9831535612654838,
369
+ "avg_response_len": 211.18913468781386,
370
+ "resp_tokens_sum": 80745209,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9831535612654838,
373
+ "pass@2": 0.9884301378809491,
374
+ "pass@4": 0.9913265402106796,
375
+ "pass@8": 0.9935214791008893,
376
+ "pass@16": 0.995443499346777,
377
+ "pass@32": 0.9969765780679748,
378
+ "pass@64": 0.9980987822841344,
379
+ "pass@128": 0.9989956478071644
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.9421175373134328,
384
+ "pass@2": 0.9587580062286991,
385
+ "pass@4": 0.9685092193699665,
386
+ "pass@8": 0.9760498065360441,
387
+ "pass@16": 0.9825003934245147,
388
+ "pass@32": 0.9871221085392369,
389
+ "pass@64": 0.9894357905723764,
390
+ "pass@128": 0.991044776119403
391
+ },
392
+ "2": {
393
+ "pass@1": 0.9999330357142857,
394
+ "pass@2": 0.9999996484814399,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9966517857142857,
404
+ "pass@2": 0.9971243629622828,
405
+ "pass@4": 0.9972639609917566,
406
+ "pass@8": 0.99743893615483,
407
+ "pass@16": 0.9977703679897155,
408
+ "pass@32": 0.9983643626179382,
409
+ "pass@64": 0.9992768761047727,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9989720394736842,
414
+ "pass@2": 0.9998701334208223,
415
+ "pass@4": 0.9999958955240242,
416
+ "pass@8": 0.9999999968048913,
417
+ "pass@16": 0.9999999999999999,
418
+ "pass@32": 1.0,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.974172794117647,
424
+ "pass@2": 0.9799802425891616,
425
+ "pass@4": 0.9851549995588786,
426
+ "pass@8": 0.9903396378716414,
427
+ "pass@16": 0.9949618620195534,
428
+ "pass@32": 0.9978909857983171,
429
+ "pass@64": 0.9992638569390833,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.98957511829653,
434
+ "pass@2": 0.9909034315308379,
435
+ "pass@4": 0.9923643261075016,
436
+ "pass@8": 0.9937184520602704,
437
+ "pass@16": 0.9949870433431968,
438
+ "pass@32": 0.9963097387793499,
439
+ "pass@64": 0.9980377056438621,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.984375,
444
+ "pass@2": 0.9912449074304795,
445
+ "pass@4": 0.994530023244721,
446
+ "pass@8": 0.9967387415639558,
447
+ "pass@16": 0.9985927711809854,
448
+ "pass@32": 0.9996839030827411,
449
+ "pass@64": 0.9999901863988568,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9845626876876877,
454
+ "pass@2": 0.9914509736114068,
455
+ "pass@4": 0.9942883507804767,
456
+ "pass@8": 0.9953259254730494,
457
+ "pass@16": 0.9963787369181493,
458
+ "pass@32": 0.9978019463673319,
459
+ "pass@64": 0.9992136480632735,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9767132234726688,
464
+ "pass@2": 0.9858248727751473,
465
+ "pass@4": 0.9893246072335813,
466
+ "pass@8": 0.9916823703543539,
467
+ "pass@16": 0.9934710727259167,
468
+ "pass@32": 0.995379009876187,
469
+ "pass@64": 0.9975736224486328,
470
+ "pass@128": 1.0
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.615015625,
478
+ "avg_response_len": 367.26526484375,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.615015625,
484
+ "pass@2": 0.688048412893687,
485
+ "pass@4": 0.7440958128702582,
486
+ "pass@8": 0.7877747365946025,
487
+ "pass@16": 0.8222755379094328,
488
+ "pass@32": 0.8496575803035179,
489
+ "pass@64": 0.8714758455012228,
490
+ "pass@128": 0.8895
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.9379609375,
495
+ "pass@2": 0.9578695866141734,
496
+ "pass@4": 0.9690707025684286,
497
+ "pass@8": 0.9761228584925753,
498
+ "pass@16": 0.9809705725972232,
499
+ "pass@32": 0.9846269650875722,
500
+ "pass@64": 0.9872749236211029,
501
+ "pass@128": 0.989
502
+ },
503
+ "12": {
504
+ "pass@1": 0.89421875,
505
+ "pass@2": 0.9220222687007882,
506
+ "pass@4": 0.9362151971316084,
507
+ "pass@8": 0.9447333294059417,
508
+ "pass@16": 0.9509831248030044,
509
+ "pass@32": 0.9562493532460535,
510
+ "pass@64": 0.961528048501336,
511
+ "pass@128": 0.967
512
+ },
513
+ "13": {
514
+ "pass@1": 0.8007421875,
515
+ "pass@2": 0.850632997047245,
516
+ "pass@4": 0.8801703857330323,
517
+ "pass@8": 0.900917953033891,
518
+ "pass@16": 0.9170591607756168,
519
+ "pass@32": 0.9292019660423374,
520
+ "pass@64": 0.9391975690385227,
521
+ "pass@128": 0.949
522
+ },
523
+ "14": {
524
+ "pass@1": 0.777703125,
525
+ "pass@2": 0.8398607283464579,
526
+ "pass@4": 0.8762118897637787,
527
+ "pass@8": 0.9005872635175092,
528
+ "pass@16": 0.9194266016183247,
529
+ "pass@32": 0.9342833957716893,
530
+ "pass@64": 0.9452691557195909,
531
+ "pass@128": 0.954
532
+ },
533
+ "15": {
534
+ "pass@1": 0.69453125,
535
+ "pass@2": 0.768718626968505,
536
+ "pass@4": 0.8212702295650541,
537
+ "pass@8": 0.8608277460887901,
538
+ "pass@16": 0.8904122978547405,
539
+ "pass@32": 0.9119605369492704,
540
+ "pass@64": 0.927476651698037,
541
+ "pass@128": 0.938
542
+ },
543
+ "16": {
544
+ "pass@1": 0.5662421875,
545
+ "pass@2": 0.6677634104330711,
546
+ "pass@4": 0.7407699299775025,
547
+ "pass@8": 0.7935083278076046,
548
+ "pass@16": 0.8317948982183991,
549
+ "pass@32": 0.8604213110568941,
550
+ "pass@64": 0.8841963329743866,
551
+ "pass@128": 0.905
552
+ },
553
+ "17": {
554
+ "pass@1": 0.4808125,
555
+ "pass@2": 0.585482406496063,
556
+ "pass@4": 0.6681984115110617,
557
+ "pass@8": 0.7304857882018998,
558
+ "pass@16": 0.7780592856710695,
559
+ "pass@32": 0.8142376854501494,
560
+ "pass@64": 0.8410238580115026,
561
+ "pass@128": 0.864
562
+ },
563
+ "18": {
564
+ "pass@1": 0.44084375,
565
+ "pass@2": 0.5405799704724411,
566
+ "pass@4": 0.6218796379827516,
567
+ "pass@8": 0.6872290020729789,
568
+ "pass@16": 0.7397756567660226,
569
+ "pass@32": 0.7814495225157587,
570
+ "pass@64": 0.8142298574628336,
571
+ "pass@128": 0.84
572
+ },
573
+ "19": {
574
+ "pass@1": 0.2833359375,
575
+ "pass@2": 0.37453592519685047,
576
+ "pass@4": 0.4594816270153731,
577
+ "pass@8": 0.5354858260969473,
578
+ "pass@16": 0.6025362039793238,
579
+ "pass@32": 0.6610735591981041,
580
+ "pass@64": 0.710950493166852,
581
+ "pass@128": 0.753
582
+ },
583
+ "20": {
584
+ "pass@1": 0.273765625,
585
+ "pass@2": 0.37301820866141744,
586
+ "pass@4": 0.4676901174540681,
587
+ "pass@8": 0.5478492712279519,
588
+ "pass@16": 0.6117375768104771,
589
+ "pass@32": 0.6630715077173114,
590
+ "pass@64": 0.7036115648179885,
591
+ "pass@128": 0.736
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.9379609375,
596
+ "12": 0.89421875,
597
+ "13": 0.8007421875,
598
+ "14": 0.777703125,
599
+ "15": 0.69453125,
600
+ "16": 0.5662421875,
601
+ "17": 0.4808125,
602
+ "18": 0.44084375,
603
+ "19": 0.2833359375,
604
+ "20": 0.273765625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 297.83946875,
608
+ "12": 308.6558984375,
609
+ "13": 327.6826796875,
610
+ "14": 353.76734375,
611
+ "15": 371.4339140625,
612
+ "16": 376.514875,
613
+ "17": 397.728140625,
614
+ "18": 409.4316171875,
615
+ "19": 405.357671875,
616
+ "20": 424.2410390625
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 297.83946875,
621
+ "12": 308.6558984375,
622
+ "13": 327.6826796875,
623
+ "14": 353.76734375,
624
+ "15": 371.4339140625,
625
+ "16": 376.514875,
626
+ "17": 397.728140625,
627
+ "18": 409.4316171875,
628
+ "19": 405.357671875,
629
+ "20": 424.2410390625
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 268071,
636
+ "answer_accuracy": 0.6223788075780089,
637
+ "avg_response_len": 348.7750650074294,
638
+ "resp_tokens_sum": 150224396,
639
+ "pass_at_k": {
640
+ "pass@1": 0.6223788075780089,
641
+ "pass@2": 0.6935501880754923,
642
+ "pass@4": 0.7469774392428321,
643
+ "pass@8": 0.7885196429188797,
644
+ "pass@16": 0.8221776320825549,
645
+ "pass@32": 0.8495236946886684,
646
+ "pass@64": 0.8715822180293861,
647
+ "pass@128": 0.8900445765230312
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9369688385269122,
652
+ "pass@2": 0.9556891436729048,
653
+ "pass@4": 0.9681532095138248,
654
+ "pass@8": 0.9773977476860249,
655
+ "pass@16": 0.9836760616393073,
656
+ "pass@32": 0.9879154034727532,
657
+ "pass@64": 0.9905512293835983,
658
+ "pass@128": 0.9915014164305949
659
+ },
660
+ "12": {
661
+ "pass@1": 0.896340925141243,
662
+ "pass@2": 0.9266628019484855,
663
+ "pass@4": 0.9416773099337153,
664
+ "pass@8": 0.9499502893000226,
665
+ "pass@16": 0.9556055452848286,
666
+ "pass@32": 0.9601108862491841,
667
+ "pass@64": 0.9653730933928352,
668
+ "pass@128": 0.9717514124293786
669
+ },
670
+ "13": {
671
+ "pass@1": 0.8189410266457681,
672
+ "pass@2": 0.8701832590773324,
673
+ "pass@4": 0.9008475028003936,
674
+ "pass@8": 0.9223142658352539,
675
+ "pass@16": 0.9375126996972292,
676
+ "pass@32": 0.9461958750908102,
677
+ "pass@64": 0.9519937435789704,
678
+ "pass@128": 0.9592476489028213
679
+ },
680
+ "14": {
681
+ "pass@1": 0.8048091900311527,
682
+ "pass@2": 0.8633829138764197,
683
+ "pass@4": 0.8936096031220395,
684
+ "pass@8": 0.9133137052944442,
685
+ "pass@16": 0.9308204065370922,
686
+ "pass@32": 0.9458697424997475,
687
+ "pass@64": 0.9578949685863664,
688
+ "pass@128": 0.9688473520249221
689
+ },
690
+ "15": {
691
+ "pass@1": 0.7142792630057804,
692
+ "pass@2": 0.7852296777570432,
693
+ "pass@4": 0.8336160666874729,
694
+ "pass@8": 0.8703838255639478,
695
+ "pass@16": 0.8991492741726431,
696
+ "pass@32": 0.9217285219094363,
697
+ "pass@64": 0.9390414573776548,
698
+ "pass@128": 0.9508670520231214
699
+ },
700
+ "16": {
701
+ "pass@1": 0.5767389112903226,
702
+ "pass@2": 0.671137207899416,
703
+ "pass@4": 0.7352816832371761,
704
+ "pass@8": 0.7801929643943304,
705
+ "pass@16": 0.8153526018049102,
706
+ "pass@32": 0.8451755884363006,
707
+ "pass@64": 0.8721477928442585,
708
+ "pass@128": 0.896774193548387
709
+ },
710
+ "17": {
711
+ "pass@1": 0.49742593930635837,
712
+ "pass@2": 0.6042979950844295,
713
+ "pass@4": 0.6877961173386561,
714
+ "pass@8": 0.750236429513258,
715
+ "pass@16": 0.797521632533749,
716
+ "pass@32": 0.8336931159590578,
717
+ "pass@64": 0.861816202176084,
718
+ "pass@128": 0.8872832369942196
719
+ },
720
+ "18": {
721
+ "pass@1": 0.4068080357142857,
722
+ "pass@2": 0.5067449381327332,
723
+ "pass@4": 0.5904474240719905,
724
+ "pass@8": 0.6602663145937516,
725
+ "pass@16": 0.7184114317717843,
726
+ "pass@32": 0.7636522860956406,
727
+ "pass@64": 0.7975986551347283,
728
+ "pass@128": 0.8228571428571428
729
+ },
730
+ "19": {
731
+ "pass@1": 0.28638795045045046,
732
+ "pass@2": 0.37785669527795496,
733
+ "pass@4": 0.45953530245656216,
734
+ "pass@8": 0.5290022873626415,
735
+ "pass@16": 0.5902007338149695,
736
+ "pass@32": 0.6466124054432371,
737
+ "pass@64": 0.6966004194500958,
738
+ "pass@128": 0.7417417417417418
739
+ },
740
+ "20": {
741
+ "pass@1": 0.2728978978978979,
742
+ "pass@2": 0.36531228571090785,
743
+ "pass@4": 0.4511123266123264,
744
+ "pass@8": 0.5250073329043563,
745
+ "pass@16": 0.5869595546727128,
746
+ "pass@32": 0.638416275402605,
747
+ "pass@64": 0.6777993410094819,
748
+ "pass@128": 0.7057057057057057
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 265030,
755
+ "answer_accuracy": 0.615684470710675,
756
+ "avg_response_len": 363.1408782151353,
757
+ "resp_tokens_sum": 156319075,
758
+ "pass_at_k": {
759
+ "pass@1": 0.615684470710675,
760
+ "pass@2": 0.6895557930091535,
761
+ "pass@4": 0.7462455992343082,
762
+ "pass@8": 0.7906967253680187,
763
+ "pass@16": 0.8259765555967202,
764
+ "pass@32": 0.8537529719929998,
765
+ "pass@64": 0.875734930364443,
766
+ "pass@128": 0.8941421349985132
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.942371046686747,
771
+ "pass@2": 0.9614907711554884,
772
+ "pass@4": 0.9709818822534232,
773
+ "pass@8": 0.9762108984873451,
774
+ "pass@16": 0.9798418328465113,
775
+ "pass@32": 0.9832802016718227,
776
+ "pass@64": 0.9869836840656171,
777
+ "pass@128": 0.9909638554216867
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8921189263322884,
781
+ "pass@2": 0.9154603923431991,
782
+ "pass@4": 0.9262470747191082,
783
+ "pass@8": 0.932674949530915,
784
+ "pass@16": 0.9378135612950852,
785
+ "pass@32": 0.942572203878569,
786
+ "pass@64": 0.947465209542479,
787
+ "pass@128": 0.9529780564263323
788
+ },
789
+ "13": {
790
+ "pass@1": 0.7801995798319328,
791
+ "pass@2": 0.8263980789166057,
792
+ "pass@4": 0.8551042666410398,
793
+ "pass@8": 0.8787522658307619,
794
+ "pass@16": 0.8997253159536589,
795
+ "pass@32": 0.9175379858563458,
796
+ "pass@64": 0.9326565965286922,
797
+ "pass@128": 0.9439775910364145
798
+ },
799
+ "14": {
800
+ "pass@1": 0.7647112573099415,
801
+ "pass@2": 0.8316522626283556,
802
+ "pass@4": 0.8720206605205054,
803
+ "pass@8": 0.899213977549253,
804
+ "pass@16": 0.9185417582132726,
805
+ "pass@32": 0.9327907985067769,
806
+ "pass@64": 0.9426059507605182,
807
+ "pass@128": 0.9502923976608187
808
+ },
809
+ "15": {
810
+ "pass@1": 0.6902225378787878,
811
+ "pass@2": 0.7702934860415173,
812
+ "pass@4": 0.8277060339048522,
813
+ "pass@8": 0.870181513451221,
814
+ "pass@16": 0.9009560991102984,
815
+ "pass@32": 0.9213126526056608,
816
+ "pass@64": 0.9341966745085428,
817
+ "pass@128": 0.9424242424242424
818
+ },
819
+ "16": {
820
+ "pass@1": 0.5663951210826211,
821
+ "pass@2": 0.6729072728088475,
822
+ "pass@4": 0.747877350267114,
823
+ "pass@8": 0.8014397473398795,
824
+ "pass@16": 0.8403655118855234,
825
+ "pass@32": 0.8670101699239859,
826
+ "pass@64": 0.8852311741120641,
827
+ "pass@128": 0.9002849002849003
828
+ },
829
+ "17": {
830
+ "pass@1": 0.499708850931677,
831
+ "pass@2": 0.6042562020100748,
832
+ "pass@4": 0.6857853701905272,
833
+ "pass@8": 0.7446535294212936,
834
+ "pass@16": 0.787459898111565,
835
+ "pass@32": 0.8184923783964188,
836
+ "pass@64": 0.840558376080876,
837
+ "pass@128": 0.860248447204969
838
+ },
839
+ "18": {
840
+ "pass@1": 0.47116268382352944,
841
+ "pass@2": 0.5732706837656322,
842
+ "pass@4": 0.6578277650036389,
843
+ "pass@8": 0.7272127940450731,
844
+ "pass@16": 0.7824553129641325,
845
+ "pass@32": 0.8258777832780535,
846
+ "pass@64": 0.858667717455225,
847
+ "pass@128": 0.8823529411764706
848
+ },
849
+ "19": {
850
+ "pass@1": 0.2761346726190476,
851
+ "pass@2": 0.365666449896888,
852
+ "pass@4": 0.45070808615440894,
853
+ "pass@8": 0.529863381029973,
854
+ "pass@16": 0.6031771625339937,
855
+ "pass@32": 0.6680454701301732,
856
+ "pass@64": 0.7240389650730391,
857
+ "pass@128": 0.7708333333333334
858
+ },
859
+ "20": {
860
+ "pass@1": 0.277156624251497,
861
+ "pass@2": 0.3758873738742987,
862
+ "pass@4": 0.46919662278367874,
863
+ "pass@8": 0.547041580122116,
864
+ "pass@16": 0.608975135388662,
865
+ "pass@32": 0.6595693283084222,
866
+ "pass@64": 0.7034790052169011,
867
+ "pass@128": 0.7455089820359282
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 254119,
874
+ "answer_accuracy": 0.6067557113386308,
875
+ "avg_response_len": 390.5201042940098,
876
+ "resp_tokens_sum": 163556068,
877
+ "pass_at_k": {
878
+ "pass@1": 0.6067557113386308,
879
+ "pass@2": 0.6808409578769081,
880
+ "pass@4": 0.7389227064259096,
881
+ "pass@8": 0.7840054034570462,
882
+ "pass@16": 0.818572277098011,
883
+ "pass@32": 0.8455859797051636,
884
+ "pass@64": 0.8669889121416895,
885
+ "pass@128": 0.8841687041564792
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.9344246031746032,
890
+ "pass@2": 0.956496453568304,
891
+ "pass@4": 0.9680845546092448,
892
+ "pass@8": 0.9746013817828894,
893
+ "pass@16": 0.979128362963511,
894
+ "pass@32": 0.9823612720845869,
895
+ "pass@64": 0.9839103382187553,
896
+ "pass@128": 0.9841269841269841
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8939698012232415,
900
+ "pass@2": 0.9233999133135886,
901
+ "pass@4": 0.940026338164151,
902
+ "pass@8": 0.9508489849950189,
903
+ "pass@16": 0.9588264089267363,
904
+ "pass@32": 0.9654115182769986,
905
+ "pass@64": 0.9710843168080171,
906
+ "pass@128": 0.9755351681957186
907
+ },
908
+ "13": {
909
+ "pass@1": 0.8054591049382716,
910
+ "pass@2": 0.8580877260134149,
911
+ "pass@4": 0.8874315097186924,
912
+ "pass@8": 0.9042751337372295,
913
+ "pass@16": 0.9160205919035335,
914
+ "pass@32": 0.9253222868446103,
915
+ "pass@64": 0.9338060490002721,
916
+ "pass@128": 0.9444444444444444
917
+ },
918
+ "14": {
919
+ "pass@1": 0.7650686201780416,
920
+ "pass@2": 0.8257855761816872,
921
+ "pass@4": 0.8638935942539817,
922
+ "pass@8": 0.8898587056265527,
923
+ "pass@16": 0.909471720507359,
924
+ "pass@32": 0.9247618852817582,
925
+ "pass@64": 0.9359455063598505,
926
+ "pass@128": 0.9436201780415431
927
+ },
928
+ "15": {
929
+ "pass@1": 0.6778308256172839,
930
+ "pass@2": 0.7494824323174885,
931
+ "pass@4": 0.801531108958602,
932
+ "pass@8": 0.8410958117430899,
933
+ "pass@16": 0.8703430132240998,
934
+ "pass@32": 0.8920039907677088,
935
+ "pass@64": 0.9082821137578678,
936
+ "pass@128": 0.9197530864197531
937
+ },
938
+ "16": {
939
+ "pass@1": 0.5564850663716814,
940
+ "pass@2": 0.659352280909577,
941
+ "pass@4": 0.7384296702956372,
942
+ "pass@8": 0.7974724410886839,
943
+ "pass@16": 0.8379566282804083,
944
+ "pass@32": 0.867540734508324,
945
+ "pass@64": 0.8941426993490582,
946
+ "pass@128": 0.9174041297935103
947
+ },
948
+ "17": {
949
+ "pass@1": 0.44517131024096385,
950
+ "pass@2": 0.5476650697277298,
951
+ "pass@4": 0.630717065393181,
952
+ "pass@8": 0.696161286495951,
953
+ "pass@16": 0.7486587759712936,
954
+ "pass@32": 0.7898353056767135,
955
+ "pass@64": 0.8198061896401653,
956
+ "pass@128": 0.8433734939759037
957
+ },
958
+ "18": {
959
+ "pass@1": 0.44601814516129035,
960
+ "pass@2": 0.5429264827279653,
961
+ "pass@4": 0.6179406434074768,
962
+ "pass@8": 0.6738175548059356,
963
+ "pass@16": 0.7170866104454663,
964
+ "pass@32": 0.7528154066701483,
965
+ "pass@64": 0.7842687233254886,
966
+ "pass@128": 0.8129032258064516
967
+ },
968
+ "19": {
969
+ "pass@1": 0.28757552870090636,
970
+ "pass@2": 0.3801985453291147,
971
+ "pass@4": 0.4683336989409553,
972
+ "pass@8": 0.5477159165532208,
973
+ "pass@16": 0.6142955679985431,
974
+ "pass@32": 0.6685448647183322,
975
+ "pass@64": 0.7121010913154943,
976
+ "pass@128": 0.7462235649546828
977
+ },
978
+ "20": {
979
+ "pass@1": 0.2712321696696697,
980
+ "pass@2": 0.377846350287295,
981
+ "pass@4": 0.48275687892617025,
982
+ "pass@8": 0.5715013261561981,
983
+ "pass@16": 0.6392863359899448,
984
+ "pass@32": 0.6912394364961897,
985
+ "pass@64": 0.7295567463044594,
986
+ "pass@128": 0.7567567567567568
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.7891920230263157,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.7891920230263157,
1002
+ "pass@2": 0.8298406094591582,
1003
+ "pass@4": 0.8605736495421483,
1004
+ "pass@8": 0.8844814000933748,
1005
+ "pass@16": 0.9034334424348239,
1006
+ "pass@32": 0.9185140308325449,
1007
+ "pass@64": 0.930575438810868,
1008
+ "pass@128": 0.9406315789473684
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.92821875,
1013
+ "pass@2": 0.9428015501968511,
1014
+ "pass@4": 0.95275325712411,
1015
+ "pass@8": 0.9602836900431464,
1016
+ "pass@16": 0.9667643456431163,
1017
+ "pass@32": 0.9724262544552759,
1018
+ "pass@64": 0.9773916206622802,
1019
+ "pass@128": 0.983
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.999953125,
1023
+ "pass@2": 0.9999998769685039,
1024
+ "pass@4": 1.0,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.99825,
1033
+ "pass@2": 0.9988725393700788,
1034
+ "pass@4": 0.9990494026996625,
1035
+ "pass@8": 0.9991214535331311,
1036
+ "pass@16": 0.9992352362176316,
1037
+ "pass@32": 0.9994389763779528,
1038
+ "pass@64": 0.999751968503937,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.99721875,
1043
+ "pass@2": 0.9981986958661418,
1044
+ "pass@4": 0.99854321287964,
1045
+ "pass@8": 0.9989217378407179,
1046
+ "pass@16": 0.9993741635872916,
1047
+ "pass@32": 0.9997590674152546,
1048
+ "pass@64": 0.9999711690328693,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.980671875,
1053
+ "pass@2": 0.9855286663385826,
1054
+ "pass@4": 0.9891177830896137,
1055
+ "pass@8": 0.9923601897966485,
1056
+ "pass@16": 0.9951406369282718,
1057
+ "pass@32": 0.9969241199883416,
1058
+ "pass@64": 0.9980015759808923,
1059
+ "pass@128": 0.999
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9930234375,
1063
+ "pass@2": 0.994091658464567,
1064
+ "pass@4": 0.9947228139295088,
1065
+ "pass@8": 0.9952883871401448,
1066
+ "pass@16": 0.9959038226355749,
1067
+ "pass@32": 0.9965991523224128,
1068
+ "pass@64": 0.9973491337914664,
1069
+ "pass@128": 0.998
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9889375,
1073
+ "pass@2": 0.9931058070866141,
1074
+ "pass@4": 0.9949000196850394,
1075
+ "pass@8": 0.9961006060233324,
1076
+ "pass@16": 0.9972014482074887,
1077
+ "pass@32": 0.998169276945154,
1078
+ "pass@64": 0.99912682009508,
1079
+ "pass@128": 1.0
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.9813515625,
1083
+ "pass@2": 0.9889160925196855,
1084
+ "pass@4": 0.9925639879077615,
1085
+ "pass@8": 0.9947117324737054,
1086
+ "pass@16": 0.9964938152859911,
1087
+ "pass@32": 0.9980639650979307,
1088
+ "pass@64": 0.9992073394908063,
1089
+ "pass@128": 1.0
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.9768671875,
1093
+ "pass@2": 0.9849725639763786,
1094
+ "pass@4": 0.9882907352830895,
1095
+ "pass@8": 0.9906114389769584,
1096
+ "pass@16": 0.9923665586621832,
1097
+ "pass@32": 0.9938099701806373,
1098
+ "pass@64": 0.9953752548370095,
1099
+ "pass@128": 0.997
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.9379609375,
1103
+ "pass@2": 0.9578695866141734,
1104
+ "pass@4": 0.9690707025684286,
1105
+ "pass@8": 0.9761228584925753,
1106
+ "pass@16": 0.9809705725972232,
1107
+ "pass@32": 0.9846269650875722,
1108
+ "pass@64": 0.9872749236211029,
1109
+ "pass@128": 0.989
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.89421875,
1113
+ "pass@2": 0.9220222687007882,
1114
+ "pass@4": 0.9362151971316084,
1115
+ "pass@8": 0.9447333294059417,
1116
+ "pass@16": 0.9509831248030044,
1117
+ "pass@32": 0.9562493532460535,
1118
+ "pass@64": 0.961528048501336,
1119
+ "pass@128": 0.967
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.8007421875,
1123
+ "pass@2": 0.850632997047245,
1124
+ "pass@4": 0.8801703857330323,
1125
+ "pass@8": 0.900917953033891,
1126
+ "pass@16": 0.9170591607756168,
1127
+ "pass@32": 0.9292019660423374,
1128
+ "pass@64": 0.9391975690385227,
1129
+ "pass@128": 0.949
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.777703125,
1133
+ "pass@2": 0.8398607283464579,
1134
+ "pass@4": 0.8762118897637787,
1135
+ "pass@8": 0.9005872635175092,
1136
+ "pass@16": 0.9194266016183247,
1137
+ "pass@32": 0.9342833957716893,
1138
+ "pass@64": 0.9452691557195909,
1139
+ "pass@128": 0.954
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.69453125,
1143
+ "pass@2": 0.768718626968505,
1144
+ "pass@4": 0.8212702295650541,
1145
+ "pass@8": 0.8608277460887901,
1146
+ "pass@16": 0.8904122978547405,
1147
+ "pass@32": 0.9119605369492704,
1148
+ "pass@64": 0.927476651698037,
1149
+ "pass@128": 0.938
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.5662421875,
1153
+ "pass@2": 0.6677634104330711,
1154
+ "pass@4": 0.7407699299775025,
1155
+ "pass@8": 0.7935083278076046,
1156
+ "pass@16": 0.8317948982183991,
1157
+ "pass@32": 0.8604213110568941,
1158
+ "pass@64": 0.8841963329743866,
1159
+ "pass@128": 0.905
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.4808125,
1163
+ "pass@2": 0.585482406496063,
1164
+ "pass@4": 0.6681984115110617,
1165
+ "pass@8": 0.7304857882018998,
1166
+ "pass@16": 0.7780592856710695,
1167
+ "pass@32": 0.8142376854501494,
1168
+ "pass@64": 0.8410238580115026,
1169
+ "pass@128": 0.864
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.44084375,
1173
+ "pass@2": 0.5405799704724411,
1174
+ "pass@4": 0.6218796379827516,
1175
+ "pass@8": 0.6872290020729789,
1176
+ "pass@16": 0.7397756567660226,
1177
+ "pass@32": 0.7814495225157587,
1178
+ "pass@64": 0.8142298574628336,
1179
+ "pass@128": 0.84
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.2833359375,
1183
+ "pass@2": 0.37453592519685047,
1184
+ "pass@4": 0.4594816270153731,
1185
+ "pass@8": 0.5354858260969473,
1186
+ "pass@16": 0.6025362039793238,
1187
+ "pass@32": 0.6610735591981041,
1188
+ "pass@64": 0.710950493166852,
1189
+ "pass@128": 0.753
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.273765625,
1193
+ "pass@2": 0.37301820866141744,
1194
+ "pass@4": 0.4676901174540681,
1195
+ "pass@8": 0.5478492712279519,
1196
+ "pass@16": 0.6117375768104771,
1197
+ "pass@32": 0.6630715077173114,
1198
+ "pass@64": 0.7036115648179885,
1199
+ "pass@128": 0.736
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.92821875,
1204
+ "2": 0.999953125,
1205
+ "3": 0.99825,
1206
+ "4": 0.99721875,
1207
+ "5": 0.980671875,
1208
+ "6": 0.9930234375,
1209
+ "7": 0.9889375,
1210
+ "8": 0.9813515625,
1211
+ "9": 0.9768671875,
1212
+ "11": 0.9379609375,
1213
+ "12": 0.89421875,
1214
+ "13": 0.8007421875,
1215
+ "14": 0.777703125,
1216
+ "15": 0.69453125,
1217
+ "16": 0.5662421875,
1218
+ "17": 0.4808125,
1219
+ "18": 0.44084375,
1220
+ "19": 0.2833359375,
1221
+ "20": 0.273765625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 279.5917890625,
1225
+ "2": 115.366984375,
1226
+ "3": 144.562125,
1227
+ "4": 159.6835546875,
1228
+ "5": 174.0367578125,
1229
+ "6": 198.8411171875,
1230
+ "7": 223.6661796875,
1231
+ "8": 243.4614921875,
1232
+ "9": 258.6710703125,
1233
+ "11": 297.83946875,
1234
+ "12": 308.6558984375,
1235
+ "13": 327.6826796875,
1236
+ "14": 353.76734375,
1237
+ "15": 371.4339140625,
1238
+ "16": 376.514875,
1239
+ "17": 397.728140625,
1240
+ "18": 409.4316171875,
1241
+ "19": 405.357671875,
1242
+ "20": 424.2410390625
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 279.5917890625,
1247
+ "2": 115.366984375,
1248
+ "3": 144.562125,
1249
+ "4": 159.6835546875,
1250
+ "5": 174.0367578125,
1251
+ "6": 198.8411171875,
1252
+ "7": 223.6661796875,
1253
+ "8": 243.4614921875,
1254
+ "9": 258.6710703125,
1255
+ "11": 297.83946875,
1256
+ "12": 308.6558984375,
1257
+ "13": 327.6826796875,
1258
+ "14": 353.76734375,
1259
+ "15": 371.4339140625,
1260
+ "16": 376.514875,
1261
+ "17": 397.728140625,
1262
+ "18": 409.4316171875,
1263
+ "19": 405.357671875,
1264
+ "20": 424.2410390625
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 640313,
1271
+ "answer_accuracy": 0.790775420882074,
1272
+ "avg_response_len": 274.5671521794973,
1273
+ "resp_tokens_sum": 222324711,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.790775420882074,
1276
+ "pass@2": 0.8310315267138889,
1277
+ "pass@4": 0.8607817283045475,
1278
+ "pass@8": 0.8837751849257428,
1279
+ "pass@16": 0.902361735206419,
1280
+ "pass@32": 0.9174584369632929,
1281
+ "pass@64": 0.9296724648662624,
1282
+ "pass@128": 0.9399304457793234
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9259080014749262,
1287
+ "pass@2": 0.9433224020393468,
1288
+ "pass@4": 0.9555353711317055,
1289
+ "pass@8": 0.9626092113862927,
1290
+ "pass@16": 0.9672899140408969,
1291
+ "pass@32": 0.971873515401767,
1292
+ "pass@64": 0.9768429344531748,
1293
+ "pass@128": 0.9823008849557522
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9999513239875389,
1297
+ "pass@2": 1.0,
1298
+ "pass@4": 1.0,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9999044342507645,
1307
+ "pass@2": 0.9999977425413568,
1308
+ "pass@4": 0.9999999997133384,
1309
+ "pass@8": 1.0,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.9938720703125,
1317
+ "pass@2": 0.9946638933316929,
1318
+ "pass@4": 0.995454402184102,
1319
+ "pass@8": 0.9966304348689696,
1320
+ "pass@16": 0.9980442612102862,
1321
+ "pass@32": 0.9992470856726705,
1322
+ "pass@64": 0.9999099032277166,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9814980158730159,
1327
+ "pass@2": 0.9859626921634798,
1328
+ "pass@4": 0.9891130867570126,
1329
+ "pass@8": 0.9915862161820935,
1330
+ "pass@16": 0.9933890901170934,
1331
+ "pass@32": 0.9943969337435874,
1332
+ "pass@64": 0.9952380498015997,
1333
+ "pass@128": 0.9968253968253968
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.992374625748503,
1337
+ "pass@2": 0.9937386545806025,
1338
+ "pass@4": 0.9939952238380383,
1339
+ "pass@8": 0.9940119126755125,
1340
+ "pass@16": 0.9940119760476699,
1341
+ "pass@32": 0.9940119760479041,
1342
+ "pass@64": 0.9940119760479041,
1343
+ "pass@128": 0.9940119760479041
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9904539571005917,
1347
+ "pass@2": 0.9942641114941992,
1348
+ "pass@4": 0.995715497378656,
1349
+ "pass@8": 0.9967252188259338,
1350
+ "pass@16": 0.9975999557269617,
1351
+ "pass@32": 0.9983364053352927,
1352
+ "pass@64": 0.9992661788080938,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9777393830128205,
1357
+ "pass@2": 0.9862255987532803,
1358
+ "pass@4": 0.990327881490775,
1359
+ "pass@8": 0.9937928780958446,
1360
+ "pass@16": 0.996912117506564,
1361
+ "pass@32": 0.9990931142481682,
1362
+ "pass@64": 0.9999455717474616,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9797535211267606,
1367
+ "pass@2": 0.9886772901186649,
1368
+ "pass@4": 0.9919861029695232,
1369
+ "pass@8": 0.9939178049019348,
1370
+ "pass@16": 0.9951668490880456,
1371
+ "pass@32": 0.996002421656276,
1372
+ "pass@64": 0.9968393035287623,
1373
+ "pass@128": 0.9971830985915493
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9369688385269122,
1377
+ "pass@2": 0.9556891436729048,
1378
+ "pass@4": 0.9681532095138248,
1379
+ "pass@8": 0.9773977476860249,
1380
+ "pass@16": 0.9836760616393073,
1381
+ "pass@32": 0.9879154034727532,
1382
+ "pass@64": 0.9905512293835983,
1383
+ "pass@128": 0.9915014164305949
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.896340925141243,
1387
+ "pass@2": 0.9266628019484855,
1388
+ "pass@4": 0.9416773099337153,
1389
+ "pass@8": 0.9499502893000226,
1390
+ "pass@16": 0.9556055452848286,
1391
+ "pass@32": 0.9601108862491841,
1392
+ "pass@64": 0.9653730933928352,
1393
+ "pass@128": 0.9717514124293786
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.8189410266457681,
1397
+ "pass@2": 0.8701832590773324,
1398
+ "pass@4": 0.9008475028003936,
1399
+ "pass@8": 0.9223142658352539,
1400
+ "pass@16": 0.9375126996972292,
1401
+ "pass@32": 0.9461958750908102,
1402
+ "pass@64": 0.9519937435789704,
1403
+ "pass@128": 0.9592476489028213
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.8048091900311527,
1407
+ "pass@2": 0.8633829138764197,
1408
+ "pass@4": 0.8936096031220395,
1409
+ "pass@8": 0.9133137052944442,
1410
+ "pass@16": 0.9308204065370922,
1411
+ "pass@32": 0.9458697424997475,
1412
+ "pass@64": 0.9578949685863664,
1413
+ "pass@128": 0.9688473520249221
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.7142792630057804,
1417
+ "pass@2": 0.7852296777570432,
1418
+ "pass@4": 0.8336160666874729,
1419
+ "pass@8": 0.8703838255639478,
1420
+ "pass@16": 0.8991492741726431,
1421
+ "pass@32": 0.9217285219094363,
1422
+ "pass@64": 0.9390414573776548,
1423
+ "pass@128": 0.9508670520231214
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.5767389112903226,
1427
+ "pass@2": 0.671137207899416,
1428
+ "pass@4": 0.7352816832371761,
1429
+ "pass@8": 0.7801929643943304,
1430
+ "pass@16": 0.8153526018049102,
1431
+ "pass@32": 0.8451755884363006,
1432
+ "pass@64": 0.8721477928442585,
1433
+ "pass@128": 0.896774193548387
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.49742593930635837,
1437
+ "pass@2": 0.6042979950844295,
1438
+ "pass@4": 0.6877961173386561,
1439
+ "pass@8": 0.750236429513258,
1440
+ "pass@16": 0.797521632533749,
1441
+ "pass@32": 0.8336931159590578,
1442
+ "pass@64": 0.861816202176084,
1443
+ "pass@128": 0.8872832369942196
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.4068080357142857,
1447
+ "pass@2": 0.5067449381327332,
1448
+ "pass@4": 0.5904474240719905,
1449
+ "pass@8": 0.6602663145937516,
1450
+ "pass@16": 0.7184114317717843,
1451
+ "pass@32": 0.7636522860956406,
1452
+ "pass@64": 0.7975986551347283,
1453
+ "pass@128": 0.8228571428571428
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.28638795045045046,
1457
+ "pass@2": 0.37785669527795496,
1458
+ "pass@4": 0.45953530245656216,
1459
+ "pass@8": 0.5290022873626415,
1460
+ "pass@16": 0.5902007338149695,
1461
+ "pass@32": 0.6466124054432371,
1462
+ "pass@64": 0.6966004194500958,
1463
+ "pass@128": 0.7417417417417418
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.2728978978978979,
1467
+ "pass@2": 0.36531228571090785,
1468
+ "pass@4": 0.4511123266123264,
1469
+ "pass@8": 0.5250073329043563,
1470
+ "pass@16": 0.5869595546727128,
1471
+ "pass@32": 0.638416275402605,
1472
+ "pass@64": 0.6777993410094819,
1473
+ "pass@128": 0.7057057057057057
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 648988,
1480
+ "answer_accuracy": 0.7903692517537022,
1481
+ "avg_response_len": 284.4923129384256,
1482
+ "resp_tokens_sum": 233602328,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.7903692517537022,
1485
+ "pass@2": 0.8308205308976789,
1486
+ "pass@4": 0.8615349426362584,
1487
+ "pass@8": 0.8856540052607581,
1488
+ "pass@16": 0.9049316985397561,
1489
+ "pass@32": 0.9202179500084126,
1490
+ "pass@64": 0.9324577617228758,
1491
+ "pass@128": 0.9429462197973499
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9163391487730062,
1496
+ "pass@2": 0.9258629564996861,
1497
+ "pass@4": 0.9336692571212339,
1498
+ "pass@8": 0.9416640558086454,
1499
+ "pass@16": 0.9500473404479759,
1500
+ "pass@32": 0.9578994674031669,
1501
+ "pass@64": 0.9655855093279391,
1502
+ "pass@128": 0.9754601226993865
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 0.9999762537993921,
1506
+ "pass@2": 1.0,
1507
+ "pass@4": 1.0,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9982717803030303,
1516
+ "pass@2": 0.9995746092817943,
1517
+ "pass@4": 0.9999632247673588,
1518
+ "pass@8": 0.9999996922182559,
1519
+ "pass@16": 0.9999999999913911,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9986131656804734,
1526
+ "pass@2": 0.999854036597866,
1527
+ "pass@4": 0.9999976565429319,
1528
+ "pass@8": 0.9999999993354288,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.9863224637681159,
1536
+ "pass@2": 0.9906003937007875,
1537
+ "pass@4": 0.9930274229308294,
1538
+ "pass@8": 0.9950581299215392,
1539
+ "pass@16": 0.9969160592890997,
1540
+ "pass@32": 0.998278697645461,
1541
+ "pass@64": 0.9992808085046374,
1542
+ "pass@128": 1.0
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.9967765042979942,
1546
+ "pass@2": 0.9973253869322924,
1547
+ "pass@4": 0.9975613690301607,
1548
+ "pass@8": 0.9979359856831456,
1549
+ "pass@16": 0.9985470770655008,
1550
+ "pass@32": 0.9993380089666443,
1551
+ "pass@64": 0.9999174243620693,
1552
+ "pass@128": 1.0
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9916230130057804,
1556
+ "pass@2": 0.9936738342587959,
1557
+ "pass@4": 0.9944413127911034,
1558
+ "pass@8": 0.9949076292657711,
1559
+ "pass@16": 0.9955414667011108,
1560
+ "pass@32": 0.9966227126233496,
1561
+ "pass@64": 0.9982021755951027,
1562
+ "pass@128": 1.0
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9815140845070423,
1566
+ "pass@2": 0.9889029056227125,
1567
+ "pass@4": 0.9929117410499744,
1568
+ "pass@8": 0.9949431586627508,
1569
+ "pass@16": 0.9962341274090126,
1570
+ "pass@32": 0.9974052544005088,
1571
+ "pass@64": 0.9985526092972625,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9739427395209581,
1576
+ "pass@2": 0.9802412890753925,
1577
+ "pass@4": 0.9834003469439074,
1578
+ "pass@8": 0.9861000031633753,
1579
+ "pass@16": 0.9883617473597814,
1580
+ "pass@32": 0.9900186779076805,
1581
+ "pass@64": 0.9917721721639942,
1582
+ "pass@128": 0.9940119760479041
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.942371046686747,
1586
+ "pass@2": 0.9614907711554884,
1587
+ "pass@4": 0.9709818822534232,
1588
+ "pass@8": 0.9762108984873451,
1589
+ "pass@16": 0.9798418328465113,
1590
+ "pass@32": 0.9832802016718227,
1591
+ "pass@64": 0.9869836840656171,
1592
+ "pass@128": 0.9909638554216867
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8921189263322884,
1596
+ "pass@2": 0.9154603923431991,
1597
+ "pass@4": 0.9262470747191082,
1598
+ "pass@8": 0.932674949530915,
1599
+ "pass@16": 0.9378135612950852,
1600
+ "pass@32": 0.942572203878569,
1601
+ "pass@64": 0.947465209542479,
1602
+ "pass@128": 0.9529780564263323
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.7801995798319328,
1606
+ "pass@2": 0.8263980789166057,
1607
+ "pass@4": 0.8551042666410398,
1608
+ "pass@8": 0.8787522658307619,
1609
+ "pass@16": 0.8997253159536589,
1610
+ "pass@32": 0.9175379858563458,
1611
+ "pass@64": 0.9326565965286922,
1612
+ "pass@128": 0.9439775910364145
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.7647112573099415,
1616
+ "pass@2": 0.8316522626283556,
1617
+ "pass@4": 0.8720206605205054,
1618
+ "pass@8": 0.899213977549253,
1619
+ "pass@16": 0.9185417582132726,
1620
+ "pass@32": 0.9327907985067769,
1621
+ "pass@64": 0.9426059507605182,
1622
+ "pass@128": 0.9502923976608187
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.6902225378787878,
1626
+ "pass@2": 0.7702934860415173,
1627
+ "pass@4": 0.8277060339048522,
1628
+ "pass@8": 0.870181513451221,
1629
+ "pass@16": 0.9009560991102984,
1630
+ "pass@32": 0.9213126526056608,
1631
+ "pass@64": 0.9341966745085428,
1632
+ "pass@128": 0.9424242424242424
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.5663951210826211,
1636
+ "pass@2": 0.6729072728088475,
1637
+ "pass@4": 0.747877350267114,
1638
+ "pass@8": 0.8014397473398795,
1639
+ "pass@16": 0.8403655118855234,
1640
+ "pass@32": 0.8670101699239859,
1641
+ "pass@64": 0.8852311741120641,
1642
+ "pass@128": 0.9002849002849003
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.499708850931677,
1646
+ "pass@2": 0.6042562020100748,
1647
+ "pass@4": 0.6857853701905272,
1648
+ "pass@8": 0.7446535294212936,
1649
+ "pass@16": 0.787459898111565,
1650
+ "pass@32": 0.8184923783964188,
1651
+ "pass@64": 0.840558376080876,
1652
+ "pass@128": 0.860248447204969
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.47116268382352944,
1656
+ "pass@2": 0.5732706837656322,
1657
+ "pass@4": 0.6578277650036389,
1658
+ "pass@8": 0.7272127940450731,
1659
+ "pass@16": 0.7824553129641325,
1660
+ "pass@32": 0.8258777832780535,
1661
+ "pass@64": 0.858667717455225,
1662
+ "pass@128": 0.8823529411764706
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.2761346726190476,
1666
+ "pass@2": 0.365666449896888,
1667
+ "pass@4": 0.45070808615440894,
1668
+ "pass@8": 0.529863381029973,
1669
+ "pass@16": 0.6031771625339937,
1670
+ "pass@32": 0.6680454701301732,
1671
+ "pass@64": 0.7240389650730391,
1672
+ "pass@128": 0.7708333333333334
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.277156624251497,
1676
+ "pass@2": 0.3758873738742987,
1677
+ "pass@4": 0.46919662278367874,
1678
+ "pass@8": 0.547041580122116,
1679
+ "pass@16": 0.608975135388662,
1680
+ "pass@32": 0.6595693283084222,
1681
+ "pass@64": 0.7034790052169011,
1682
+ "pass@128": 0.7455089820359282
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 630014,
1689
+ "answer_accuracy": 0.7863851054481547,
1690
+ "avg_response_len": 304.9374862697715,
1691
+ "resp_tokens_sum": 244301277,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.7863851054481547,
1694
+ "pass@2": 0.8276325988214681,
1695
+ "pass@4": 0.8593780909146573,
1696
+ "pass@8": 0.8839933436948046,
1697
+ "pass@16": 0.902981023040986,
1698
+ "pass@32": 0.917834536552863,
1699
+ "pass@64": 0.9295588405832169,
1700
+ "pass@128": 0.9389678862438089
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.9421175373134328,
1705
+ "pass@2": 0.9587580062286991,
1706
+ "pass@4": 0.9685092193699665,
1707
+ "pass@8": 0.9760498065360441,
1708
+ "pass@16": 0.9825003934245147,
1709
+ "pass@32": 0.9871221085392369,
1710
+ "pass@64": 0.9894357905723764,
1711
+ "pass@128": 0.991044776119403
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 0.9999330357142857,
1715
+ "pass@2": 0.9999996484814399,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9966517857142857,
1725
+ "pass@2": 0.9971243629622828,
1726
+ "pass@4": 0.9972639609917566,
1727
+ "pass@8": 0.99743893615483,
1728
+ "pass@16": 0.9977703679897155,
1729
+ "pass@32": 0.9983643626179382,
1730
+ "pass@64": 0.9992768761047727,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9989720394736842,
1735
+ "pass@2": 0.9998701334208223,
1736
+ "pass@4": 0.9999958955240242,
1737
+ "pass@8": 0.9999999968048913,
1738
+ "pass@16": 0.9999999999999999,
1739
+ "pass@32": 1.0,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.974172794117647,
1745
+ "pass@2": 0.9799802425891616,
1746
+ "pass@4": 0.9851549995588786,
1747
+ "pass@8": 0.9903396378716414,
1748
+ "pass@16": 0.9949618620195534,
1749
+ "pass@32": 0.9978909857983171,
1750
+ "pass@64": 0.9992638569390833,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.98957511829653,
1755
+ "pass@2": 0.9909034315308379,
1756
+ "pass@4": 0.9923643261075016,
1757
+ "pass@8": 0.9937184520602704,
1758
+ "pass@16": 0.9949870433431968,
1759
+ "pass@32": 0.9963097387793499,
1760
+ "pass@64": 0.9980377056438621,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.984375,
1765
+ "pass@2": 0.9912449074304795,
1766
+ "pass@4": 0.994530023244721,
1767
+ "pass@8": 0.9967387415639558,
1768
+ "pass@16": 0.9985927711809854,
1769
+ "pass@32": 0.9996839030827411,
1770
+ "pass@64": 0.9999901863988568,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9845626876876877,
1775
+ "pass@2": 0.9914509736114068,
1776
+ "pass@4": 0.9942883507804767,
1777
+ "pass@8": 0.9953259254730494,
1778
+ "pass@16": 0.9963787369181493,
1779
+ "pass@32": 0.9978019463673319,
1780
+ "pass@64": 0.9992136480632735,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9767132234726688,
1785
+ "pass@2": 0.9858248727751473,
1786
+ "pass@4": 0.9893246072335813,
1787
+ "pass@8": 0.9916823703543539,
1788
+ "pass@16": 0.9934710727259167,
1789
+ "pass@32": 0.995379009876187,
1790
+ "pass@64": 0.9975736224486328,
1791
+ "pass@128": 1.0
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.9344246031746032,
1795
+ "pass@2": 0.956496453568304,
1796
+ "pass@4": 0.9680845546092448,
1797
+ "pass@8": 0.9746013817828894,
1798
+ "pass@16": 0.979128362963511,
1799
+ "pass@32": 0.9823612720845869,
1800
+ "pass@64": 0.9839103382187553,
1801
+ "pass@128": 0.9841269841269841
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8939698012232415,
1805
+ "pass@2": 0.9233999133135886,
1806
+ "pass@4": 0.940026338164151,
1807
+ "pass@8": 0.9508489849950189,
1808
+ "pass@16": 0.9588264089267363,
1809
+ "pass@32": 0.9654115182769986,
1810
+ "pass@64": 0.9710843168080171,
1811
+ "pass@128": 0.9755351681957186
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.8054591049382716,
1815
+ "pass@2": 0.8580877260134149,
1816
+ "pass@4": 0.8874315097186924,
1817
+ "pass@8": 0.9042751337372295,
1818
+ "pass@16": 0.9160205919035335,
1819
+ "pass@32": 0.9253222868446103,
1820
+ "pass@64": 0.9338060490002721,
1821
+ "pass@128": 0.9444444444444444
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.7650686201780416,
1825
+ "pass@2": 0.8257855761816872,
1826
+ "pass@4": 0.8638935942539817,
1827
+ "pass@8": 0.8898587056265527,
1828
+ "pass@16": 0.909471720507359,
1829
+ "pass@32": 0.9247618852817582,
1830
+ "pass@64": 0.9359455063598505,
1831
+ "pass@128": 0.9436201780415431
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.6778308256172839,
1835
+ "pass@2": 0.7494824323174885,
1836
+ "pass@4": 0.801531108958602,
1837
+ "pass@8": 0.8410958117430899,
1838
+ "pass@16": 0.8703430132240998,
1839
+ "pass@32": 0.8920039907677088,
1840
+ "pass@64": 0.9082821137578678,
1841
+ "pass@128": 0.9197530864197531
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.5564850663716814,
1845
+ "pass@2": 0.659352280909577,
1846
+ "pass@4": 0.7384296702956372,
1847
+ "pass@8": 0.7974724410886839,
1848
+ "pass@16": 0.8379566282804083,
1849
+ "pass@32": 0.867540734508324,
1850
+ "pass@64": 0.8941426993490582,
1851
+ "pass@128": 0.9174041297935103
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.44517131024096385,
1855
+ "pass@2": 0.5476650697277298,
1856
+ "pass@4": 0.630717065393181,
1857
+ "pass@8": 0.696161286495951,
1858
+ "pass@16": 0.7486587759712936,
1859
+ "pass@32": 0.7898353056767135,
1860
+ "pass@64": 0.8198061896401653,
1861
+ "pass@128": 0.8433734939759037
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.44601814516129035,
1865
+ "pass@2": 0.5429264827279653,
1866
+ "pass@4": 0.6179406434074768,
1867
+ "pass@8": 0.6738175548059356,
1868
+ "pass@16": 0.7170866104454663,
1869
+ "pass@32": 0.7528154066701483,
1870
+ "pass@64": 0.7842687233254886,
1871
+ "pass@128": 0.8129032258064516
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.28757552870090636,
1875
+ "pass@2": 0.3801985453291147,
1876
+ "pass@4": 0.4683336989409553,
1877
+ "pass@8": 0.5477159165532208,
1878
+ "pass@16": 0.6142955679985431,
1879
+ "pass@32": 0.6685448647183322,
1880
+ "pass@64": 0.7121010913154943,
1881
+ "pass@128": 0.7462235649546828
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.2712321696696697,
1885
+ "pass@2": 0.377846350287295,
1886
+ "pass@4": 0.48275687892617025,
1887
+ "pass@8": 0.5715013261561981,
1888
+ "pass@16": 0.6392863359899448,
1889
+ "pass@32": 0.6912394364961897,
1890
+ "pass@64": 0.7295567463044594,
1891
+ "pass@128": 0.7567567567567568
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block4-epoch1-op11-14/SDAR-100M-block4-epoch1-op11-14_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bf9380e9142a1e3bf1513ee5fc0db29b04a052b87ae04445cf228f7743116a1
3
+ size 2688868264
results/composition/SDAR-100M-block4-epoch1-op11-14/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block4-epoch1-op11-14,0.9827213541666666,0.615015625,0.7891920230263157,0.0,0.0,0.0,199.76456336805555,367.26526484375,1152000,1280000,2432000
results/composition/SDAR-100M-block4-epoch1-op11-14/summary.json ADDED
@@ -0,0 +1,645 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "checkpoint": "SDAR-100M-block4-epoch1-op11-14",
4
+ "id_acc": 0.9827213541666666,
5
+ "ood_acc": 0.615015625,
6
+ "total_acc": 0.7891920230263157,
7
+ "id_avg_loss": 0.0,
8
+ "ood_avg_loss": 0.0,
9
+ "total_avg_loss": 0.0,
10
+ "id_avg_resp_len": 199.76456336805555,
11
+ "ood_avg_resp_len": 367.26526484375,
12
+ "count_id": 1152000,
13
+ "count_ood": 1280000,
14
+ "count_total": 2432000,
15
+ "template_metrics": {
16
+ "crazy_zootopia": {
17
+ "count": 809728,
18
+ "correct": 640313,
19
+ "answer_accuracy": 0.790775420882074,
20
+ "avg_response_len": 274.5671521794973,
21
+ "resp_tokens_sum": 222324711,
22
+ "pass_at_k": {
23
+ "pass@1": 0.790775420882074,
24
+ "pass@2": 0.8310315267138889,
25
+ "pass@4": 0.8607817283045475,
26
+ "pass@8": 0.8837751849257428,
27
+ "pass@16": 0.902361735206419,
28
+ "pass@32": 0.9174584369632929,
29
+ "pass@64": 0.9296724648662624,
30
+ "pass@128": 0.9399304457793234
31
+ },
32
+ "per_op_pass_at_k": {
33
+ "10": {
34
+ "pass@1": 0.9259080014749262,
35
+ "pass@2": 0.9433224020393468,
36
+ "pass@4": 0.9555353711317055,
37
+ "pass@8": 0.9626092113862927,
38
+ "pass@16": 0.9672899140408969,
39
+ "pass@32": 0.971873515401767,
40
+ "pass@64": 0.9768429344531748,
41
+ "pass@128": 0.9823008849557522
42
+ },
43
+ "2": {
44
+ "pass@1": 0.9999513239875389,
45
+ "pass@2": 1.0,
46
+ "pass@4": 1.0,
47
+ "pass@8": 1.0,
48
+ "pass@16": 1.0,
49
+ "pass@32": 1.0,
50
+ "pass@64": 1.0,
51
+ "pass@128": 1.0
52
+ },
53
+ "3": {
54
+ "pass@1": 0.9999044342507645,
55
+ "pass@2": 0.9999977425413568,
56
+ "pass@4": 0.9999999997133384,
57
+ "pass@8": 1.0,
58
+ "pass@16": 1.0,
59
+ "pass@32": 1.0,
60
+ "pass@64": 1.0,
61
+ "pass@128": 1.0
62
+ },
63
+ "4": {
64
+ "pass@1": 0.9938720703125,
65
+ "pass@2": 0.9946638933316929,
66
+ "pass@4": 0.995454402184102,
67
+ "pass@8": 0.9966304348689696,
68
+ "pass@16": 0.9980442612102862,
69
+ "pass@32": 0.9992470856726705,
70
+ "pass@64": 0.9999099032277166,
71
+ "pass@128": 1.0
72
+ },
73
+ "5": {
74
+ "pass@1": 0.9814980158730159,
75
+ "pass@2": 0.9859626921634798,
76
+ "pass@4": 0.9891130867570126,
77
+ "pass@8": 0.9915862161820935,
78
+ "pass@16": 0.9933890901170934,
79
+ "pass@32": 0.9943969337435874,
80
+ "pass@64": 0.9952380498015997,
81
+ "pass@128": 0.9968253968253968
82
+ },
83
+ "6": {
84
+ "pass@1": 0.992374625748503,
85
+ "pass@2": 0.9937386545806025,
86
+ "pass@4": 0.9939952238380383,
87
+ "pass@8": 0.9940119126755125,
88
+ "pass@16": 0.9940119760476699,
89
+ "pass@32": 0.9940119760479041,
90
+ "pass@64": 0.9940119760479041,
91
+ "pass@128": 0.9940119760479041
92
+ },
93
+ "7": {
94
+ "pass@1": 0.9904539571005917,
95
+ "pass@2": 0.9942641114941992,
96
+ "pass@4": 0.995715497378656,
97
+ "pass@8": 0.9967252188259338,
98
+ "pass@16": 0.9975999557269617,
99
+ "pass@32": 0.9983364053352927,
100
+ "pass@64": 0.9992661788080938,
101
+ "pass@128": 1.0
102
+ },
103
+ "8": {
104
+ "pass@1": 0.9777393830128205,
105
+ "pass@2": 0.9862255987532803,
106
+ "pass@4": 0.990327881490775,
107
+ "pass@8": 0.9937928780958446,
108
+ "pass@16": 0.996912117506564,
109
+ "pass@32": 0.9990931142481682,
110
+ "pass@64": 0.9999455717474616,
111
+ "pass@128": 1.0
112
+ },
113
+ "9": {
114
+ "pass@1": 0.9797535211267606,
115
+ "pass@2": 0.9886772901186649,
116
+ "pass@4": 0.9919861029695232,
117
+ "pass@8": 0.9939178049019348,
118
+ "pass@16": 0.9951668490880456,
119
+ "pass@32": 0.996002421656276,
120
+ "pass@64": 0.9968393035287623,
121
+ "pass@128": 0.9971830985915493
122
+ },
123
+ "11": {
124
+ "pass@1": 0.9369688385269122,
125
+ "pass@2": 0.9556891436729048,
126
+ "pass@4": 0.9681532095138248,
127
+ "pass@8": 0.9773977476860249,
128
+ "pass@16": 0.9836760616393073,
129
+ "pass@32": 0.9879154034727532,
130
+ "pass@64": 0.9905512293835983,
131
+ "pass@128": 0.9915014164305949
132
+ },
133
+ "12": {
134
+ "pass@1": 0.896340925141243,
135
+ "pass@2": 0.9266628019484855,
136
+ "pass@4": 0.9416773099337153,
137
+ "pass@8": 0.9499502893000226,
138
+ "pass@16": 0.9556055452848286,
139
+ "pass@32": 0.9601108862491841,
140
+ "pass@64": 0.9653730933928352,
141
+ "pass@128": 0.9717514124293786
142
+ },
143
+ "13": {
144
+ "pass@1": 0.8189410266457681,
145
+ "pass@2": 0.8701832590773324,
146
+ "pass@4": 0.9008475028003936,
147
+ "pass@8": 0.9223142658352539,
148
+ "pass@16": 0.9375126996972292,
149
+ "pass@32": 0.9461958750908102,
150
+ "pass@64": 0.9519937435789704,
151
+ "pass@128": 0.9592476489028213
152
+ },
153
+ "14": {
154
+ "pass@1": 0.8048091900311527,
155
+ "pass@2": 0.8633829138764197,
156
+ "pass@4": 0.8936096031220395,
157
+ "pass@8": 0.9133137052944442,
158
+ "pass@16": 0.9308204065370922,
159
+ "pass@32": 0.9458697424997475,
160
+ "pass@64": 0.9578949685863664,
161
+ "pass@128": 0.9688473520249221
162
+ },
163
+ "15": {
164
+ "pass@1": 0.7142792630057804,
165
+ "pass@2": 0.7852296777570432,
166
+ "pass@4": 0.8336160666874729,
167
+ "pass@8": 0.8703838255639478,
168
+ "pass@16": 0.8991492741726431,
169
+ "pass@32": 0.9217285219094363,
170
+ "pass@64": 0.9390414573776548,
171
+ "pass@128": 0.9508670520231214
172
+ },
173
+ "16": {
174
+ "pass@1": 0.5767389112903226,
175
+ "pass@2": 0.671137207899416,
176
+ "pass@4": 0.7352816832371761,
177
+ "pass@8": 0.7801929643943304,
178
+ "pass@16": 0.8153526018049102,
179
+ "pass@32": 0.8451755884363006,
180
+ "pass@64": 0.8721477928442585,
181
+ "pass@128": 0.896774193548387
182
+ },
183
+ "17": {
184
+ "pass@1": 0.49742593930635837,
185
+ "pass@2": 0.6042979950844295,
186
+ "pass@4": 0.6877961173386561,
187
+ "pass@8": 0.750236429513258,
188
+ "pass@16": 0.797521632533749,
189
+ "pass@32": 0.8336931159590578,
190
+ "pass@64": 0.861816202176084,
191
+ "pass@128": 0.8872832369942196
192
+ },
193
+ "18": {
194
+ "pass@1": 0.4068080357142857,
195
+ "pass@2": 0.5067449381327332,
196
+ "pass@4": 0.5904474240719905,
197
+ "pass@8": 0.6602663145937516,
198
+ "pass@16": 0.7184114317717843,
199
+ "pass@32": 0.7636522860956406,
200
+ "pass@64": 0.7975986551347283,
201
+ "pass@128": 0.8228571428571428
202
+ },
203
+ "19": {
204
+ "pass@1": 0.28638795045045046,
205
+ "pass@2": 0.37785669527795496,
206
+ "pass@4": 0.45953530245656216,
207
+ "pass@8": 0.5290022873626415,
208
+ "pass@16": 0.5902007338149695,
209
+ "pass@32": 0.6466124054432371,
210
+ "pass@64": 0.6966004194500958,
211
+ "pass@128": 0.7417417417417418
212
+ },
213
+ "20": {
214
+ "pass@1": 0.2728978978978979,
215
+ "pass@2": 0.36531228571090785,
216
+ "pass@4": 0.4511123266123264,
217
+ "pass@8": 0.5250073329043563,
218
+ "pass@16": 0.5869595546727128,
219
+ "pass@32": 0.638416275402605,
220
+ "pass@64": 0.6777993410094819,
221
+ "pass@128": 0.7057057057057057
222
+ }
223
+ }
224
+ },
225
+ "teachers_in_school": {
226
+ "count": 821120,
227
+ "correct": 648988,
228
+ "answer_accuracy": 0.7903692517537022,
229
+ "avg_response_len": 284.4923129384256,
230
+ "resp_tokens_sum": 233602328,
231
+ "pass_at_k": {
232
+ "pass@1": 0.7903692517537022,
233
+ "pass@2": 0.8308205308976789,
234
+ "pass@4": 0.8615349426362584,
235
+ "pass@8": 0.8856540052607581,
236
+ "pass@16": 0.9049316985397561,
237
+ "pass@32": 0.9202179500084126,
238
+ "pass@64": 0.9324577617228758,
239
+ "pass@128": 0.9429462197973499
240
+ },
241
+ "per_op_pass_at_k": {
242
+ "10": {
243
+ "pass@1": 0.9163391487730062,
244
+ "pass@2": 0.9258629564996861,
245
+ "pass@4": 0.9336692571212339,
246
+ "pass@8": 0.9416640558086454,
247
+ "pass@16": 0.9500473404479759,
248
+ "pass@32": 0.9578994674031669,
249
+ "pass@64": 0.9655855093279391,
250
+ "pass@128": 0.9754601226993865
251
+ },
252
+ "2": {
253
+ "pass@1": 0.9999762537993921,
254
+ "pass@2": 1.0,
255
+ "pass@4": 1.0,
256
+ "pass@8": 1.0,
257
+ "pass@16": 1.0,
258
+ "pass@32": 1.0,
259
+ "pass@64": 1.0,
260
+ "pass@128": 1.0
261
+ },
262
+ "3": {
263
+ "pass@1": 0.9982717803030303,
264
+ "pass@2": 0.9995746092817943,
265
+ "pass@4": 0.9999632247673588,
266
+ "pass@8": 0.9999996922182559,
267
+ "pass@16": 0.9999999999913911,
268
+ "pass@32": 1.0,
269
+ "pass@64": 1.0,
270
+ "pass@128": 1.0
271
+ },
272
+ "4": {
273
+ "pass@1": 0.9986131656804734,
274
+ "pass@2": 0.999854036597866,
275
+ "pass@4": 0.9999976565429319,
276
+ "pass@8": 0.9999999993354288,
277
+ "pass@16": 1.0,
278
+ "pass@32": 1.0,
279
+ "pass@64": 1.0,
280
+ "pass@128": 1.0
281
+ },
282
+ "5": {
283
+ "pass@1": 0.9863224637681159,
284
+ "pass@2": 0.9906003937007875,
285
+ "pass@4": 0.9930274229308294,
286
+ "pass@8": 0.9950581299215392,
287
+ "pass@16": 0.9969160592890997,
288
+ "pass@32": 0.998278697645461,
289
+ "pass@64": 0.9992808085046374,
290
+ "pass@128": 1.0
291
+ },
292
+ "6": {
293
+ "pass@1": 0.9967765042979942,
294
+ "pass@2": 0.9973253869322924,
295
+ "pass@4": 0.9975613690301607,
296
+ "pass@8": 0.9979359856831456,
297
+ "pass@16": 0.9985470770655008,
298
+ "pass@32": 0.9993380089666443,
299
+ "pass@64": 0.9999174243620693,
300
+ "pass@128": 1.0
301
+ },
302
+ "7": {
303
+ "pass@1": 0.9916230130057804,
304
+ "pass@2": 0.9936738342587959,
305
+ "pass@4": 0.9944413127911034,
306
+ "pass@8": 0.9949076292657711,
307
+ "pass@16": 0.9955414667011108,
308
+ "pass@32": 0.9966227126233496,
309
+ "pass@64": 0.9982021755951027,
310
+ "pass@128": 1.0
311
+ },
312
+ "8": {
313
+ "pass@1": 0.9815140845070423,
314
+ "pass@2": 0.9889029056227125,
315
+ "pass@4": 0.9929117410499744,
316
+ "pass@8": 0.9949431586627508,
317
+ "pass@16": 0.9962341274090126,
318
+ "pass@32": 0.9974052544005088,
319
+ "pass@64": 0.9985526092972625,
320
+ "pass@128": 1.0
321
+ },
322
+ "9": {
323
+ "pass@1": 0.9739427395209581,
324
+ "pass@2": 0.9802412890753925,
325
+ "pass@4": 0.9834003469439074,
326
+ "pass@8": 0.9861000031633753,
327
+ "pass@16": 0.9883617473597814,
328
+ "pass@32": 0.9900186779076805,
329
+ "pass@64": 0.9917721721639942,
330
+ "pass@128": 0.9940119760479041
331
+ },
332
+ "11": {
333
+ "pass@1": 0.942371046686747,
334
+ "pass@2": 0.9614907711554884,
335
+ "pass@4": 0.9709818822534232,
336
+ "pass@8": 0.9762108984873451,
337
+ "pass@16": 0.9798418328465113,
338
+ "pass@32": 0.9832802016718227,
339
+ "pass@64": 0.9869836840656171,
340
+ "pass@128": 0.9909638554216867
341
+ },
342
+ "12": {
343
+ "pass@1": 0.8921189263322884,
344
+ "pass@2": 0.9154603923431991,
345
+ "pass@4": 0.9262470747191082,
346
+ "pass@8": 0.932674949530915,
347
+ "pass@16": 0.9378135612950852,
348
+ "pass@32": 0.942572203878569,
349
+ "pass@64": 0.947465209542479,
350
+ "pass@128": 0.9529780564263323
351
+ },
352
+ "13": {
353
+ "pass@1": 0.7801995798319328,
354
+ "pass@2": 0.8263980789166057,
355
+ "pass@4": 0.8551042666410398,
356
+ "pass@8": 0.8787522658307619,
357
+ "pass@16": 0.8997253159536589,
358
+ "pass@32": 0.9175379858563458,
359
+ "pass@64": 0.9326565965286922,
360
+ "pass@128": 0.9439775910364145
361
+ },
362
+ "14": {
363
+ "pass@1": 0.7647112573099415,
364
+ "pass@2": 0.8316522626283556,
365
+ "pass@4": 0.8720206605205054,
366
+ "pass@8": 0.899213977549253,
367
+ "pass@16": 0.9185417582132726,
368
+ "pass@32": 0.9327907985067769,
369
+ "pass@64": 0.9426059507605182,
370
+ "pass@128": 0.9502923976608187
371
+ },
372
+ "15": {
373
+ "pass@1": 0.6902225378787878,
374
+ "pass@2": 0.7702934860415173,
375
+ "pass@4": 0.8277060339048522,
376
+ "pass@8": 0.870181513451221,
377
+ "pass@16": 0.9009560991102984,
378
+ "pass@32": 0.9213126526056608,
379
+ "pass@64": 0.9341966745085428,
380
+ "pass@128": 0.9424242424242424
381
+ },
382
+ "16": {
383
+ "pass@1": 0.5663951210826211,
384
+ "pass@2": 0.6729072728088475,
385
+ "pass@4": 0.747877350267114,
386
+ "pass@8": 0.8014397473398795,
387
+ "pass@16": 0.8403655118855234,
388
+ "pass@32": 0.8670101699239859,
389
+ "pass@64": 0.8852311741120641,
390
+ "pass@128": 0.9002849002849003
391
+ },
392
+ "17": {
393
+ "pass@1": 0.499708850931677,
394
+ "pass@2": 0.6042562020100748,
395
+ "pass@4": 0.6857853701905272,
396
+ "pass@8": 0.7446535294212936,
397
+ "pass@16": 0.787459898111565,
398
+ "pass@32": 0.8184923783964188,
399
+ "pass@64": 0.840558376080876,
400
+ "pass@128": 0.860248447204969
401
+ },
402
+ "18": {
403
+ "pass@1": 0.47116268382352944,
404
+ "pass@2": 0.5732706837656322,
405
+ "pass@4": 0.6578277650036389,
406
+ "pass@8": 0.7272127940450731,
407
+ "pass@16": 0.7824553129641325,
408
+ "pass@32": 0.8258777832780535,
409
+ "pass@64": 0.858667717455225,
410
+ "pass@128": 0.8823529411764706
411
+ },
412
+ "19": {
413
+ "pass@1": 0.2761346726190476,
414
+ "pass@2": 0.365666449896888,
415
+ "pass@4": 0.45070808615440894,
416
+ "pass@8": 0.529863381029973,
417
+ "pass@16": 0.6031771625339937,
418
+ "pass@32": 0.6680454701301732,
419
+ "pass@64": 0.7240389650730391,
420
+ "pass@128": 0.7708333333333334
421
+ },
422
+ "20": {
423
+ "pass@1": 0.277156624251497,
424
+ "pass@2": 0.3758873738742987,
425
+ "pass@4": 0.46919662278367874,
426
+ "pass@8": 0.547041580122116,
427
+ "pass@16": 0.608975135388662,
428
+ "pass@32": 0.6595693283084222,
429
+ "pass@64": 0.7034790052169011,
430
+ "pass@128": 0.7455089820359282
431
+ }
432
+ }
433
+ },
434
+ "movie_festival_awards": {
435
+ "count": 801152,
436
+ "correct": 630014,
437
+ "answer_accuracy": 0.7863851054481547,
438
+ "avg_response_len": 304.9374862697715,
439
+ "resp_tokens_sum": 244301277,
440
+ "pass_at_k": {
441
+ "pass@1": 0.7863851054481547,
442
+ "pass@2": 0.8276325988214681,
443
+ "pass@4": 0.8593780909146573,
444
+ "pass@8": 0.8839933436948046,
445
+ "pass@16": 0.902981023040986,
446
+ "pass@32": 0.917834536552863,
447
+ "pass@64": 0.9295588405832169,
448
+ "pass@128": 0.9389678862438089
449
+ },
450
+ "per_op_pass_at_k": {
451
+ "10": {
452
+ "pass@1": 0.9421175373134328,
453
+ "pass@2": 0.9587580062286991,
454
+ "pass@4": 0.9685092193699665,
455
+ "pass@8": 0.9760498065360441,
456
+ "pass@16": 0.9825003934245147,
457
+ "pass@32": 0.9871221085392369,
458
+ "pass@64": 0.9894357905723764,
459
+ "pass@128": 0.991044776119403
460
+ },
461
+ "2": {
462
+ "pass@1": 0.9999330357142857,
463
+ "pass@2": 0.9999996484814399,
464
+ "pass@4": 1.0,
465
+ "pass@8": 1.0,
466
+ "pass@16": 1.0,
467
+ "pass@32": 1.0,
468
+ "pass@64": 1.0,
469
+ "pass@128": 1.0
470
+ },
471
+ "3": {
472
+ "pass@1": 0.9966517857142857,
473
+ "pass@2": 0.9971243629622828,
474
+ "pass@4": 0.9972639609917566,
475
+ "pass@8": 0.99743893615483,
476
+ "pass@16": 0.9977703679897155,
477
+ "pass@32": 0.9983643626179382,
478
+ "pass@64": 0.9992768761047727,
479
+ "pass@128": 1.0
480
+ },
481
+ "4": {
482
+ "pass@1": 0.9989720394736842,
483
+ "pass@2": 0.9998701334208223,
484
+ "pass@4": 0.9999958955240242,
485
+ "pass@8": 0.9999999968048913,
486
+ "pass@16": 0.9999999999999999,
487
+ "pass@32": 1.0,
488
+ "pass@64": 1.0,
489
+ "pass@128": 1.0
490
+ },
491
+ "5": {
492
+ "pass@1": 0.974172794117647,
493
+ "pass@2": 0.9799802425891616,
494
+ "pass@4": 0.9851549995588786,
495
+ "pass@8": 0.9903396378716414,
496
+ "pass@16": 0.9949618620195534,
497
+ "pass@32": 0.9978909857983171,
498
+ "pass@64": 0.9992638569390833,
499
+ "pass@128": 1.0
500
+ },
501
+ "6": {
502
+ "pass@1": 0.98957511829653,
503
+ "pass@2": 0.9909034315308379,
504
+ "pass@4": 0.9923643261075016,
505
+ "pass@8": 0.9937184520602704,
506
+ "pass@16": 0.9949870433431968,
507
+ "pass@32": 0.9963097387793499,
508
+ "pass@64": 0.9980377056438621,
509
+ "pass@128": 1.0
510
+ },
511
+ "7": {
512
+ "pass@1": 0.984375,
513
+ "pass@2": 0.9912449074304795,
514
+ "pass@4": 0.994530023244721,
515
+ "pass@8": 0.9967387415639558,
516
+ "pass@16": 0.9985927711809854,
517
+ "pass@32": 0.9996839030827411,
518
+ "pass@64": 0.9999901863988568,
519
+ "pass@128": 1.0
520
+ },
521
+ "8": {
522
+ "pass@1": 0.9845626876876877,
523
+ "pass@2": 0.9914509736114068,
524
+ "pass@4": 0.9942883507804767,
525
+ "pass@8": 0.9953259254730494,
526
+ "pass@16": 0.9963787369181493,
527
+ "pass@32": 0.9978019463673319,
528
+ "pass@64": 0.9992136480632735,
529
+ "pass@128": 1.0
530
+ },
531
+ "9": {
532
+ "pass@1": 0.9767132234726688,
533
+ "pass@2": 0.9858248727751473,
534
+ "pass@4": 0.9893246072335813,
535
+ "pass@8": 0.9916823703543539,
536
+ "pass@16": 0.9934710727259167,
537
+ "pass@32": 0.995379009876187,
538
+ "pass@64": 0.9975736224486328,
539
+ "pass@128": 1.0
540
+ },
541
+ "11": {
542
+ "pass@1": 0.9344246031746032,
543
+ "pass@2": 0.956496453568304,
544
+ "pass@4": 0.9680845546092448,
545
+ "pass@8": 0.9746013817828894,
546
+ "pass@16": 0.979128362963511,
547
+ "pass@32": 0.9823612720845869,
548
+ "pass@64": 0.9839103382187553,
549
+ "pass@128": 0.9841269841269841
550
+ },
551
+ "12": {
552
+ "pass@1": 0.8939698012232415,
553
+ "pass@2": 0.9233999133135886,
554
+ "pass@4": 0.940026338164151,
555
+ "pass@8": 0.9508489849950189,
556
+ "pass@16": 0.9588264089267363,
557
+ "pass@32": 0.9654115182769986,
558
+ "pass@64": 0.9710843168080171,
559
+ "pass@128": 0.9755351681957186
560
+ },
561
+ "13": {
562
+ "pass@1": 0.8054591049382716,
563
+ "pass@2": 0.8580877260134149,
564
+ "pass@4": 0.8874315097186924,
565
+ "pass@8": 0.9042751337372295,
566
+ "pass@16": 0.9160205919035335,
567
+ "pass@32": 0.9253222868446103,
568
+ "pass@64": 0.9338060490002721,
569
+ "pass@128": 0.9444444444444444
570
+ },
571
+ "14": {
572
+ "pass@1": 0.7650686201780416,
573
+ "pass@2": 0.8257855761816872,
574
+ "pass@4": 0.8638935942539817,
575
+ "pass@8": 0.8898587056265527,
576
+ "pass@16": 0.909471720507359,
577
+ "pass@32": 0.9247618852817582,
578
+ "pass@64": 0.9359455063598505,
579
+ "pass@128": 0.9436201780415431
580
+ },
581
+ "15": {
582
+ "pass@1": 0.6778308256172839,
583
+ "pass@2": 0.7494824323174885,
584
+ "pass@4": 0.801531108958602,
585
+ "pass@8": 0.8410958117430899,
586
+ "pass@16": 0.8703430132240998,
587
+ "pass@32": 0.8920039907677088,
588
+ "pass@64": 0.9082821137578678,
589
+ "pass@128": 0.9197530864197531
590
+ },
591
+ "16": {
592
+ "pass@1": 0.5564850663716814,
593
+ "pass@2": 0.659352280909577,
594
+ "pass@4": 0.7384296702956372,
595
+ "pass@8": 0.7974724410886839,
596
+ "pass@16": 0.8379566282804083,
597
+ "pass@32": 0.867540734508324,
598
+ "pass@64": 0.8941426993490582,
599
+ "pass@128": 0.9174041297935103
600
+ },
601
+ "17": {
602
+ "pass@1": 0.44517131024096385,
603
+ "pass@2": 0.5476650697277298,
604
+ "pass@4": 0.630717065393181,
605
+ "pass@8": 0.696161286495951,
606
+ "pass@16": 0.7486587759712936,
607
+ "pass@32": 0.7898353056767135,
608
+ "pass@64": 0.8198061896401653,
609
+ "pass@128": 0.8433734939759037
610
+ },
611
+ "18": {
612
+ "pass@1": 0.44601814516129035,
613
+ "pass@2": 0.5429264827279653,
614
+ "pass@4": 0.6179406434074768,
615
+ "pass@8": 0.6738175548059356,
616
+ "pass@16": 0.7170866104454663,
617
+ "pass@32": 0.7528154066701483,
618
+ "pass@64": 0.7842687233254886,
619
+ "pass@128": 0.8129032258064516
620
+ },
621
+ "19": {
622
+ "pass@1": 0.28757552870090636,
623
+ "pass@2": 0.3801985453291147,
624
+ "pass@4": 0.4683336989409553,
625
+ "pass@8": 0.5477159165532208,
626
+ "pass@16": 0.6142955679985431,
627
+ "pass@32": 0.6685448647183322,
628
+ "pass@64": 0.7121010913154943,
629
+ "pass@128": 0.7462235649546828
630
+ },
631
+ "20": {
632
+ "pass@1": 0.2712321696696697,
633
+ "pass@2": 0.377846350287295,
634
+ "pass@4": 0.48275687892617025,
635
+ "pass@8": 0.5715013261561981,
636
+ "pass@16": 0.6392863359899448,
637
+ "pass@32": 0.6912394364961897,
638
+ "pass@64": 0.7295567463044594,
639
+ "pass@128": 0.7567567567567568
640
+ }
641
+ }
642
+ }
643
+ }
644
+ }
645
+ ]
results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_id_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c249d46d93be9a0a91328ad5dc4c638c329356edc18bedf25d6979e61a56eeb
3
+ size 1792415605
results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_metrics.json ADDED
@@ -0,0 +1,1897 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "SDAR-100M-block4-epoch1-op17-20",
3
+ "id": {
4
+ "answer_accuracy": 0.9801223958333334,
5
+ "avg_response_len": 199.76062586805557,
6
+ "avg_loss": 0.0,
7
+ "count": 1152000,
8
+ "validation_metrics": {},
9
+ "pass_at_k": {
10
+ "pass@1": 0.9801223958333334,
11
+ "pass@2": 0.9871357447506511,
12
+ "pass@4": 0.9909350812919213,
13
+ "pass@8": 0.9933636757087748,
14
+ "pass@16": 0.9950767615885424,
15
+ "pass@32": 0.9964329006344513,
16
+ "pass@64": 0.997616406382146,
17
+ "pass@128": 0.9984444444444445
18
+ },
19
+ "per_op_pass_at_k": {
20
+ "10": {
21
+ "pass@1": 0.9231171875,
22
+ "pass@2": 0.9456815944881896,
23
+ "pass@4": 0.9603091880389951,
24
+ "pass@8": 0.9705846474455383,
25
+ "pass@16": 0.9782733645292219,
26
+ "pass@32": 0.984408624764542,
27
+ "pass@64": 0.9896463722943792,
28
+ "pass@128": 0.993
29
+ },
30
+ "2": {
31
+ "pass@1": 0.9999453125,
32
+ "pass@2": 0.9999987696850394,
33
+ "pass@4": 0.9999999995313086,
34
+ "pass@8": 1.0,
35
+ "pass@16": 1.0,
36
+ "pass@32": 1.0,
37
+ "pass@64": 1.0,
38
+ "pass@128": 1.0
39
+ },
40
+ "3": {
41
+ "pass@1": 0.9976875,
42
+ "pass@2": 0.9987242864173229,
43
+ "pass@4": 0.9990599458192727,
44
+ "pass@8": 0.9991769200116146,
45
+ "pass@16": 0.9993323490285145,
46
+ "pass@32": 0.9995814585676791,
47
+ "pass@64": 0.9998779527559055,
48
+ "pass@128": 1.0
49
+ },
50
+ "4": {
51
+ "pass@1": 0.9980625,
52
+ "pass@2": 0.9988898868110238,
53
+ "pass@4": 0.9991574159167605,
54
+ "pass@8": 0.9993262463197925,
55
+ "pass@16": 0.9995588902104531,
56
+ "pass@32": 0.9998290717623712,
57
+ "pass@64": 0.9999861763011332,
58
+ "pass@128": 1.0
59
+ },
60
+ "5": {
61
+ "pass@1": 0.978015625,
62
+ "pass@2": 0.9848229576771658,
63
+ "pass@4": 0.988770462129734,
64
+ "pass@8": 0.991504906288707,
65
+ "pass@16": 0.993650861087387,
66
+ "pass@32": 0.9955520574897151,
67
+ "pass@64": 0.9970482140193007,
68
+ "pass@128": 0.998
69
+ },
70
+ "6": {
71
+ "pass@1": 0.9920078125,
72
+ "pass@2": 0.9939662893700788,
73
+ "pass@4": 0.9950509417885265,
74
+ "pass@8": 0.9958147741453353,
75
+ "pass@16": 0.9964508524538155,
76
+ "pass@32": 0.9971523647058177,
77
+ "pass@64": 0.99794013537104,
78
+ "pass@128": 0.999
79
+ },
80
+ "7": {
81
+ "pass@1": 0.9861875,
82
+ "pass@2": 0.9924363927165356,
83
+ "pass@4": 0.9952590014998124,
84
+ "pass@8": 0.9965242680443928,
85
+ "pass@16": 0.9971608534876047,
86
+ "pass@32": 0.9976626144839646,
87
+ "pass@64": 0.9982518300815967,
88
+ "pass@128": 0.999
89
+ },
90
+ "8": {
91
+ "pass@1": 0.97428125,
92
+ "pass@2": 0.9852411417322838,
93
+ "pass@4": 0.99118949015748,
94
+ "pass@8": 0.9948321772055063,
95
+ "pass@16": 0.9971259158175276,
96
+ "pass@32": 0.998483992181002,
97
+ "pass@64": 0.9994439058232791,
98
+ "pass@128": 1.0
99
+ },
100
+ "9": {
101
+ "pass@1": 0.971796875,
102
+ "pass@2": 0.9844603838582684,
103
+ "pass@4": 0.9896192867454066,
104
+ "pass@8": 0.9925091419180848,
105
+ "pass@16": 0.9941377676823531,
106
+ "pass@32": 0.9952259217549699,
107
+ "pass@64": 0.9963530707926768,
108
+ "pass@128": 0.997
109
+ }
110
+ },
111
+ "per_op_accuracy": {
112
+ "10": 0.9231171875,
113
+ "2": 0.9999453125,
114
+ "3": 0.9976875,
115
+ "4": 0.9980625,
116
+ "5": 0.978015625,
117
+ "6": 0.9920078125,
118
+ "7": 0.9861875,
119
+ "8": 0.97428125,
120
+ "9": 0.971796875
121
+ },
122
+ "per_op_avg_response_len": {
123
+ "10": 279.0174609375,
124
+ "2": 115.26925,
125
+ "3": 144.67940625,
126
+ "4": 159.4561328125,
127
+ "5": 174.2389375,
128
+ "6": 198.7633125,
129
+ "7": 223.944734375,
130
+ "8": 243.9508515625,
131
+ "9": 258.525546875
132
+ },
133
+ "per_op_avg_loss": {},
134
+ "per_op_length": {
135
+ "10": 279.0174609375,
136
+ "2": 115.26925,
137
+ "3": 144.67940625,
138
+ "4": 159.4561328125,
139
+ "5": 174.2389375,
140
+ "6": 198.7633125,
141
+ "7": 223.944734375,
142
+ "8": 243.9508515625,
143
+ "9": 258.525546875
144
+ },
145
+ "per_op_loss": {},
146
+ "per_template": {
147
+ "crazy_zootopia": {
148
+ "count": 379008,
149
+ "correct": 371099,
150
+ "answer_accuracy": 0.979132366599122,
151
+ "avg_response_len": 190.26763287318474,
152
+ "resp_tokens_sum": 72112955,
153
+ "pass_at_k": {
154
+ "pass@1": 0.979132366599122,
155
+ "pass@2": 0.9861054602217292,
156
+ "pass@4": 0.9899340251224931,
157
+ "pass@8": 0.9924987218896144,
158
+ "pass@16": 0.9943021555612757,
159
+ "pass@32": 0.9957042334254549,
160
+ "pass@64": 0.9968508765699006,
161
+ "pass@128": 0.9976359338061466
162
+ },
163
+ "per_op_pass_at_k": {
164
+ "10": {
165
+ "pass@1": 0.9192477876106194,
166
+ "pass@2": 0.9396735564304464,
167
+ "pass@4": 0.9543033865788902,
168
+ "pass@8": 0.96485112476411,
169
+ "pass@16": 0.9724661046772268,
170
+ "pass@32": 0.9786862175801799,
171
+ "pass@64": 0.9841720997110378,
172
+ "pass@128": 0.9882005899705014
173
+ },
174
+ "2": {
175
+ "pass@1": 0.9998296339563862,
176
+ "pass@2": 0.9999961672431132,
177
+ "pass@4": 0.9999999985399022,
178
+ "pass@8": 1.0,
179
+ "pass@16": 1.0,
180
+ "pass@32": 1.0,
181
+ "pass@64": 1.0,
182
+ "pass@128": 1.0
183
+ },
184
+ "3": {
185
+ "pass@1": 0.9995460626911316,
186
+ "pass@2": 0.999965009391028,
187
+ "pass@4": 0.9999997130519236,
188
+ "pass@8": 0.9999999999935767,
189
+ "pass@16": 1.0,
190
+ "pass@32": 1.0,
191
+ "pass@64": 1.0,
192
+ "pass@128": 1.0
193
+ },
194
+ "4": {
195
+ "pass@1": 0.995849609375,
196
+ "pass@2": 0.996728515625,
197
+ "pass@4": 0.9973700640935508,
198
+ "pass@8": 0.9978945203621713,
199
+ "pass@16": 0.9986215319076661,
200
+ "pass@32": 0.99946584925741,
201
+ "pass@64": 0.999956800941041,
202
+ "pass@128": 1.0
203
+ },
204
+ "5": {
205
+ "pass@1": 0.9784474206349206,
206
+ "pass@2": 0.9849374296962881,
207
+ "pass@4": 0.9890754283690729,
208
+ "pass@8": 0.9921342884438079,
209
+ "pass@16": 0.9943983611486136,
210
+ "pass@32": 0.9960648708043484,
211
+ "pass@64": 0.9967769255677263,
212
+ "pass@128": 0.9968253968253968
213
+ },
214
+ "6": {
215
+ "pass@1": 0.9922342814371258,
216
+ "pass@2": 0.9939438298599651,
217
+ "pass@4": 0.9941038252516338,
218
+ "pass@8": 0.9941991013702666,
219
+ "pass@16": 0.9943862275449101,
220
+ "pass@32": 0.9947604790419161,
221
+ "pass@64": 0.9955089820359282,
222
+ "pass@128": 0.9970059880239521
223
+ },
224
+ "7": {
225
+ "pass@1": 0.9887435281065089,
226
+ "pass@2": 0.9931025922517821,
227
+ "pass@4": 0.995356032973689,
228
+ "pass@8": 0.99687459097789,
229
+ "pass@16": 0.997698902492233,
230
+ "pass@32": 0.9983400676934957,
231
+ "pass@64": 0.9992661790057259,
232
+ "pass@128": 1.0
233
+ },
234
+ "8": {
235
+ "pass@1": 0.9668469551282052,
236
+ "pass@2": 0.9809139505097918,
237
+ "pass@4": 0.9884821623859518,
238
+ "pass@8": 0.9934550534186496,
239
+ "pass@16": 0.9968600221493318,
240
+ "pass@32": 0.9988651869448563,
241
+ "pass@64": 0.9998085724581219,
242
+ "pass@128": 1.0
243
+ },
244
+ "9": {
245
+ "pass@1": 0.9736575704225352,
246
+ "pass@2": 0.987104566374626,
247
+ "pass@4": 0.9918345429532572,
248
+ "pass@8": 0.9940607002019898,
249
+ "pass@16": 0.9952133611440159,
250
+ "pass@32": 0.9960034370145185,
251
+ "pass@64": 0.9968393035368946,
252
+ "pass@128": 0.9971830985915493
253
+ }
254
+ }
255
+ },
256
+ "teachers_in_school": {
257
+ "count": 390656,
258
+ "correct": 383005,
259
+ "answer_accuracy": 0.980414994266055,
260
+ "avg_response_len": 197.6951153956422,
261
+ "resp_tokens_sum": 77230783,
262
+ "pass_at_k": {
263
+ "pass@1": 0.980414994266055,
264
+ "pass@2": 0.9862497387797863,
265
+ "pass@4": 0.9896156322837595,
266
+ "pass@8": 0.9920707717579302,
267
+ "pass@16": 0.993936643245879,
268
+ "pass@32": 0.9954022931445885,
269
+ "pass@64": 0.9968295599606264,
270
+ "pass@128": 0.9980340760157274
271
+ },
272
+ "per_op_pass_at_k": {
273
+ "10": {
274
+ "pass@1": 0.9164350076687117,
275
+ "pass@2": 0.9361730713492108,
276
+ "pass@4": 0.9499516876916461,
277
+ "pass@8": 0.9618509716297367,
278
+ "pass@16": 0.9718630754382047,
279
+ "pass@32": 0.9796931696551145,
280
+ "pass@64": 0.986608430448021,
281
+ "pass@128": 0.99079754601227
282
+ },
283
+ "2": {
284
+ "pass@1": 1.0,
285
+ "pass@2": 1.0,
286
+ "pass@4": 1.0,
287
+ "pass@8": 1.0,
288
+ "pass@16": 1.0,
289
+ "pass@32": 1.0,
290
+ "pass@64": 1.0,
291
+ "pass@128": 1.0
292
+ },
293
+ "3": {
294
+ "pass@1": 0.9973958333333334,
295
+ "pass@2": 0.9992700131233596,
296
+ "pass@4": 0.9999176963674994,
297
+ "pass@8": 0.9999987979293221,
298
+ "pass@16": 0.9999999998399193,
299
+ "pass@32": 1.0,
300
+ "pass@64": 1.0,
301
+ "pass@128": 1.0
302
+ },
303
+ "4": {
304
+ "pass@1": 0.9990292159763313,
305
+ "pass@2": 0.9998897084517542,
306
+ "pass@4": 0.9999981377142946,
307
+ "pass@8": 0.9999999995762411,
308
+ "pass@16": 1.0,
309
+ "pass@32": 1.0,
310
+ "pass@64": 1.0,
311
+ "pass@128": 1.0
312
+ },
313
+ "5": {
314
+ "pass@1": 0.980570652173913,
315
+ "pass@2": 0.9865802949903003,
316
+ "pass@4": 0.9889661506985535,
317
+ "pass@8": 0.9902841199998196,
318
+ "pass@16": 0.9917663843037684,
319
+ "pass@32": 0.9936594509266741,
320
+ "pass@64": 0.9954604899202021,
321
+ "pass@128": 0.9971014492753624
322
+ },
323
+ "6": {
324
+ "pass@1": 0.995008058739255,
325
+ "pass@2": 0.9963252713038379,
326
+ "pass@4": 0.9970167348350797,
327
+ "pass@8": 0.9973006814242409,
328
+ "pass@16": 0.997492802179556,
329
+ "pass@32": 0.9978510028653035,
330
+ "pass@64": 0.998567335243553,
331
+ "pass@128": 1.0
332
+ },
333
+ "7": {
334
+ "pass@1": 0.9898843930635838,
335
+ "pass@2": 0.9931856196804879,
336
+ "pass@4": 0.9941058232063475,
337
+ "pass@8": 0.9943886704848257,
338
+ "pass@16": 0.9945808983691612,
339
+ "pass@32": 0.9949421965317783,
340
+ "pass@64": 0.9956647398843931,
341
+ "pass@128": 0.9971098265895953
342
+ },
343
+ "8": {
344
+ "pass@1": 0.9714788732394366,
345
+ "pass@2": 0.980618207275147,
346
+ "pass@4": 0.9870130438800786,
347
+ "pass@8": 0.9915782021625016,
348
+ "pass@16": 0.9946980159453276,
349
+ "pass@32": 0.9967270976372445,
350
+ "pass@64": 0.9986017780742946,
351
+ "pass@128": 1.0
352
+ },
353
+ "9": {
354
+ "pass@1": 0.9722352170658682,
355
+ "pass@2": 0.9828459468857563,
356
+ "pass@4": 0.9884671896364751,
357
+ "pass@8": 0.9924020194851084,
358
+ "pass@16": 0.9944314239658919,
359
+ "pass@32": 0.9953209435293606,
360
+ "pass@64": 0.9962633785477046,
361
+ "pass@128": 0.9970059880239521
362
+ }
363
+ }
364
+ },
365
+ "movie_festival_awards": {
366
+ "count": 382336,
367
+ "correct": 374997,
368
+ "answer_accuracy": 0.9808048418145296,
369
+ "avg_response_len": 211.28144616253766,
370
+ "resp_tokens_sum": 80780503,
371
+ "pass_at_k": {
372
+ "pass@1": 0.9808048418145296,
373
+ "pass@2": 0.9890623476007588,
374
+ "pass@4": 0.993275585373138,
375
+ "pass@8": 0.9955421394237094,
376
+ "pass@16": 0.9970095535632802,
377
+ "pass@32": 0.9982082597455659,
378
+ "pass@64": 0.9991792416859733,
379
+ "pass@128": 0.9996652159357214
380
+ },
381
+ "per_op_pass_at_k": {
382
+ "10": {
383
+ "pass@1": 0.933535447761194,
384
+ "pass@2": 0.9610144405923143,
385
+ "pass@4": 0.9764659397052976,
386
+ "pass@8": 0.9848856698483921,
387
+ "pass@16": 0.9903880371665308,
388
+ "pass@32": 0.994788130439683,
389
+ "pass@64": 0.9981423348247224,
390
+ "pass@128": 1.0
391
+ },
392
+ "2": {
393
+ "pass@1": 1.0,
394
+ "pass@2": 1.0,
395
+ "pass@4": 1.0,
396
+ "pass@8": 1.0,
397
+ "pass@16": 1.0,
398
+ "pass@32": 1.0,
399
+ "pass@64": 1.0,
400
+ "pass@128": 1.0
401
+ },
402
+ "3": {
403
+ "pass@1": 0.9961962463556852,
404
+ "pass@2": 0.9970163965473705,
405
+ "pass@4": 0.997338775072941,
406
+ "pass@8": 0.9976015064053606,
407
+ "pass@16": 0.9980534958639686,
408
+ "pass@32": 0.9987797625879855,
409
+ "pass@64": 0.9996441771309199,
410
+ "pass@128": 1.0
411
+ },
412
+ "4": {
413
+ "pass@1": 0.9991776315789473,
414
+ "pass@2": 0.999924094603306,
415
+ "pass@4": 0.9999989030976391,
416
+ "pass@8": 0.999999999845404,
417
+ "pass@16": 1.0,
418
+ "pass@32": 1.0,
419
+ "pass@64": 1.0,
420
+ "pass@128": 1.0
421
+ },
422
+ "5": {
423
+ "pass@1": 0.9750229779411764,
424
+ "pass@2": 0.9829337222093564,
425
+ "pass@4": 0.9882893535366903,
426
+ "pass@8": 0.9921605412616765,
427
+ "pass@16": 0.9948705139434515,
428
+ "pass@32": 0.9969973900489489,
429
+ "pass@64": 0.998910627773521,
430
+ "pass@128": 1.0
431
+ },
432
+ "6": {
433
+ "pass@1": 0.9884660883280757,
434
+ "pass@2": 0.9913928407312652,
435
+ "pass@4": 0.9938846173408135,
436
+ "pass@8": 0.9958811876044368,
437
+ "pass@16": 0.9974790678017365,
438
+ "pass@32": 0.9989033586934598,
439
+ "pass@64": 0.9998111525900316,
440
+ "pass@128": 1.0
441
+ },
442
+ "7": {
443
+ "pass@1": 0.9794056566455697,
444
+ "pass@2": 0.9909034560948872,
445
+ "pass@4": 0.9964178719155359,
446
+ "pass@8": 0.9984878997029001,
447
+ "pass@16": 0.9994102329414561,
448
+ "pass@32": 0.9999166822897716,
449
+ "pass@64": 0.9999995619546246,
450
+ "pass@128": 1.0
451
+ },
452
+ "8": {
453
+ "pass@1": 0.9842342342342343,
454
+ "pass@2": 0.9942238005722256,
455
+ "pass@4": 0.9981784532001067,
456
+ "pass@8": 0.9995914077213195,
457
+ "pass@16": 0.9999633430821163,
458
+ "pass@32": 0.9999998023813371,
459
+ "pass@64": 0.9999999999999107,
460
+ "pass@128": 1.0
461
+ },
462
+ "9": {
463
+ "pass@1": 0.9692021704180064,
464
+ "pass@2": 0.983175937412968,
465
+ "pass@4": 0.9883279185158123,
466
+ "pass@8": 0.9908531152358594,
467
+ "pass@16": 0.9925946265968474,
468
+ "pass@32": 0.9942363552283263,
469
+ "pass@64": 0.9958943717110796,
470
+ "pass@128": 0.9967845659163987
471
+ }
472
+ }
473
+ }
474
+ }
475
+ },
476
+ "ood": {
477
+ "answer_accuracy": 0.62273671875,
478
+ "avg_response_len": 372.483859375,
479
+ "avg_loss": 0.0,
480
+ "count": 1280000,
481
+ "validation_metrics": {},
482
+ "pass_at_k": {
483
+ "pass@1": 0.62273671875,
484
+ "pass@2": 0.7036788877952637,
485
+ "pass@4": 0.7644948062710833,
486
+ "pass@8": 0.8099061159706736,
487
+ "pass@16": 0.8439089202647532,
488
+ "pass@32": 0.8701182617095065,
489
+ "pass@64": 0.891005192451688,
490
+ "pass@128": 0.9073
491
+ },
492
+ "per_op_pass_at_k": {
493
+ "11": {
494
+ "pass@1": 0.9151171875,
495
+ "pass@2": 0.9456545275590561,
496
+ "pass@4": 0.9612200796775406,
497
+ "pass@8": 0.9704336964892437,
498
+ "pass@16": 0.9765719952540661,
499
+ "pass@32": 0.9812601474160046,
500
+ "pass@64": 0.9858234208718162,
501
+ "pass@128": 0.99
502
+ },
503
+ "12": {
504
+ "pass@1": 0.8489609375,
505
+ "pass@2": 0.8991227854330703,
506
+ "pass@4": 0.9283502003187095,
507
+ "pass@8": 0.9455536075739712,
508
+ "pass@16": 0.9560643833010687,
509
+ "pass@32": 0.9634465601168278,
510
+ "pass@64": 0.9699825720446108,
511
+ "pass@128": 0.976
512
+ },
513
+ "13": {
514
+ "pass@1": 0.7550703125,
515
+ "pass@2": 0.8208409202755905,
516
+ "pass@4": 0.8626456775403079,
517
+ "pass@8": 0.890856563637159,
518
+ "pass@16": 0.9113103719914247,
519
+ "pass@32": 0.9282444428549655,
520
+ "pass@64": 0.9434633808607322,
521
+ "pass@128": 0.956
522
+ },
523
+ "14": {
524
+ "pass@1": 0.7476953125,
525
+ "pass@2": 0.8176723671259848,
526
+ "pass@4": 0.8618266462317206,
527
+ "pass@8": 0.8918874643176163,
528
+ "pass@16": 0.9141369979757303,
529
+ "pass@32": 0.9316738111035253,
530
+ "pass@64": 0.9455826385211541,
531
+ "pass@128": 0.956
532
+ },
533
+ "15": {
534
+ "pass@1": 0.688828125,
535
+ "pass@2": 0.7694915108267728,
536
+ "pass@4": 0.8278998459880011,
537
+ "pass@8": 0.8700518416925825,
538
+ "pass@16": 0.8990736716148421,
539
+ "pass@32": 0.9185506095706097,
540
+ "pass@64": 0.9329091553818495,
541
+ "pass@128": 0.944
542
+ },
543
+ "16": {
544
+ "pass@1": 0.569984375,
545
+ "pass@2": 0.6710280511811033,
546
+ "pass@4": 0.7460000776152982,
547
+ "pass@8": 0.8009541520148893,
548
+ "pass@16": 0.8402356139913965,
549
+ "pass@32": 0.8684268111847793,
550
+ "pass@64": 0.8891133101235712,
551
+ "pass@128": 0.905
552
+ },
553
+ "17": {
554
+ "pass@1": 0.5203671875,
555
+ "pass@2": 0.6227608267716539,
556
+ "pass@4": 0.7001498685789274,
557
+ "pass@8": 0.7556166147417005,
558
+ "pass@16": 0.7964709061393579,
559
+ "pass@32": 0.8294317709926918,
560
+ "pass@64": 0.8556712149259554,
561
+ "pass@128": 0.873
562
+ },
563
+ "18": {
564
+ "pass@1": 0.4865859375,
565
+ "pass@2": 0.5871370570866156,
566
+ "pass@4": 0.6677094760030015,
567
+ "pass@8": 0.7302382482448312,
568
+ "pass@16": 0.777847828734549,
569
+ "pass@32": 0.8141039437356614,
570
+ "pass@64": 0.8426591360477864,
571
+ "pass@128": 0.867
572
+ },
573
+ "19": {
574
+ "pass@1": 0.3416015625,
575
+ "pass@2": 0.4410281742125985,
576
+ "pass@4": 0.5310940967379066,
577
+ "pass@8": 0.6078161499766409,
578
+ "pass@16": 0.672866477645552,
579
+ "pass@32": 0.728375162906233,
580
+ "pass@64": 0.7747117439388957,
581
+ "pass@128": 0.812
582
+ },
583
+ "20": {
584
+ "pass@1": 0.35315625,
585
+ "pass@2": 0.4620526574803147,
586
+ "pass@4": 0.5580520940194974,
587
+ "pass@8": 0.6356528210181952,
588
+ "pass@16": 0.6945109559994428,
589
+ "pass@32": 0.7376693572137135,
590
+ "pass@64": 0.7701353518004574,
591
+ "pass@128": 0.794
592
+ }
593
+ },
594
+ "per_op_accuracy": {
595
+ "11": 0.9151171875,
596
+ "12": 0.8489609375,
597
+ "13": 0.7550703125,
598
+ "14": 0.7476953125,
599
+ "15": 0.688828125,
600
+ "16": 0.569984375,
601
+ "17": 0.5203671875,
602
+ "18": 0.4865859375,
603
+ "19": 0.3416015625,
604
+ "20": 0.35315625
605
+ },
606
+ "per_op_avg_response_len": {
607
+ "11": 297.70178125,
608
+ "12": 307.7838359375,
609
+ "13": 325.8640703125,
610
+ "14": 352.953453125,
611
+ "15": 372.9394609375,
612
+ "16": 377.9108046875,
613
+ "17": 403.39990625,
614
+ "18": 419.2847265625,
615
+ "19": 419.6512578125,
616
+ "20": 447.349296875
617
+ },
618
+ "per_op_avg_loss": {},
619
+ "per_op_length": {
620
+ "11": 297.70178125,
621
+ "12": 307.7838359375,
622
+ "13": 325.8640703125,
623
+ "14": 352.953453125,
624
+ "15": 372.9394609375,
625
+ "16": 377.9108046875,
626
+ "17": 403.39990625,
627
+ "18": 419.2847265625,
628
+ "19": 419.6512578125,
629
+ "20": 447.349296875
630
+ },
631
+ "per_op_loss": {},
632
+ "per_template": {
633
+ "crazy_zootopia": {
634
+ "count": 430720,
635
+ "correct": 268987,
636
+ "answer_accuracy": 0.6245054791976226,
637
+ "avg_response_len": 354.4513581909361,
638
+ "resp_tokens_sum": 152669289,
639
+ "pass_at_k": {
640
+ "pass@1": 0.6245054791976226,
641
+ "pass@2": 0.7043694279346253,
642
+ "pass@4": 0.7637833167306537,
643
+ "pass@8": 0.8083928321539314,
644
+ "pass@16": 0.842271539248331,
645
+ "pass@32": 0.8685320453378127,
646
+ "pass@64": 0.889781440443194,
647
+ "pass@128": 0.9069836552748886
648
+ },
649
+ "per_op_pass_at_k": {
650
+ "11": {
651
+ "pass@1": 0.9049442280453258,
652
+ "pass@2": 0.9343667049586221,
653
+ "pass@4": 0.9518926340829211,
654
+ "pass@8": 0.963721513224062,
655
+ "pass@16": 0.9715522567392245,
656
+ "pass@32": 0.9772850487323939,
657
+ "pass@64": 0.9837385106596693,
658
+ "pass@128": 0.9915014164305949
659
+ },
660
+ "12": {
661
+ "pass@1": 0.8367319915254238,
662
+ "pass@2": 0.8906847780150366,
663
+ "pass@4": 0.9210060177859121,
664
+ "pass@8": 0.9395740180895393,
665
+ "pass@16": 0.9513328654341852,
666
+ "pass@32": 0.9589354102848824,
667
+ "pass@64": 0.9654301877762941,
668
+ "pass@128": 0.9717514124293786
669
+ },
670
+ "13": {
671
+ "pass@1": 0.7604574843260188,
672
+ "pass@2": 0.82680597894503,
673
+ "pass@4": 0.8719512272956471,
674
+ "pass@8": 0.9045499773982028,
675
+ "pass@16": 0.9266465799635933,
676
+ "pass@32": 0.9410437423571842,
677
+ "pass@64": 0.9509158308973618,
678
+ "pass@128": 0.9592476489028213
679
+ },
680
+ "14": {
681
+ "pass@1": 0.7687889408099688,
682
+ "pass@2": 0.8359775523094659,
683
+ "pass@4": 0.8736571856672118,
684
+ "pass@8": 0.8984409718591754,
685
+ "pass@16": 0.9181805371311775,
686
+ "pass@32": 0.9350351705266242,
687
+ "pass@64": 0.9502913100353946,
688
+ "pass@128": 0.9626168224299065
689
+ },
690
+ "15": {
691
+ "pass@1": 0.6999638728323699,
692
+ "pass@2": 0.7756271761640345,
693
+ "pass@4": 0.8292882124380401,
694
+ "pass@8": 0.869390789970966,
695
+ "pass@16": 0.8979501614129438,
696
+ "pass@32": 0.9183570437604774,
697
+ "pass@64": 0.9343777336335435,
698
+ "pass@128": 0.9450867052023122
699
+ },
700
+ "16": {
701
+ "pass@1": 0.5773941532258065,
702
+ "pass@2": 0.6756885795021594,
703
+ "pass@4": 0.7475061202027162,
704
+ "pass@8": 0.7991479896927378,
705
+ "pass@16": 0.8357307006684896,
706
+ "pass@32": 0.8628873839423076,
707
+ "pass@64": 0.8835885830672884,
708
+ "pass@128": 0.9
709
+ },
710
+ "17": {
711
+ "pass@1": 0.5140895953757225,
712
+ "pass@2": 0.6143314624960177,
713
+ "pass@4": 0.6916131304576808,
714
+ "pass@8": 0.7469650535201684,
715
+ "pass@16": 0.7860427028084981,
716
+ "pass@32": 0.8170511372956204,
717
+ "pass@64": 0.8421452842507103,
718
+ "pass@128": 0.8583815028901735
719
+ },
720
+ "18": {
721
+ "pass@1": 0.4582589285714286,
722
+ "pass@2": 0.5605532902137236,
723
+ "pass@4": 0.6421960956666121,
724
+ "pass@8": 0.7057777001868978,
725
+ "pass@16": 0.7555140844320166,
726
+ "pass@32": 0.7930716900033024,
727
+ "pass@64": 0.822862301855235,
728
+ "pass@128": 0.8514285714285714
729
+ },
730
+ "19": {
731
+ "pass@1": 0.3429288663663664,
732
+ "pass@2": 0.44343987195857243,
733
+ "pass@4": 0.531017780998096,
734
+ "pass@8": 0.602912347397304,
735
+ "pass@16": 0.6667933588207599,
736
+ "pass@32": 0.725113626356638,
737
+ "pass@64": 0.775691031987435,
738
+ "pass@128": 0.8168168168168168
739
+ },
740
+ "20": {
741
+ "pass@1": 0.36878284534534533,
742
+ "pass@2": 0.4766399026979738,
743
+ "pass@4": 0.570342090403114,
744
+ "pass@8": 0.6477507011049768,
745
+ "pass@16": 0.7086113239762887,
746
+ "pass@32": 0.7532795279937846,
747
+ "pass@64": 0.7862179025168473,
748
+ "pass@128": 0.8108108108108109
749
+ }
750
+ }
751
+ },
752
+ "teachers_in_school": {
753
+ "count": 430464,
754
+ "correct": 268409,
755
+ "answer_accuracy": 0.6235341399048469,
756
+ "avg_response_len": 367.8219200676479,
757
+ "resp_tokens_sum": 158334095,
758
+ "pass_at_k": {
759
+ "pass@1": 0.6235341399048469,
760
+ "pass@2": 0.7053542370540048,
761
+ "pass@4": 0.7665724703123105,
762
+ "pass@8": 0.8122417615531752,
763
+ "pass@16": 0.8466933293355862,
764
+ "pass@32": 0.8734423092418954,
765
+ "pass@64": 0.8947083272749418,
766
+ "pass@128": 0.9107939339875112
767
+ },
768
+ "per_op_pass_at_k": {
769
+ "11": {
770
+ "pass@1": 0.9257341867469879,
771
+ "pass@2": 0.9553495725026092,
772
+ "pass@4": 0.9682735500494665,
773
+ "pass@8": 0.9742548441493746,
774
+ "pass@16": 0.978185160269226,
775
+ "pass@32": 0.9822869779847758,
776
+ "pass@64": 0.9871868036674503,
777
+ "pass@128": 0.9909638554216867
778
+ },
779
+ "12": {
780
+ "pass@1": 0.8492358934169278,
781
+ "pass@2": 0.8949850973761508,
782
+ "pass@4": 0.9207448059000464,
783
+ "pass@8": 0.9354093985604018,
784
+ "pass@16": 0.943759425744798,
785
+ "pass@32": 0.9496060392426146,
786
+ "pass@64": 0.9561026755224603,
787
+ "pass@128": 0.9655172413793104
788
+ },
789
+ "13": {
790
+ "pass@1": 0.740983893557423,
791
+ "pass@2": 0.8052852538653259,
792
+ "pass@4": 0.845815909040781,
793
+ "pass@8": 0.8738550513505369,
794
+ "pass@16": 0.895515341616072,
795
+ "pass@32": 0.9155451542423412,
796
+ "pass@64": 0.9353134265154689,
797
+ "pass@128": 0.9523809523809523
798
+ },
799
+ "14": {
800
+ "pass@1": 0.7266081871345029,
801
+ "pass@2": 0.7990618668554583,
802
+ "pass@4": 0.8478731022109073,
803
+ "pass@8": 0.8837960270998939,
804
+ "pass@16": 0.9105606572428289,
805
+ "pass@32": 0.930775272521883,
806
+ "pass@64": 0.9463526228698942,
807
+ "pass@128": 0.9590643274853801
808
+ },
809
+ "15": {
810
+ "pass@1": 0.6870501893939394,
811
+ "pass@2": 0.7726907361011697,
812
+ "pass@4": 0.8339367397257161,
813
+ "pass@8": 0.8770493599523325,
814
+ "pass@16": 0.9072087108288012,
815
+ "pass@32": 0.9277764122574453,
816
+ "pass@64": 0.9424641653105227,
817
+ "pass@128": 0.9545454545454546
818
+ },
819
+ "16": {
820
+ "pass@1": 0.5663728632478633,
821
+ "pass@2": 0.668794655876349,
822
+ "pass@4": 0.74511328258113,
823
+ "pass@8": 0.8007343935830623,
824
+ "pass@16": 0.8421667071713081,
825
+ "pass@32": 0.8733083123762971,
826
+ "pass@64": 0.8957480080042212,
827
+ "pass@128": 0.9116809116809117
828
+ },
829
+ "17": {
830
+ "pass@1": 0.5376067546583851,
831
+ "pass@2": 0.6423940553626447,
832
+ "pass@4": 0.7202086238637875,
833
+ "pass@8": 0.7743041381948508,
834
+ "pass@16": 0.814345713501841,
835
+ "pass@32": 0.8470088033255796,
836
+ "pass@64": 0.8726913345902738,
837
+ "pass@128": 0.8881987577639752
838
+ },
839
+ "18": {
840
+ "pass@1": 0.5092830882352941,
841
+ "pass@2": 0.6142293162343674,
842
+ "pass@4": 0.7005960897351063,
843
+ "pass@8": 0.7701591745990866,
844
+ "pass@16": 0.8221200586922632,
845
+ "pass@32": 0.8587938588657354,
846
+ "pass@64": 0.8857339251123276,
847
+ "pass@128": 0.9058823529411765
848
+ },
849
+ "19": {
850
+ "pass@1": 0.34770275297619047,
851
+ "pass@2": 0.44574237790588667,
852
+ "pass@4": 0.5338633851014157,
853
+ "pass@8": 0.611502029947303,
854
+ "pass@16": 0.6783513518981501,
855
+ "pass@32": 0.7327937015644227,
856
+ "pass@64": 0.7742047877222114,
857
+ "pass@128": 0.8035714285714286
858
+ },
859
+ "20": {
860
+ "pass@1": 0.35043974550898205,
861
+ "pass@2": 0.4594935375312366,
862
+ "pass@4": 0.5528386059714089,
863
+ "pass@8": 0.6238549348742458,
864
+ "pass@16": 0.6762174612820911,
865
+ "pass@32": 0.7171414832463683,
866
+ "pass@64": 0.7511741428622221,
867
+ "pass@128": 0.7754491017964071
868
+ }
869
+ }
870
+ },
871
+ "movie_festival_awards": {
872
+ "count": 418816,
873
+ "correct": 259707,
874
+ "answer_accuracy": 0.6200980860330073,
875
+ "avg_response_len": 395.8204939639364,
876
+ "resp_tokens_sum": 165775956,
877
+ "pass_at_k": {
878
+ "pass@1": 0.6200980860330073,
879
+ "pass@2": 0.7012467768154549,
880
+ "pass@4": 0.7630910709816561,
881
+ "pass@8": 0.8090618078867852,
882
+ "pass@16": 0.8427309922131079,
883
+ "pass@32": 0.8683330680173509,
884
+ "pass@64": 0.8884576017114336,
885
+ "pass@128": 0.9040342298288508
886
+ },
887
+ "per_op_pass_at_k": {
888
+ "11": {
889
+ "pass@1": 0.915327380952381,
890
+ "pass@2": 0.948085786151731,
891
+ "pass@4": 0.9642386070788763,
892
+ "pass@8": 0.9739282351224063,
893
+ "pass@16": 0.9804970648118635,
894
+ "pass@32": 0.9846325349921398,
895
+ "pass@64": 0.986722881877331,
896
+ "pass@128": 0.9873015873015873
897
+ },
898
+ "12": {
899
+ "pass@1": 0.8619313837920489,
900
+ "pass@2": 0.912293969274483,
901
+ "pass@4": 0.9437201129675306,
902
+ "pass@8": 0.9619229572767756,
903
+ "pass@16": 0.9731904957944275,
904
+ "pass@32": 0.9818321050690701,
905
+ "pass@64": 0.9884511684407877,
906
+ "pass@128": 0.9908256880733946
907
+ },
908
+ "13": {
909
+ "pass@1": 0.7652874228395061,
910
+ "pass@2": 0.8321079548216199,
911
+ "pass@4": 0.8720276435352986,
912
+ "pass@8": 0.896107615787007,
913
+ "pass@16": 0.9136146173645132,
914
+ "pass@32": 0.929635398081815,
915
+ "pass@64": 0.9451059800569461,
916
+ "pass@128": 0.9567901234567902
917
+ },
918
+ "14": {
919
+ "pass@1": 0.7490031528189911,
920
+ "pass@2": 0.8191228913058717,
921
+ "pass@4": 0.864718364024971,
922
+ "pass@8": 0.8938565907497251,
923
+ "pass@16": 0.9139148391085333,
924
+ "pass@32": 0.9293839114599252,
925
+ "pass@64": 0.9403161156625768,
926
+ "pass@128": 0.9465875370919882
927
+ },
928
+ "15": {
929
+ "pass@1": 0.6787471064814815,
930
+ "pass@2": 0.7596807560513266,
931
+ "pass@4": 0.8202685196757814,
932
+ "pass@8": 0.8636306777109823,
933
+ "pass@16": 0.8919877814582083,
934
+ "pass@32": 0.9093606678534814,
935
+ "pass@64": 0.9216089042968235,
936
+ "pass@128": 0.9320987654320988
937
+ },
938
+ "16": {
939
+ "pass@1": 0.5669478244837758,
940
+ "pass@2": 0.6690786646691279,
941
+ "pass@4": 0.7455410565382864,
942
+ "pass@8": 0.8028333423672143,
943
+ "pass@16": 0.8423557007877164,
944
+ "pass@32": 0.8684380664265011,
945
+ "pass@64": 0.8872958659682277,
946
+ "pass@128": 0.9026548672566371
947
+ },
948
+ "17": {
949
+ "pass@1": 0.5101891942771084,
950
+ "pass@2": 0.6125037798833123,
951
+ "pass@4": 0.6895920137241878,
952
+ "pass@8": 0.7465083545933107,
953
+ "pass@16": 0.7900024434338143,
954
+ "pass@32": 0.8252868759565376,
955
+ "pass@64": 0.85326005071428,
956
+ "pass@128": 0.8734939759036144
957
+ },
958
+ "18": {
959
+ "pass@1": 0.4936743951612903,
960
+ "pass@2": 0.5874368967487934,
961
+ "pass@4": 0.6604457161604794,
962
+ "pass@8": 0.7140707542442847,
963
+ "pass@16": 0.7545067071870171,
964
+ "pass@32": 0.7888352910327632,
965
+ "pass@64": 0.8177670834201991,
966
+ "pass@128": 0.8419354838709677
967
+ },
968
+ "19": {
969
+ "pass@1": 0.3340728851963746,
970
+ "pass@2": 0.4338164890453644,
971
+ "pass@4": 0.5283597530860003,
972
+ "pass@8": 0.6090080248672055,
973
+ "pass@16": 0.6734085647143847,
974
+ "pass@32": 0.7271711226701715,
975
+ "pass@64": 0.7742411529076022,
976
+ "pass@128": 0.8157099697885196
977
+ },
978
+ "20": {
979
+ "pass@1": 0.34025431681681684,
980
+ "pass@2": 0.45003221725662673,
981
+ "pass@4": 0.5509912418042338,
982
+ "pass@8": 0.6353882561628813,
983
+ "pass@16": 0.6987590181595222,
984
+ "pass@32": 0.7426487056681589,
985
+ "pass@64": 0.7730709505896831,
986
+ "pass@128": 0.7957957957957958
987
+ }
988
+ }
989
+ }
990
+ }
991
+ },
992
+ "total": {
993
+ "answer_accuracy": 0.7920246710526315,
994
+ "avg_loss": 0.0,
995
+ "count": 2432000,
996
+ "validation_metrics": {
997
+ "id": {},
998
+ "ood": {}
999
+ },
1000
+ "pass_at_k": {
1001
+ "pass@1": 0.7920246710526315,
1002
+ "pass@2": 0.837947925300431,
1003
+ "pass@4": 0.871755989175678,
1004
+ "pass@8": 0.8968070653203173,
1005
+ "pass@16": 0.9155147398391692,
1006
+ "pass@32": 0.9299515117265957,
1007
+ "pass@64": 0.9415052411555864,
1008
+ "pass@128": 0.9504736842105264
1009
+ },
1010
+ "per_op_pass_at_k": {
1011
+ "10": {
1012
+ "pass@1": 0.9231171875,
1013
+ "pass@2": 0.9456815944881896,
1014
+ "pass@4": 0.9603091880389951,
1015
+ "pass@8": 0.9705846474455383,
1016
+ "pass@16": 0.9782733645292219,
1017
+ "pass@32": 0.984408624764542,
1018
+ "pass@64": 0.9896463722943792,
1019
+ "pass@128": 0.993
1020
+ },
1021
+ "2": {
1022
+ "pass@1": 0.9999453125,
1023
+ "pass@2": 0.9999987696850394,
1024
+ "pass@4": 0.9999999995313086,
1025
+ "pass@8": 1.0,
1026
+ "pass@16": 1.0,
1027
+ "pass@32": 1.0,
1028
+ "pass@64": 1.0,
1029
+ "pass@128": 1.0
1030
+ },
1031
+ "3": {
1032
+ "pass@1": 0.9976875,
1033
+ "pass@2": 0.9987242864173229,
1034
+ "pass@4": 0.9990599458192727,
1035
+ "pass@8": 0.9991769200116146,
1036
+ "pass@16": 0.9993323490285145,
1037
+ "pass@32": 0.9995814585676791,
1038
+ "pass@64": 0.9998779527559055,
1039
+ "pass@128": 1.0
1040
+ },
1041
+ "4": {
1042
+ "pass@1": 0.9980625,
1043
+ "pass@2": 0.9988898868110238,
1044
+ "pass@4": 0.9991574159167605,
1045
+ "pass@8": 0.9993262463197925,
1046
+ "pass@16": 0.9995588902104531,
1047
+ "pass@32": 0.9998290717623712,
1048
+ "pass@64": 0.9999861763011332,
1049
+ "pass@128": 1.0
1050
+ },
1051
+ "5": {
1052
+ "pass@1": 0.978015625,
1053
+ "pass@2": 0.9848229576771658,
1054
+ "pass@4": 0.988770462129734,
1055
+ "pass@8": 0.991504906288707,
1056
+ "pass@16": 0.993650861087387,
1057
+ "pass@32": 0.9955520574897151,
1058
+ "pass@64": 0.9970482140193007,
1059
+ "pass@128": 0.998
1060
+ },
1061
+ "6": {
1062
+ "pass@1": 0.9920078125,
1063
+ "pass@2": 0.9939662893700788,
1064
+ "pass@4": 0.9950509417885265,
1065
+ "pass@8": 0.9958147741453353,
1066
+ "pass@16": 0.9964508524538155,
1067
+ "pass@32": 0.9971523647058177,
1068
+ "pass@64": 0.99794013537104,
1069
+ "pass@128": 0.999
1070
+ },
1071
+ "7": {
1072
+ "pass@1": 0.9861875,
1073
+ "pass@2": 0.9924363927165356,
1074
+ "pass@4": 0.9952590014998124,
1075
+ "pass@8": 0.9965242680443928,
1076
+ "pass@16": 0.9971608534876047,
1077
+ "pass@32": 0.9976626144839646,
1078
+ "pass@64": 0.9982518300815967,
1079
+ "pass@128": 0.999
1080
+ },
1081
+ "8": {
1082
+ "pass@1": 0.97428125,
1083
+ "pass@2": 0.9852411417322838,
1084
+ "pass@4": 0.99118949015748,
1085
+ "pass@8": 0.9948321772055063,
1086
+ "pass@16": 0.9971259158175276,
1087
+ "pass@32": 0.998483992181002,
1088
+ "pass@64": 0.9994439058232791,
1089
+ "pass@128": 1.0
1090
+ },
1091
+ "9": {
1092
+ "pass@1": 0.971796875,
1093
+ "pass@2": 0.9844603838582684,
1094
+ "pass@4": 0.9896192867454066,
1095
+ "pass@8": 0.9925091419180848,
1096
+ "pass@16": 0.9941377676823531,
1097
+ "pass@32": 0.9952259217549699,
1098
+ "pass@64": 0.9963530707926768,
1099
+ "pass@128": 0.997
1100
+ },
1101
+ "11": {
1102
+ "pass@1": 0.9151171875,
1103
+ "pass@2": 0.9456545275590561,
1104
+ "pass@4": 0.9612200796775406,
1105
+ "pass@8": 0.9704336964892437,
1106
+ "pass@16": 0.9765719952540661,
1107
+ "pass@32": 0.9812601474160046,
1108
+ "pass@64": 0.9858234208718162,
1109
+ "pass@128": 0.99
1110
+ },
1111
+ "12": {
1112
+ "pass@1": 0.8489609375,
1113
+ "pass@2": 0.8991227854330703,
1114
+ "pass@4": 0.9283502003187095,
1115
+ "pass@8": 0.9455536075739712,
1116
+ "pass@16": 0.9560643833010687,
1117
+ "pass@32": 0.9634465601168278,
1118
+ "pass@64": 0.9699825720446108,
1119
+ "pass@128": 0.976
1120
+ },
1121
+ "13": {
1122
+ "pass@1": 0.7550703125,
1123
+ "pass@2": 0.8208409202755905,
1124
+ "pass@4": 0.8626456775403079,
1125
+ "pass@8": 0.890856563637159,
1126
+ "pass@16": 0.9113103719914247,
1127
+ "pass@32": 0.9282444428549655,
1128
+ "pass@64": 0.9434633808607322,
1129
+ "pass@128": 0.956
1130
+ },
1131
+ "14": {
1132
+ "pass@1": 0.7476953125,
1133
+ "pass@2": 0.8176723671259848,
1134
+ "pass@4": 0.8618266462317206,
1135
+ "pass@8": 0.8918874643176163,
1136
+ "pass@16": 0.9141369979757303,
1137
+ "pass@32": 0.9316738111035253,
1138
+ "pass@64": 0.9455826385211541,
1139
+ "pass@128": 0.956
1140
+ },
1141
+ "15": {
1142
+ "pass@1": 0.688828125,
1143
+ "pass@2": 0.7694915108267728,
1144
+ "pass@4": 0.8278998459880011,
1145
+ "pass@8": 0.8700518416925825,
1146
+ "pass@16": 0.8990736716148421,
1147
+ "pass@32": 0.9185506095706097,
1148
+ "pass@64": 0.9329091553818495,
1149
+ "pass@128": 0.944
1150
+ },
1151
+ "16": {
1152
+ "pass@1": 0.569984375,
1153
+ "pass@2": 0.6710280511811033,
1154
+ "pass@4": 0.7460000776152982,
1155
+ "pass@8": 0.8009541520148893,
1156
+ "pass@16": 0.8402356139913965,
1157
+ "pass@32": 0.8684268111847793,
1158
+ "pass@64": 0.8891133101235712,
1159
+ "pass@128": 0.905
1160
+ },
1161
+ "17": {
1162
+ "pass@1": 0.5203671875,
1163
+ "pass@2": 0.6227608267716539,
1164
+ "pass@4": 0.7001498685789274,
1165
+ "pass@8": 0.7556166147417005,
1166
+ "pass@16": 0.7964709061393579,
1167
+ "pass@32": 0.8294317709926918,
1168
+ "pass@64": 0.8556712149259554,
1169
+ "pass@128": 0.873
1170
+ },
1171
+ "18": {
1172
+ "pass@1": 0.4865859375,
1173
+ "pass@2": 0.5871370570866156,
1174
+ "pass@4": 0.6677094760030015,
1175
+ "pass@8": 0.7302382482448312,
1176
+ "pass@16": 0.777847828734549,
1177
+ "pass@32": 0.8141039437356614,
1178
+ "pass@64": 0.8426591360477864,
1179
+ "pass@128": 0.867
1180
+ },
1181
+ "19": {
1182
+ "pass@1": 0.3416015625,
1183
+ "pass@2": 0.4410281742125985,
1184
+ "pass@4": 0.5310940967379066,
1185
+ "pass@8": 0.6078161499766409,
1186
+ "pass@16": 0.672866477645552,
1187
+ "pass@32": 0.728375162906233,
1188
+ "pass@64": 0.7747117439388957,
1189
+ "pass@128": 0.812
1190
+ },
1191
+ "20": {
1192
+ "pass@1": 0.35315625,
1193
+ "pass@2": 0.4620526574803147,
1194
+ "pass@4": 0.5580520940194974,
1195
+ "pass@8": 0.6356528210181952,
1196
+ "pass@16": 0.6945109559994428,
1197
+ "pass@32": 0.7376693572137135,
1198
+ "pass@64": 0.7701353518004574,
1199
+ "pass@128": 0.794
1200
+ }
1201
+ },
1202
+ "per_op_accuracy": {
1203
+ "10": 0.9231171875,
1204
+ "2": 0.9999453125,
1205
+ "3": 0.9976875,
1206
+ "4": 0.9980625,
1207
+ "5": 0.978015625,
1208
+ "6": 0.9920078125,
1209
+ "7": 0.9861875,
1210
+ "8": 0.97428125,
1211
+ "9": 0.971796875,
1212
+ "11": 0.9151171875,
1213
+ "12": 0.8489609375,
1214
+ "13": 0.7550703125,
1215
+ "14": 0.7476953125,
1216
+ "15": 0.688828125,
1217
+ "16": 0.569984375,
1218
+ "17": 0.5203671875,
1219
+ "18": 0.4865859375,
1220
+ "19": 0.3416015625,
1221
+ "20": 0.35315625
1222
+ },
1223
+ "per_op_avg_response_len": {
1224
+ "10": 279.0174609375,
1225
+ "2": 115.26925,
1226
+ "3": 144.67940625,
1227
+ "4": 159.4561328125,
1228
+ "5": 174.2389375,
1229
+ "6": 198.7633125,
1230
+ "7": 223.944734375,
1231
+ "8": 243.9508515625,
1232
+ "9": 258.525546875,
1233
+ "11": 297.70178125,
1234
+ "12": 307.7838359375,
1235
+ "13": 325.8640703125,
1236
+ "14": 352.953453125,
1237
+ "15": 372.9394609375,
1238
+ "16": 377.9108046875,
1239
+ "17": 403.39990625,
1240
+ "18": 419.2847265625,
1241
+ "19": 419.6512578125,
1242
+ "20": 447.349296875
1243
+ },
1244
+ "per_op_avg_loss": {},
1245
+ "per_op_length": {
1246
+ "10": 279.0174609375,
1247
+ "2": 115.26925,
1248
+ "3": 144.67940625,
1249
+ "4": 159.4561328125,
1250
+ "5": 174.2389375,
1251
+ "6": 198.7633125,
1252
+ "7": 223.944734375,
1253
+ "8": 243.9508515625,
1254
+ "9": 258.525546875,
1255
+ "11": 297.70178125,
1256
+ "12": 307.7838359375,
1257
+ "13": 325.8640703125,
1258
+ "14": 352.953453125,
1259
+ "15": 372.9394609375,
1260
+ "16": 377.9108046875,
1261
+ "17": 403.39990625,
1262
+ "18": 419.2847265625,
1263
+ "19": 419.6512578125,
1264
+ "20": 447.349296875
1265
+ },
1266
+ "per_op_loss": {},
1267
+ "per_template": {
1268
+ "crazy_zootopia": {
1269
+ "count": 809728,
1270
+ "correct": 640086,
1271
+ "answer_accuracy": 0.790495079829276,
1272
+ "avg_response_len": 277.60216270154916,
1273
+ "resp_tokens_sum": 224782244,
1274
+ "pass_at_k": {
1275
+ "pass@1": 0.790495079829276,
1276
+ "pass@2": 0.8362411306855028,
1277
+ "pass@4": 0.8696372919991009,
1278
+ "pass@8": 0.8945669610675093,
1279
+ "pass@16": 0.913432249792536,
1280
+ "pass@32": 0.9280573139004922,
1281
+ "pass@64": 0.9398972482792994,
1282
+ "pass@128": 0.9494151122352197
1283
+ },
1284
+ "per_op_pass_at_k": {
1285
+ "10": {
1286
+ "pass@1": 0.9192477876106194,
1287
+ "pass@2": 0.9396735564304464,
1288
+ "pass@4": 0.9543033865788902,
1289
+ "pass@8": 0.96485112476411,
1290
+ "pass@16": 0.9724661046772268,
1291
+ "pass@32": 0.9786862175801799,
1292
+ "pass@64": 0.9841720997110378,
1293
+ "pass@128": 0.9882005899705014
1294
+ },
1295
+ "2": {
1296
+ "pass@1": 0.9998296339563862,
1297
+ "pass@2": 0.9999961672431132,
1298
+ "pass@4": 0.9999999985399022,
1299
+ "pass@8": 1.0,
1300
+ "pass@16": 1.0,
1301
+ "pass@32": 1.0,
1302
+ "pass@64": 1.0,
1303
+ "pass@128": 1.0
1304
+ },
1305
+ "3": {
1306
+ "pass@1": 0.9995460626911316,
1307
+ "pass@2": 0.999965009391028,
1308
+ "pass@4": 0.9999997130519236,
1309
+ "pass@8": 0.9999999999935767,
1310
+ "pass@16": 1.0,
1311
+ "pass@32": 1.0,
1312
+ "pass@64": 1.0,
1313
+ "pass@128": 1.0
1314
+ },
1315
+ "4": {
1316
+ "pass@1": 0.995849609375,
1317
+ "pass@2": 0.996728515625,
1318
+ "pass@4": 0.9973700640935508,
1319
+ "pass@8": 0.9978945203621713,
1320
+ "pass@16": 0.9986215319076661,
1321
+ "pass@32": 0.99946584925741,
1322
+ "pass@64": 0.999956800941041,
1323
+ "pass@128": 1.0
1324
+ },
1325
+ "5": {
1326
+ "pass@1": 0.9784474206349206,
1327
+ "pass@2": 0.9849374296962881,
1328
+ "pass@4": 0.9890754283690729,
1329
+ "pass@8": 0.9921342884438079,
1330
+ "pass@16": 0.9943983611486136,
1331
+ "pass@32": 0.9960648708043484,
1332
+ "pass@64": 0.9967769255677263,
1333
+ "pass@128": 0.9968253968253968
1334
+ },
1335
+ "6": {
1336
+ "pass@1": 0.9922342814371258,
1337
+ "pass@2": 0.9939438298599651,
1338
+ "pass@4": 0.9941038252516338,
1339
+ "pass@8": 0.9941991013702666,
1340
+ "pass@16": 0.9943862275449101,
1341
+ "pass@32": 0.9947604790419161,
1342
+ "pass@64": 0.9955089820359282,
1343
+ "pass@128": 0.9970059880239521
1344
+ },
1345
+ "7": {
1346
+ "pass@1": 0.9887435281065089,
1347
+ "pass@2": 0.9931025922517821,
1348
+ "pass@4": 0.995356032973689,
1349
+ "pass@8": 0.99687459097789,
1350
+ "pass@16": 0.997698902492233,
1351
+ "pass@32": 0.9983400676934957,
1352
+ "pass@64": 0.9992661790057259,
1353
+ "pass@128": 1.0
1354
+ },
1355
+ "8": {
1356
+ "pass@1": 0.9668469551282052,
1357
+ "pass@2": 0.9809139505097918,
1358
+ "pass@4": 0.9884821623859518,
1359
+ "pass@8": 0.9934550534186496,
1360
+ "pass@16": 0.9968600221493318,
1361
+ "pass@32": 0.9988651869448563,
1362
+ "pass@64": 0.9998085724581219,
1363
+ "pass@128": 1.0
1364
+ },
1365
+ "9": {
1366
+ "pass@1": 0.9736575704225352,
1367
+ "pass@2": 0.987104566374626,
1368
+ "pass@4": 0.9918345429532572,
1369
+ "pass@8": 0.9940607002019898,
1370
+ "pass@16": 0.9952133611440159,
1371
+ "pass@32": 0.9960034370145185,
1372
+ "pass@64": 0.9968393035368946,
1373
+ "pass@128": 0.9971830985915493
1374
+ },
1375
+ "11": {
1376
+ "pass@1": 0.9049442280453258,
1377
+ "pass@2": 0.9343667049586221,
1378
+ "pass@4": 0.9518926340829211,
1379
+ "pass@8": 0.963721513224062,
1380
+ "pass@16": 0.9715522567392245,
1381
+ "pass@32": 0.9772850487323939,
1382
+ "pass@64": 0.9837385106596693,
1383
+ "pass@128": 0.9915014164305949
1384
+ },
1385
+ "12": {
1386
+ "pass@1": 0.8367319915254238,
1387
+ "pass@2": 0.8906847780150366,
1388
+ "pass@4": 0.9210060177859121,
1389
+ "pass@8": 0.9395740180895393,
1390
+ "pass@16": 0.9513328654341852,
1391
+ "pass@32": 0.9589354102848824,
1392
+ "pass@64": 0.9654301877762941,
1393
+ "pass@128": 0.9717514124293786
1394
+ },
1395
+ "13": {
1396
+ "pass@1": 0.7604574843260188,
1397
+ "pass@2": 0.82680597894503,
1398
+ "pass@4": 0.8719512272956471,
1399
+ "pass@8": 0.9045499773982028,
1400
+ "pass@16": 0.9266465799635933,
1401
+ "pass@32": 0.9410437423571842,
1402
+ "pass@64": 0.9509158308973618,
1403
+ "pass@128": 0.9592476489028213
1404
+ },
1405
+ "14": {
1406
+ "pass@1": 0.7687889408099688,
1407
+ "pass@2": 0.8359775523094659,
1408
+ "pass@4": 0.8736571856672118,
1409
+ "pass@8": 0.8984409718591754,
1410
+ "pass@16": 0.9181805371311775,
1411
+ "pass@32": 0.9350351705266242,
1412
+ "pass@64": 0.9502913100353946,
1413
+ "pass@128": 0.9626168224299065
1414
+ },
1415
+ "15": {
1416
+ "pass@1": 0.6999638728323699,
1417
+ "pass@2": 0.7756271761640345,
1418
+ "pass@4": 0.8292882124380401,
1419
+ "pass@8": 0.869390789970966,
1420
+ "pass@16": 0.8979501614129438,
1421
+ "pass@32": 0.9183570437604774,
1422
+ "pass@64": 0.9343777336335435,
1423
+ "pass@128": 0.9450867052023122
1424
+ },
1425
+ "16": {
1426
+ "pass@1": 0.5773941532258065,
1427
+ "pass@2": 0.6756885795021594,
1428
+ "pass@4": 0.7475061202027162,
1429
+ "pass@8": 0.7991479896927378,
1430
+ "pass@16": 0.8357307006684896,
1431
+ "pass@32": 0.8628873839423076,
1432
+ "pass@64": 0.8835885830672884,
1433
+ "pass@128": 0.9
1434
+ },
1435
+ "17": {
1436
+ "pass@1": 0.5140895953757225,
1437
+ "pass@2": 0.6143314624960177,
1438
+ "pass@4": 0.6916131304576808,
1439
+ "pass@8": 0.7469650535201684,
1440
+ "pass@16": 0.7860427028084981,
1441
+ "pass@32": 0.8170511372956204,
1442
+ "pass@64": 0.8421452842507103,
1443
+ "pass@128": 0.8583815028901735
1444
+ },
1445
+ "18": {
1446
+ "pass@1": 0.4582589285714286,
1447
+ "pass@2": 0.5605532902137236,
1448
+ "pass@4": 0.6421960956666121,
1449
+ "pass@8": 0.7057777001868978,
1450
+ "pass@16": 0.7555140844320166,
1451
+ "pass@32": 0.7930716900033024,
1452
+ "pass@64": 0.822862301855235,
1453
+ "pass@128": 0.8514285714285714
1454
+ },
1455
+ "19": {
1456
+ "pass@1": 0.3429288663663664,
1457
+ "pass@2": 0.44343987195857243,
1458
+ "pass@4": 0.531017780998096,
1459
+ "pass@8": 0.602912347397304,
1460
+ "pass@16": 0.6667933588207599,
1461
+ "pass@32": 0.725113626356638,
1462
+ "pass@64": 0.775691031987435,
1463
+ "pass@128": 0.8168168168168168
1464
+ },
1465
+ "20": {
1466
+ "pass@1": 0.36878284534534533,
1467
+ "pass@2": 0.4766399026979738,
1468
+ "pass@4": 0.570342090403114,
1469
+ "pass@8": 0.6477507011049768,
1470
+ "pass@16": 0.7086113239762887,
1471
+ "pass@32": 0.7532795279937846,
1472
+ "pass@64": 0.7862179025168473,
1473
+ "pass@128": 0.8108108108108109
1474
+ }
1475
+ }
1476
+ },
1477
+ "teachers_in_school": {
1478
+ "count": 821120,
1479
+ "correct": 651414,
1480
+ "answer_accuracy": 0.7933237529228371,
1481
+ "avg_response_len": 286.8824020849571,
1482
+ "resp_tokens_sum": 235564878,
1483
+ "pass_at_k": {
1484
+ "pass@1": 0.7933237529228371,
1485
+ "pass@2": 0.8389930634401326,
1486
+ "pass@4": 0.8726874711442432,
1487
+ "pass@8": 0.8977972002351476,
1488
+ "pass@16": 0.9167457991803611,
1489
+ "pass@32": 0.9314659835787653,
1490
+ "pass@64": 0.9432935185698375,
1491
+ "pass@128": 0.9522992985190959
1492
+ },
1493
+ "per_op_pass_at_k": {
1494
+ "10": {
1495
+ "pass@1": 0.9164350076687117,
1496
+ "pass@2": 0.9361730713492108,
1497
+ "pass@4": 0.9499516876916461,
1498
+ "pass@8": 0.9618509716297367,
1499
+ "pass@16": 0.9718630754382047,
1500
+ "pass@32": 0.9796931696551145,
1501
+ "pass@64": 0.986608430448021,
1502
+ "pass@128": 0.99079754601227
1503
+ },
1504
+ "2": {
1505
+ "pass@1": 1.0,
1506
+ "pass@2": 1.0,
1507
+ "pass@4": 1.0,
1508
+ "pass@8": 1.0,
1509
+ "pass@16": 1.0,
1510
+ "pass@32": 1.0,
1511
+ "pass@64": 1.0,
1512
+ "pass@128": 1.0
1513
+ },
1514
+ "3": {
1515
+ "pass@1": 0.9973958333333334,
1516
+ "pass@2": 0.9992700131233596,
1517
+ "pass@4": 0.9999176963674994,
1518
+ "pass@8": 0.9999987979293221,
1519
+ "pass@16": 0.9999999998399193,
1520
+ "pass@32": 1.0,
1521
+ "pass@64": 1.0,
1522
+ "pass@128": 1.0
1523
+ },
1524
+ "4": {
1525
+ "pass@1": 0.9990292159763313,
1526
+ "pass@2": 0.9998897084517542,
1527
+ "pass@4": 0.9999981377142946,
1528
+ "pass@8": 0.9999999995762411,
1529
+ "pass@16": 1.0,
1530
+ "pass@32": 1.0,
1531
+ "pass@64": 1.0,
1532
+ "pass@128": 1.0
1533
+ },
1534
+ "5": {
1535
+ "pass@1": 0.980570652173913,
1536
+ "pass@2": 0.9865802949903003,
1537
+ "pass@4": 0.9889661506985535,
1538
+ "pass@8": 0.9902841199998196,
1539
+ "pass@16": 0.9917663843037684,
1540
+ "pass@32": 0.9936594509266741,
1541
+ "pass@64": 0.9954604899202021,
1542
+ "pass@128": 0.9971014492753624
1543
+ },
1544
+ "6": {
1545
+ "pass@1": 0.995008058739255,
1546
+ "pass@2": 0.9963252713038379,
1547
+ "pass@4": 0.9970167348350797,
1548
+ "pass@8": 0.9973006814242409,
1549
+ "pass@16": 0.997492802179556,
1550
+ "pass@32": 0.9978510028653035,
1551
+ "pass@64": 0.998567335243553,
1552
+ "pass@128": 1.0
1553
+ },
1554
+ "7": {
1555
+ "pass@1": 0.9898843930635838,
1556
+ "pass@2": 0.9931856196804879,
1557
+ "pass@4": 0.9941058232063475,
1558
+ "pass@8": 0.9943886704848257,
1559
+ "pass@16": 0.9945808983691612,
1560
+ "pass@32": 0.9949421965317783,
1561
+ "pass@64": 0.9956647398843931,
1562
+ "pass@128": 0.9971098265895953
1563
+ },
1564
+ "8": {
1565
+ "pass@1": 0.9714788732394366,
1566
+ "pass@2": 0.980618207275147,
1567
+ "pass@4": 0.9870130438800786,
1568
+ "pass@8": 0.9915782021625016,
1569
+ "pass@16": 0.9946980159453276,
1570
+ "pass@32": 0.9967270976372445,
1571
+ "pass@64": 0.9986017780742946,
1572
+ "pass@128": 1.0
1573
+ },
1574
+ "9": {
1575
+ "pass@1": 0.9722352170658682,
1576
+ "pass@2": 0.9828459468857563,
1577
+ "pass@4": 0.9884671896364751,
1578
+ "pass@8": 0.9924020194851084,
1579
+ "pass@16": 0.9944314239658919,
1580
+ "pass@32": 0.9953209435293606,
1581
+ "pass@64": 0.9962633785477046,
1582
+ "pass@128": 0.9970059880239521
1583
+ },
1584
+ "11": {
1585
+ "pass@1": 0.9257341867469879,
1586
+ "pass@2": 0.9553495725026092,
1587
+ "pass@4": 0.9682735500494665,
1588
+ "pass@8": 0.9742548441493746,
1589
+ "pass@16": 0.978185160269226,
1590
+ "pass@32": 0.9822869779847758,
1591
+ "pass@64": 0.9871868036674503,
1592
+ "pass@128": 0.9909638554216867
1593
+ },
1594
+ "12": {
1595
+ "pass@1": 0.8492358934169278,
1596
+ "pass@2": 0.8949850973761508,
1597
+ "pass@4": 0.9207448059000464,
1598
+ "pass@8": 0.9354093985604018,
1599
+ "pass@16": 0.943759425744798,
1600
+ "pass@32": 0.9496060392426146,
1601
+ "pass@64": 0.9561026755224603,
1602
+ "pass@128": 0.9655172413793104
1603
+ },
1604
+ "13": {
1605
+ "pass@1": 0.740983893557423,
1606
+ "pass@2": 0.8052852538653259,
1607
+ "pass@4": 0.845815909040781,
1608
+ "pass@8": 0.8738550513505369,
1609
+ "pass@16": 0.895515341616072,
1610
+ "pass@32": 0.9155451542423412,
1611
+ "pass@64": 0.9353134265154689,
1612
+ "pass@128": 0.9523809523809523
1613
+ },
1614
+ "14": {
1615
+ "pass@1": 0.7266081871345029,
1616
+ "pass@2": 0.7990618668554583,
1617
+ "pass@4": 0.8478731022109073,
1618
+ "pass@8": 0.8837960270998939,
1619
+ "pass@16": 0.9105606572428289,
1620
+ "pass@32": 0.930775272521883,
1621
+ "pass@64": 0.9463526228698942,
1622
+ "pass@128": 0.9590643274853801
1623
+ },
1624
+ "15": {
1625
+ "pass@1": 0.6870501893939394,
1626
+ "pass@2": 0.7726907361011697,
1627
+ "pass@4": 0.8339367397257161,
1628
+ "pass@8": 0.8770493599523325,
1629
+ "pass@16": 0.9072087108288012,
1630
+ "pass@32": 0.9277764122574453,
1631
+ "pass@64": 0.9424641653105227,
1632
+ "pass@128": 0.9545454545454546
1633
+ },
1634
+ "16": {
1635
+ "pass@1": 0.5663728632478633,
1636
+ "pass@2": 0.668794655876349,
1637
+ "pass@4": 0.74511328258113,
1638
+ "pass@8": 0.8007343935830623,
1639
+ "pass@16": 0.8421667071713081,
1640
+ "pass@32": 0.8733083123762971,
1641
+ "pass@64": 0.8957480080042212,
1642
+ "pass@128": 0.9116809116809117
1643
+ },
1644
+ "17": {
1645
+ "pass@1": 0.5376067546583851,
1646
+ "pass@2": 0.6423940553626447,
1647
+ "pass@4": 0.7202086238637875,
1648
+ "pass@8": 0.7743041381948508,
1649
+ "pass@16": 0.814345713501841,
1650
+ "pass@32": 0.8470088033255796,
1651
+ "pass@64": 0.8726913345902738,
1652
+ "pass@128": 0.8881987577639752
1653
+ },
1654
+ "18": {
1655
+ "pass@1": 0.5092830882352941,
1656
+ "pass@2": 0.6142293162343674,
1657
+ "pass@4": 0.7005960897351063,
1658
+ "pass@8": 0.7701591745990866,
1659
+ "pass@16": 0.8221200586922632,
1660
+ "pass@32": 0.8587938588657354,
1661
+ "pass@64": 0.8857339251123276,
1662
+ "pass@128": 0.9058823529411765
1663
+ },
1664
+ "19": {
1665
+ "pass@1": 0.34770275297619047,
1666
+ "pass@2": 0.44574237790588667,
1667
+ "pass@4": 0.5338633851014157,
1668
+ "pass@8": 0.611502029947303,
1669
+ "pass@16": 0.6783513518981501,
1670
+ "pass@32": 0.7327937015644227,
1671
+ "pass@64": 0.7742047877222114,
1672
+ "pass@128": 0.8035714285714286
1673
+ },
1674
+ "20": {
1675
+ "pass@1": 0.35043974550898205,
1676
+ "pass@2": 0.4594935375312366,
1677
+ "pass@4": 0.5528386059714089,
1678
+ "pass@8": 0.6238549348742458,
1679
+ "pass@16": 0.6762174612820911,
1680
+ "pass@32": 0.7171414832463683,
1681
+ "pass@64": 0.7511741428622221,
1682
+ "pass@128": 0.7754491017964071
1683
+ }
1684
+ }
1685
+ },
1686
+ "movie_festival_awards": {
1687
+ "count": 801152,
1688
+ "correct": 634704,
1689
+ "answer_accuracy": 0.7922391755871545,
1690
+ "avg_response_len": 307.75241027919793,
1691
+ "resp_tokens_sum": 246556459,
1692
+ "pass_at_k": {
1693
+ "pass@1": 0.7922391755871545,
1694
+ "pass@2": 0.8386018031672131,
1695
+ "pass@4": 0.8729426678002086,
1696
+ "pass@8": 0.8980563358146915,
1697
+ "pass@16": 0.9163577796796314,
1698
+ "pass@32": 0.9303137674409322,
1699
+ "pass@64": 0.9412975982929933,
1700
+ "pass@128": 0.9496724716408372
1701
+ },
1702
+ "per_op_pass_at_k": {
1703
+ "10": {
1704
+ "pass@1": 0.933535447761194,
1705
+ "pass@2": 0.9610144405923143,
1706
+ "pass@4": 0.9764659397052976,
1707
+ "pass@8": 0.9848856698483921,
1708
+ "pass@16": 0.9903880371665308,
1709
+ "pass@32": 0.994788130439683,
1710
+ "pass@64": 0.9981423348247224,
1711
+ "pass@128": 1.0
1712
+ },
1713
+ "2": {
1714
+ "pass@1": 1.0,
1715
+ "pass@2": 1.0,
1716
+ "pass@4": 1.0,
1717
+ "pass@8": 1.0,
1718
+ "pass@16": 1.0,
1719
+ "pass@32": 1.0,
1720
+ "pass@64": 1.0,
1721
+ "pass@128": 1.0
1722
+ },
1723
+ "3": {
1724
+ "pass@1": 0.9961962463556852,
1725
+ "pass@2": 0.9970163965473705,
1726
+ "pass@4": 0.997338775072941,
1727
+ "pass@8": 0.9976015064053606,
1728
+ "pass@16": 0.9980534958639686,
1729
+ "pass@32": 0.9987797625879855,
1730
+ "pass@64": 0.9996441771309199,
1731
+ "pass@128": 1.0
1732
+ },
1733
+ "4": {
1734
+ "pass@1": 0.9991776315789473,
1735
+ "pass@2": 0.999924094603306,
1736
+ "pass@4": 0.9999989030976391,
1737
+ "pass@8": 0.999999999845404,
1738
+ "pass@16": 1.0,
1739
+ "pass@32": 1.0,
1740
+ "pass@64": 1.0,
1741
+ "pass@128": 1.0
1742
+ },
1743
+ "5": {
1744
+ "pass@1": 0.9750229779411764,
1745
+ "pass@2": 0.9829337222093564,
1746
+ "pass@4": 0.9882893535366903,
1747
+ "pass@8": 0.9921605412616765,
1748
+ "pass@16": 0.9948705139434515,
1749
+ "pass@32": 0.9969973900489489,
1750
+ "pass@64": 0.998910627773521,
1751
+ "pass@128": 1.0
1752
+ },
1753
+ "6": {
1754
+ "pass@1": 0.9884660883280757,
1755
+ "pass@2": 0.9913928407312652,
1756
+ "pass@4": 0.9938846173408135,
1757
+ "pass@8": 0.9958811876044368,
1758
+ "pass@16": 0.9974790678017365,
1759
+ "pass@32": 0.9989033586934598,
1760
+ "pass@64": 0.9998111525900316,
1761
+ "pass@128": 1.0
1762
+ },
1763
+ "7": {
1764
+ "pass@1": 0.9794056566455697,
1765
+ "pass@2": 0.9909034560948872,
1766
+ "pass@4": 0.9964178719155359,
1767
+ "pass@8": 0.9984878997029001,
1768
+ "pass@16": 0.9994102329414561,
1769
+ "pass@32": 0.9999166822897716,
1770
+ "pass@64": 0.9999995619546246,
1771
+ "pass@128": 1.0
1772
+ },
1773
+ "8": {
1774
+ "pass@1": 0.9842342342342343,
1775
+ "pass@2": 0.9942238005722256,
1776
+ "pass@4": 0.9981784532001067,
1777
+ "pass@8": 0.9995914077213195,
1778
+ "pass@16": 0.9999633430821163,
1779
+ "pass@32": 0.9999998023813371,
1780
+ "pass@64": 0.9999999999999107,
1781
+ "pass@128": 1.0
1782
+ },
1783
+ "9": {
1784
+ "pass@1": 0.9692021704180064,
1785
+ "pass@2": 0.983175937412968,
1786
+ "pass@4": 0.9883279185158123,
1787
+ "pass@8": 0.9908531152358594,
1788
+ "pass@16": 0.9925946265968474,
1789
+ "pass@32": 0.9942363552283263,
1790
+ "pass@64": 0.9958943717110796,
1791
+ "pass@128": 0.9967845659163987
1792
+ },
1793
+ "11": {
1794
+ "pass@1": 0.915327380952381,
1795
+ "pass@2": 0.948085786151731,
1796
+ "pass@4": 0.9642386070788763,
1797
+ "pass@8": 0.9739282351224063,
1798
+ "pass@16": 0.9804970648118635,
1799
+ "pass@32": 0.9846325349921398,
1800
+ "pass@64": 0.986722881877331,
1801
+ "pass@128": 0.9873015873015873
1802
+ },
1803
+ "12": {
1804
+ "pass@1": 0.8619313837920489,
1805
+ "pass@2": 0.912293969274483,
1806
+ "pass@4": 0.9437201129675306,
1807
+ "pass@8": 0.9619229572767756,
1808
+ "pass@16": 0.9731904957944275,
1809
+ "pass@32": 0.9818321050690701,
1810
+ "pass@64": 0.9884511684407877,
1811
+ "pass@128": 0.9908256880733946
1812
+ },
1813
+ "13": {
1814
+ "pass@1": 0.7652874228395061,
1815
+ "pass@2": 0.8321079548216199,
1816
+ "pass@4": 0.8720276435352986,
1817
+ "pass@8": 0.896107615787007,
1818
+ "pass@16": 0.9136146173645132,
1819
+ "pass@32": 0.929635398081815,
1820
+ "pass@64": 0.9451059800569461,
1821
+ "pass@128": 0.9567901234567902
1822
+ },
1823
+ "14": {
1824
+ "pass@1": 0.7490031528189911,
1825
+ "pass@2": 0.8191228913058717,
1826
+ "pass@4": 0.864718364024971,
1827
+ "pass@8": 0.8938565907497251,
1828
+ "pass@16": 0.9139148391085333,
1829
+ "pass@32": 0.9293839114599252,
1830
+ "pass@64": 0.9403161156625768,
1831
+ "pass@128": 0.9465875370919882
1832
+ },
1833
+ "15": {
1834
+ "pass@1": 0.6787471064814815,
1835
+ "pass@2": 0.7596807560513266,
1836
+ "pass@4": 0.8202685196757814,
1837
+ "pass@8": 0.8636306777109823,
1838
+ "pass@16": 0.8919877814582083,
1839
+ "pass@32": 0.9093606678534814,
1840
+ "pass@64": 0.9216089042968235,
1841
+ "pass@128": 0.9320987654320988
1842
+ },
1843
+ "16": {
1844
+ "pass@1": 0.5669478244837758,
1845
+ "pass@2": 0.6690786646691279,
1846
+ "pass@4": 0.7455410565382864,
1847
+ "pass@8": 0.8028333423672143,
1848
+ "pass@16": 0.8423557007877164,
1849
+ "pass@32": 0.8684380664265011,
1850
+ "pass@64": 0.8872958659682277,
1851
+ "pass@128": 0.9026548672566371
1852
+ },
1853
+ "17": {
1854
+ "pass@1": 0.5101891942771084,
1855
+ "pass@2": 0.6125037798833123,
1856
+ "pass@4": 0.6895920137241878,
1857
+ "pass@8": 0.7465083545933107,
1858
+ "pass@16": 0.7900024434338143,
1859
+ "pass@32": 0.8252868759565376,
1860
+ "pass@64": 0.85326005071428,
1861
+ "pass@128": 0.8734939759036144
1862
+ },
1863
+ "18": {
1864
+ "pass@1": 0.4936743951612903,
1865
+ "pass@2": 0.5874368967487934,
1866
+ "pass@4": 0.6604457161604794,
1867
+ "pass@8": 0.7140707542442847,
1868
+ "pass@16": 0.7545067071870171,
1869
+ "pass@32": 0.7888352910327632,
1870
+ "pass@64": 0.8177670834201991,
1871
+ "pass@128": 0.8419354838709677
1872
+ },
1873
+ "19": {
1874
+ "pass@1": 0.3340728851963746,
1875
+ "pass@2": 0.4338164890453644,
1876
+ "pass@4": 0.5283597530860003,
1877
+ "pass@8": 0.6090080248672055,
1878
+ "pass@16": 0.6734085647143847,
1879
+ "pass@32": 0.7271711226701715,
1880
+ "pass@64": 0.7742411529076022,
1881
+ "pass@128": 0.8157099697885196
1882
+ },
1883
+ "20": {
1884
+ "pass@1": 0.34025431681681684,
1885
+ "pass@2": 0.45003221725662673,
1886
+ "pass@4": 0.5509912418042338,
1887
+ "pass@8": 0.6353882561628813,
1888
+ "pass@16": 0.6987590181595222,
1889
+ "pass@32": 0.7426487056681589,
1890
+ "pass@64": 0.7730709505896831,
1891
+ "pass@128": 0.7957957957957958
1892
+ }
1893
+ }
1894
+ }
1895
+ }
1896
+ }
1897
+ }
results/composition/SDAR-100M-block4-epoch1-op17-20/SDAR-100M-block4-epoch1-op17-20_ood_generations.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12ecad35f2fb9b4f11bfcab65b3efa37ec9d2ad82e325202191c412126f98d71
3
+ size 2707181240
results/composition/SDAR-100M-block4-epoch1-op17-20/summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
2
+ SDAR-100M-block4-epoch1-op17-20,0.9801223958333334,0.62273671875,0.7920246710526315,0.0,0.0,0.0,199.76062586805557,372.483859375,1152000,1280000,2432000